From 89096e8ff598b9a7ad0e04146870b66af60d844e Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 5 May 2026 22:06:15 +0200 Subject: [PATCH 1/7] CLDSRV-898: validate per-part checksums and x-amz-checksum-type --- .../apiUtils/integrity/validateChecksums.js | 4 +- lib/api/completeMultipartUpload.js | 112 ++++ tests/unit/api/completeMultipartUpload.js | 502 ++++++++++++++++++ 3 files changed, 617 insertions(+), 1 deletion(-) create mode 100644 tests/unit/api/completeMultipartUpload.js diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index d2f7059eb8..7fb987ab18 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -40,7 +40,9 @@ const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription ); const checksumedMethods = Object.freeze({ - completeMultipartUpload: true, + // CompleteMPU's x-amz-checksum- is the final-object checksum, + // not a body digest. Validated in completeMultipartUpload.js instead. + // 'completeMultipartUpload': true, multiObjectDelete: true, bucketPutACL: true, bucketPutCors: true, diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 581344223b..5c82a2132a 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -26,12 +26,88 @@ const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); +const { algorithms: checksumAlgorithms } = require('./apiUtils/integrity/validateChecksums'); const versionIdUtils = versioning.VersionID; let splitter = constants.splitter; const REPLICATION_ACTION = 'MPU'; +const allChecksumXmlTags = Object.values(checksumAlgorithms).map(algo => algo.xmlTag); + +/** + * Validate per-part checksums in a CompleteMultipartUpload request body + * against the stored MPU configuration and stored part metadata. + * + * Rules (per AWS): + * - If a Part element includes a Checksum field that does not match the + * MPU's configured checksumAlgorithm, return BadDigest. + * - If a Part element includes the matching Checksum field but the value + * does not match the stored part's ChecksumValue, return InvalidPart. + * - If checksumType === 'COMPOSITE' and checksumIsDefault is false, every part + * in the request body MUST include the matching Checksum field; + * missing → InvalidRequest. + * + * @param {object} jsonList - parsed CompleteMultipartUpload XML + * @param {array} storedParts - parts as returned by services.getMPUparts + * @param {string} mpuSplitter - splitter used in part keys + * @param {object} mpuChecksum - { algorithm, type, isDefault } + * @returns {Error|null} + */ +function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksum) { + const mpuAlgo = mpuChecksum.algorithm; + if (!mpuAlgo) { + // Legacy / pre-checksums MPU, no algorithm tracked, nothing to validate. + return null; + } + const expectedTag = checksumAlgorithms[mpuAlgo] ? checksumAlgorithms[mpuAlgo].xmlTag : null; + // Skip enforcement if the MPU's algorithm is unknown (shouldn't happen). + const requireForEachPart = mpuChecksum.type === 'COMPOSITE' && !mpuChecksum.isDefault && expectedTag !== null; + + const storedByPartNumber = new Map(); + storedParts.forEach(item => { + const partNumber = Number.parseInt(item.key.split(mpuSplitter)[1], 10); + storedByPartNumber.set(partNumber, item); + }); + + const parts = jsonList.Part || []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const partNumber = Number.parseInt(part.PartNumber[0], 10); + + const presentTags = allChecksumXmlTags.filter(tag => part[tag]); + + for (const tag of presentTags) { + if (tag !== expectedTag) { + const algoLabel = tag.replace(/^Checksum/, '').toLowerCase(); + return errorInstances.BadDigest.customizeDescription( + `The ${algoLabel} you specified for part ${partNumber} ` + 'did not match what we received.', + ); + } + } + + if (expectedTag && presentTags.includes(expectedTag)) { + const providedValue = part[expectedTag][0]; + const storedPart = storedByPartNumber.get(partNumber); + const storedValue = storedPart && storedPart.value && storedPart.value.ChecksumValue; + if (!storedValue || providedValue !== storedValue) { + return errorInstances.InvalidPart.customizeDescription( + 'One or more of the specified parts could not be found. ' + + 'The part may not have been uploaded, or the specified ' + + "entity tag may not match the part's entity tag.", + ); + } + } else if (requireForEachPart) { + return errorInstances.InvalidRequest.customizeDescription( + `The upload was created using a ${mpuAlgo} checksum. ` + + 'The complete request must include the checksum for each ' + + `part. It was missing for part ${partNumber} in the request.`, + ); + } + } + return null; +} + /* Format of xml request: @@ -159,6 +235,29 @@ function completeMultipartUpload(authInfo, request, log, callback) { log.error('error validating request', { error: err }); return next(err, destBucket); } + // Validate x-amz-checksum-type header (if present) matches + // the checksum type the MPU was created with. + // x-amz-checksum-algorithm is not validated: AWS ignores + // a mismatch on this header for CompleteMultipartUpload. + const headerType = request.headers['x-amz-checksum-type']; + if (headerType) { + const headerTypeUpper = headerType.toUpperCase(); + if (headerTypeUpper !== 'COMPOSITE' && headerTypeUpper !== 'FULL_OBJECT') { + const typeErr = errorInstances.InvalidRequest.customizeDescription( + 'Value for x-amz-checksum-type header is invalid.', + ); + return next(typeErr, destBucket); + } + const mpuType = storedMetadata.checksumType; + if (!mpuType || headerTypeUpper !== mpuType.toUpperCase()) { + const typeErr = errorInstances.InvalidRequest.customizeDescription( + `The upload was created using the ${mpuType} ` + + 'checksum mode. The complete request must ' + + 'use the same checksum mode.', + ); + return next(typeErr, destBucket); + } + } return next(null, destBucket, objMD, mpuBucket, storedMetadata); }, ); @@ -251,6 +350,18 @@ function completeMultipartUpload(authInfo, request, log, callback) { } const storedParts = result.Contents; const totalMPUSize = storedParts.reduce((acc, part) => acc + part.value.Size, 0); + const mpuChecksum = { + algorithm: storedMetadata.checksumAlgorithm, + type: storedMetadata.checksumType, + isDefault: storedMetadata.checksumIsDefault, + }; + const checksumErr = validatePerPartChecksums(jsonList, storedParts, splitter, mpuChecksum); + if (checksumErr) { + log.debug('per-part checksum validation failed', { + error: checksumErr, + }); + return next(checksumErr, destBucket); + } return next( null, destBucket, @@ -897,3 +1008,4 @@ function completeMultipartUpload(authInfo, request, log, callback) { } module.exports = completeMultipartUpload; +module.exports.validatePerPartChecksums = validatePerPartChecksums; diff --git a/tests/unit/api/completeMultipartUpload.js b/tests/unit/api/completeMultipartUpload.js new file mode 100644 index 0000000000..ae4f9ad05d --- /dev/null +++ b/tests/unit/api/completeMultipartUpload.js @@ -0,0 +1,502 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const async = require('async'); +const { parseString } = require('xml2js'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const objectPutPart = require('../../../lib/api/objectPutPart'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); +const { validatePerPartChecksums } = completeMultipartUpload; +const { validateMethodChecksumNoChunking } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const DummyRequest = require('../DummyRequest'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); + +const SPLITTER = '..|..'; +const UPLOAD_ID = 'upload-id-1'; + +// XML element name AWS uses for each algorithm in CompleteMultipartUpload's +// per-part body. +const TAG_BY_ALGO = { + crc32: 'ChecksumCRC32', + crc32c: 'ChecksumCRC32C', + crc64nvme: 'ChecksumCRC64NVME', + sha1: 'ChecksumSHA1', + sha256: 'ChecksumSHA256', +}; + +// Two distinct base64 placeholder digests per algorithm. Sized to the real +// digest lengths so the test data looks realistic, though the validator +// itself doesn't enforce length. +const SAMPLE_DIGESTS = { + crc32: ['AQIDBA==', 'BQYHCA=='], + crc32c: ['CQoLDA==', 'DQ4PEA=='], + crc64nvme: ['AQIDBAUGBwg=', 'CQoLDA0ODxA='], + sha1: ['YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=', 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI='], + sha256: ['YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=', 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI='], +}; + +// Every AWS-valid (algorithm, type) combination, plus the implicit default. +// See validateChecksums.getChecksumDataFromMPUHeaders for the source of truth. +const MATRIX = [ + { algorithm: 'crc32', type: 'COMPOSITE', isDefault: false }, + { algorithm: 'crc32', type: 'FULL_OBJECT', isDefault: false }, + { algorithm: 'crc32c', type: 'COMPOSITE', isDefault: false }, + { algorithm: 'crc32c', type: 'FULL_OBJECT', isDefault: false }, + { algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: false }, + { algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: true }, + { algorithm: 'sha1', type: 'COMPOSITE', isDefault: false }, + { algorithm: 'sha256', type: 'COMPOSITE', isDefault: false }, +]; + +function makeStoredPart(partNumber, checksum) { + const value = { + ETag: 'd41d8cd98f00b204e9800998ecf8427e', + Size: 5242880, + partLocations: [{ key: `data-${partNumber}`, dataStoreName: 'us-east-1' }], + }; + if (checksum) { + value.ChecksumAlgorithm = checksum.algorithm; + value.ChecksumValue = checksum.value; + } + return { + key: `${UPLOAD_ID}${SPLITTER}${partNumber}`, + value, + }; +} + +function makeJsonPart(partNumber, eTag, checksums) { + const part = { + PartNumber: [String(partNumber)], + ETag: [`"${eTag}"`], + }; + if (checksums) { + Object.entries(checksums).forEach(([tag, value]) => { + part[tag] = [value]; + }); + } + return part; +} + +function pickWrongAlgo(algo) { + return Object.keys(TAG_BY_ALGO).find(a => a !== algo); +} + +describe('validatePerPartChecksums', () => { + describe('AWS combination matrix', () => { + MATRIX.forEach(({ algorithm, type, isDefault }) => { + const label = `${algorithm}/${type}${isDefault ? ' (default)' : ''}`; + const tag = TAG_BY_ALGO[algorithm]; + const [d1, d2] = SAMPLE_DIGESTS[algorithm]; + const mpuChecksum = { algorithm, type, isDefault }; + + const stored = [makeStoredPart(1, { algorithm, value: d1 }), makeStoredPart(2, { algorithm, value: d2 })]; + + describe(label, () => { + it('should accept when every part includes the matching checksum', () => { + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { [tag]: d1 }), makeJsonPart(2, 'etag2', { [tag]: d2 })], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should return BadDigest when a part uses the wrong checksum field', () => { + const wrongAlgo = pickWrongAlgo(algorithm); + const wrongTag = TAG_BY_ALGO[wrongAlgo]; + const wrongDigest = SAMPLE_DIGESTS[wrongAlgo][0]; + const jsonList = { + Part: [ + makeJsonPart(1, 'etag1', { [wrongTag]: wrongDigest }), + makeJsonPart(2, 'etag2', { [tag]: d2 }), + ], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert(err); + assert.strictEqual(err.is.BadDigest, true); + // AWS-style message: "The {algo} you specified for part {N} did not match what we received." + assert.strictEqual( + err.description, + `The ${wrongAlgo} you specified for part 1 did ` + 'not match what we received.', + ); + }); + + it('should return InvalidPart when the matching field has the wrong value', () => { + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { [tag]: d1 }), makeJsonPart(2, 'etag2', { [tag]: d1 })], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert(err); + assert.strictEqual(err.is.InvalidPart, true); + // AWS reuses its generic InvalidPart message — no algorithm + // or part number in the wording. + assert.strictEqual( + err.description, + 'One or more of the specified parts could not be ' + + 'found. The part may not have been uploaded, or ' + + 'the specified entity tag may not match the ' + + "part's entity tag.", + ); + }); + + const requiresPerPart = type === 'COMPOSITE' && !isDefault; + const missingLabel = requiresPerPart + ? 'should return InvalidRequest when a part is missing its checksum' + : 'should accept a parts list missing per-part checksums'; + it(missingLabel, () => { + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { [tag]: d1 }), makeJsonPart(2, 'etag2')], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + if (requiresPerPart) { + assert(err); + assert.strictEqual(err.is.InvalidRequest, true); + assert(err.description.includes(algorithm)); + assert(err.description.includes('part 2 in the request')); + } else { + assert.strictEqual(err, null); + } + }); + }); + }); + }); + + describe('legacy MPU (no algorithm configured)', () => { + // Pre-feature MPUs have storedMetadata.checksumAlgorithm === undefined. + // Pre-PR CompleteMPU silently ignored any per-part Checksum body + // elements; preserve that so in-flight uploads across the upgrade + // boundary don't start failing with BadDigest. + const mpuChecksum = { algorithm: undefined, type: undefined, isDefault: undefined }; + const stored = [makeStoredPart(1), makeStoredPart(2)]; + + it('should accept when no parts include a checksum field', () => { + const jsonList = { Part: [makeJsonPart(1, 'etag1'), makeJsonPart(2, 'etag2')] }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should accept when a part includes a single Checksum field', () => { + const jsonList = { + Part: [ + makeJsonPart(1, 'etag1', { ChecksumSHA256: SAMPLE_DIGESTS.sha256[0] }), + makeJsonPart(2, 'etag2'), + ], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should accept when parts include multiple Checksum fields', () => { + const jsonList = { + Part: [ + makeJsonPart(1, 'etag1', { + ChecksumSHA256: SAMPLE_DIGESTS.sha256[0], + ChecksumCRC32: SAMPLE_DIGESTS.crc32[0], + }), + makeJsonPart(2, 'etag2', { ChecksumCRC64NVME: SAMPLE_DIGESTS.crc64nvme[1] }), + ], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + }); + describe('edge cases', () => { + it('should accept an empty parts list', () => { + const mpuChecksum = { + algorithm: 'sha256', + type: 'COMPOSITE', + isDefault: false, + }; + const err = validatePerPartChecksums({ Part: [] }, [], SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should accept a parts list with no Part array (treated as empty)', () => { + const mpuChecksum = { + algorithm: 'crc64nvme', + type: 'FULL_OBJECT', + isDefault: true, + }; + const err = validatePerPartChecksums({}, [], SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should accept a FULL_OBJECT mixed list (one part with checksum, one without)', () => { + const mpuChecksum = { + algorithm: 'crc32', + type: 'FULL_OBJECT', + isDefault: false, + }; + const [d1, d2] = SAMPLE_DIGESTS.crc32; + const stored = [ + makeStoredPart(1, { algorithm: 'crc32', value: d1 }), + makeStoredPart(2, { algorithm: 'crc32', value: d2 }), + ]; + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { ChecksumCRC32: d1 }), makeJsonPart(2, 'etag2')], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should not enforce per-part presence when MPU algorithm is unknown', () => { + // CreateMPU should never let this state through, but guard against + // an "InvalidRequest: using a undefined checksum" error if it did. + const mpuChecksum = { + algorithm: undefined, + type: 'COMPOSITE', + isDefault: false, + }; + const stored = [makeStoredPart(1, null), makeStoredPart(2, null)]; + const jsonList = { + Part: [makeJsonPart(1, 'etag1'), makeJsonPart(2, 'etag2')], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should return InvalidPart when stored part has no checksum but request does', () => { + const mpuChecksum = { + algorithm: 'sha256', + type: 'COMPOSITE', + isDefault: false, + }; + const stored = [makeStoredPart(1, null)]; + const jsonList = { + Part: [ + makeJsonPart(1, 'etag1', { + ChecksumSHA256: SAMPLE_DIGESTS.sha256[0], + }), + ], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert(err); + assert.strictEqual(err.is.InvalidPart, true); + }); + }); +}); + +describe('CompleteMultipartUpload x-amz-checksum-type header', () => { + const log = new DummyRequestLogger(); + const authInfo = makeAuthInfo('accessKey1'); + const namespace = 'default'; + const bucketName = 'bucketname-checksum-type'; + const objectKey = 'testObject'; + + const bucketPutRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + 'scality-internal-mem' + + '', + actionImplicitDenies: false, + }; + + function setupMpu(initiateHeaders, cb) { + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => { + const initiateRequest = { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + ...initiateHeaders, + }, + url: `/${objectKey}?uploads`, + actionImplicitDenies: false, + }; + initiateMultipartUpload(authInfo, initiateRequest, log, next); + }, + (xml, corsHeaders, next) => parseString(xml, next), + (json, next) => { + const uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partBody = Buffer.from('I am a part\n', 'utf8'); + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { partNumber: '1', uploadId }, + partHash, + actionImplicitDenies: false, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => next(err, uploadId, partHash)); + }, + ], + cb, + ); + } + + function makeCompleteRequest(uploadId, partHash, extraHeaders) { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"` + + '' + + ''; + return { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + ...extraHeaders, + }, + query: { uploadId }, + post: completeBody, + actionImplicitDenies: false, + }; + } + + beforeEach(() => cleanup()); + + it('should accept CompleteMPU when no x-amz-checksum-type header is sent', done => { + const initiateHeaders = { + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }; + setupMpu(initiateHeaders, (err, uploadId, partHash) => { + assert.ifError(err); + const req = makeCompleteRequest(uploadId, partHash, {}); + completeMultipartUpload(authInfo, req, log, completeErr => { + assert.ifError(completeErr); + done(); + }); + }); + }); + + it('should accept CompleteMPU when x-amz-checksum-type matches the MPU type', done => { + const initiateHeaders = { + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }; + setupMpu(initiateHeaders, (err, uploadId, partHash) => { + assert.ifError(err); + const req = makeCompleteRequest(uploadId, partHash, { + 'x-amz-checksum-type': 'FULL_OBJECT', + }); + completeMultipartUpload(authInfo, req, log, completeErr => { + assert.ifError(completeErr); + done(); + }); + }); + }); + + it('should reject CompleteMPU with InvalidRequest when x-amz-checksum-type does not match the MPU type', done => { + const initiateHeaders = { + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }; + setupMpu(initiateHeaders, (err, uploadId, partHash) => { + assert.ifError(err); + const req = makeCompleteRequest(uploadId, partHash, { + 'x-amz-checksum-type': 'COMPOSITE', + }); + completeMultipartUpload(authInfo, req, log, completeErr => { + assert(completeErr); + assert.strictEqual(completeErr.is.InvalidRequest, true); + // AWS-style mode-mismatch wording. + assert.strictEqual( + completeErr.description, + 'The upload was created using the FULL_OBJECT checksum ' + + 'mode. The complete request must use the same checksum ' + + 'mode.', + ); + done(); + }); + }); + }); + + it('should reject CompleteMPU with InvalidRequest when x-amz-checksum-type value is bogus', done => { + const initiateHeaders = { + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }; + setupMpu(initiateHeaders, (err, uploadId, partHash) => { + assert.ifError(err); + const req = makeCompleteRequest(uploadId, partHash, { + 'x-amz-checksum-type': 'BOGUS', + }); + completeMultipartUpload(authInfo, req, log, completeErr => { + assert(completeErr); + assert.strictEqual(completeErr.is.InvalidRequest, true); + assert.strictEqual(completeErr.description, 'Value for x-amz-checksum-type header is invalid.'); + done(); + }); + }); + }); + + it('should compare x-amz-checksum-type case-insensitively', done => { + const initiateHeaders = { + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }; + setupMpu(initiateHeaders, (err, uploadId, partHash) => { + assert.ifError(err); + const req = makeCompleteRequest(uploadId, partHash, { + 'x-amz-checksum-type': 'full_object', + }); + completeMultipartUpload(authInfo, req, log, completeErr => { + assert.ifError(completeErr); + done(); + }); + }); + }); +}); + +describe('CompleteMultipartUpload body-checksum bypass', () => { + const log = new DummyRequestLogger(); + + it( + 'validateMethodChecksumNoChunking returns null for completeMultipartUpload ' + + 'even when x-amz-checksum-sha256 does not match the body digest', + async () => { + const body = Buffer.from( + '1' + + '"abc"', + ); + // A syntactically valid SHA256 base64 digest that is NOT the digest of `body` + // (it's the digest of the empty string). On CompleteMPU this header carries + // the expected final-object checksum, not a body checksum, so pre-validation + // must skip it. + const finalObjectChecksum = '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; + const request = { + apiMethod: 'completeMultipartUpload', + headers: { 'x-amz-checksum-sha256': finalObjectChecksum }, + }; + const err = await validateMethodChecksumNoChunking(request, body, log); + assert.strictEqual(err, null); + }, + ); + + it( + 'validateMethodChecksumNoChunking still rejects body mismatch for methods ' + + 'that remain in checksumedMethods (sanity check)', + async () => { + const body = Buffer.from('{"Objects":[]}'); + const finalObjectChecksum = '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; + const request = { + apiMethod: 'multiObjectDelete', + headers: { 'x-amz-checksum-sha256': finalObjectChecksum }, + }; + const err = await validateMethodChecksumNoChunking(request, body, log); + assert(err, 'expected an error for body checksum mismatch'); + assert.strictEqual(err.is.BadDigest, true); + }, + ); +}); From 202081814b716b01e7a614fc2bf66702ac2420b3 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 7 May 2026 17:17:14 +0200 Subject: [PATCH 2/7] CLDSRV-898: CompleteMPU calculate and validate final checksum with checksum header --- .../apiUtils/integrity/validateChecksums.js | 160 +++++++++-- lib/api/completeMultipartUpload.js | 271 ++++++++++++++---- .../functional/raw-node/test/xAmzChecksum.js | 10 +- .../apiUtils/integrity/computeMpuChecksums.js | 16 +- .../apiUtils/integrity/validateChecksums.js | 258 +++++++++++++++++ tests/unit/api/completeMultipartUpload.js | 217 +++++++++++++- 6 files changed, 829 insertions(+), 103 deletions(-) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 7fb987ab18..d45965f1b6 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -39,10 +39,9 @@ const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription 'The x-amz-checksum-type header can only be used ' + 'with the x-amz-checksum-algorithm header.', ); +// Methods that validate BOTH Content-MD5 and x-amz-checksum-* against the +// buffered request body. For these, x-amz-checksum-* is the body digest. const checksumedMethods = Object.freeze({ - // CompleteMPU's x-amz-checksum- is the final-object checksum, - // not a body digest. Validated in completeMultipartUpload.js instead. - // 'completeMultipartUpload': true, multiObjectDelete: true, bucketPutACL: true, bucketPutCors: true, @@ -63,6 +62,15 @@ const checksumedMethods = Object.freeze({ objectRestore: true, }); +// Methods that validate Content-MD5 against the buffered request body but +// do NOT treat x-amz-checksum-* as a body digest. CompleteMPU's +// x-amz-checksum- is the asserted final-object checksum, not a +// body digest, and is validated separately by +// validateCompleteMultipartUploadChecksum. +const md5OnlyMethods = Object.freeze({ + completeMultipartUpload: true, +}); + const ChecksumError = Object.freeze({ MD5Mismatch: 'MD5Mismatch', MD5Invalid: 'MD5Invalid', @@ -338,6 +346,28 @@ function getChecksumDataFromHeaders(headers) { * @param {Buffer} body - http request body * @return {object} - error */ +// Validate a Content-MD5 header against the buffered body. Returns null on +// success, an error object otherwise. +function validateContentMd5(headers, body) { + if (!('content-md5' in headers)) { + return { error: ChecksumError.MissingChecksum, details: null }; + } + if (typeof headers['content-md5'] !== 'string') { + return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } }; + } + if (headers['content-md5'].length !== 24) { + return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } }; + } + if (!base64Regex.test(headers['content-md5'])) { + return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } }; + } + const md5 = crypto.createHash('md5').update(body).digest('base64'); + if (md5 !== headers['content-md5']) { + return { error: ChecksumError.MD5Mismatch, details: { calculated: md5, expected: headers['content-md5'] } }; + } + return null; +} + async function validateChecksumsNoChunking(headers, body) { if (!headers) { return { error: ChecksumError.MissingChecksum, details: null }; @@ -345,23 +375,10 @@ async function validateChecksumsNoChunking(headers, body) { let md5Present = false; if ('content-md5' in headers) { - if (typeof headers['content-md5'] !== 'string') { - return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } }; + const md5Err = validateContentMd5(headers, body); + if (md5Err) { + return md5Err; } - - if (headers['content-md5'].length !== 24) { - return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } }; - } - - if (!base64Regex.test(headers['content-md5'])) { - return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } }; - } - - const md5 = crypto.createHash('md5').update(body).digest('base64'); - if (md5 !== headers['content-md5']) { - return { error: ChecksumError.MD5Mismatch, details: { calculated: md5, expected: headers['content-md5'] } }; - } - md5Present = true; } @@ -428,6 +445,86 @@ function arsenalErrorFromChecksumError(err) { } } +/** + * Validate the optional x-amz-checksum- header on a CompleteMPU + * request against the computed final-object checksum. + * + * AWS contract: when present, x-amz-checksum- on CompleteMPU is the + * client's assertion of what the final-object checksum should be (not a + * body digest). Rules: + * - Multiple value headers → MultipleChecksumTypes. + * - Unknown algorithm (e.g. x-amz-checksum-bad) → AlgoNotSupported. + * - Value not a valid digest for the named algorithm → MalformedChecksum + * (`-N` suffix permitted for COMPOSITE final-object values). + * - Single known, well-formed header that doesn't match the computed value + * (including when the named algorithm differs from the MPU's, or when we + * couldn't compute one at all) → XAmzMismatch. + * - No header → null (no error). + * + * Scans ALL `x-amz-checksum-*` headers on the request (excluding the + * configuration headers `x-amz-checksum-type` / `x-amz-checksum-algorithm`) + * so an unsupported algorithm is rejected rather than silently ignored. + * + * Returns the same `{ error, details }` discriminated shape as + * validateXAmzChecksums; the caller must run it through + * arsenalErrorFromChecksumError to get an ArsenalError. + * + * @param {object} headers - request.headers (lowercased keys) + * @param {object|null} finalChecksum - { algorithm, type, value } or null + * @returns {{error: string, details: object}|null} + */ +function validateCompleteMultipartUploadChecksum(headers, finalChecksum) { + // `x-amz-checksum-type` and `x-amz-checksum-algorithm` are configuration + // headers (MPU completeness mode / SDK algorithm hint), not value + // headers. They must not count toward the "value header" tally. + const valueHeaders = Object.keys(headers).filter( + h => h.startsWith('x-amz-checksum-') && h !== 'x-amz-checksum-type' && h !== 'x-amz-checksum-algorithm', + ); + + if (valueHeaders.length === 0) { + return null; + } + + if (valueHeaders.length > 1) { + return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: valueHeaders } }; + } + + const headerName = valueHeaders[0]; + const foundAlgo = headerName.slice('x-amz-checksum-'.length); + const foundValue = headers[headerName]; + + if (!(foundAlgo in algorithms)) { + return { error: ChecksumError.AlgoNotSupported, details: { algorithm: foundAlgo } }; + } + + const dashIdx = foundValue.lastIndexOf('-'); + const hasCompositeSuffix = dashIdx > 0 && /^\d+$/.test(foundValue.slice(dashIdx + 1)); + if (hasCompositeSuffix && !compositeAlgorithms.has(foundAlgo)) { + return { error: ChecksumError.MalformedChecksum, details: { algorithm: foundAlgo, expected: foundValue } }; + } + if (!hasCompositeSuffix && !fullObjectAlgorithms.has(foundAlgo)) { + return { error: ChecksumError.MalformedChecksum, details: { algorithm: foundAlgo, expected: foundValue } }; + } + const digestPart = hasCompositeSuffix ? foundValue.slice(0, dashIdx) : foundValue; + if (!algorithms[foundAlgo].isValidDigest(digestPart)) { + return { error: ChecksumError.MalformedChecksum, details: { algorithm: foundAlgo, expected: foundValue } }; + } + + if (!finalChecksum || finalChecksum.algorithm !== foundAlgo || finalChecksum.value !== foundValue) { + return { + error: ChecksumError.XAmzMismatch, + details: { + algorithm: foundAlgo, + expected: foundValue, + computedAlgorithm: finalChecksum && finalChecksum.algorithm, + computed: finalChecksum && finalChecksum.value, + }, + }; + } + + return null; +} + async function defaultValidationFunc(request, body, log) { const err = await validateChecksumsNoChunking(request.headers, body); if (!err) { @@ -441,6 +538,22 @@ async function defaultValidationFunc(request, body, log) { return arsenalErrorFromChecksumError(err); } +// Validate ONLY Content-MD5 against the buffered body. Used for methods +// where x-amz-checksum-* has a non-body meaning (e.g. CompleteMPU's +// final-object checksum assertion) and must not be re-validated here. +function md5OnlyValidationFunc(request, body, log) { + // Content-MD5 is optional. If absent, nothing to validate. + if (!('content-md5' in request.headers)) { + return null; + } + const err = validateContentMd5(request.headers, body); + if (!err) { + return null; + } + log.debug('failed Content-MD5 validation', { method: request.apiMethod }, err); + return arsenalErrorFromChecksumError(err); +} + /** * validateMethodChecksumsNoChunking - Validate the checksums of a request. * @param {object} request - http request @@ -457,6 +570,10 @@ async function validateMethodChecksumNoChunking(request, body, log) { return await defaultValidationFunc(request, body, log); } + if (request.apiMethod in md5OnlyMethods) { + return md5OnlyValidationFunc(request, body, log); + } + return null; } @@ -571,13 +688,13 @@ const COMPOSITE_ALGOS = new Set(['crc32', 'crc32c', 'sha1', 'sha256']); * @returns {{ checksum: string, error: null } * | { checksum: null, error: { code: string, details: object } }} */ -function computeCompositeMPUChecksum(algorithm, partChecksumsBase64) { +async function computeCompositeMPUChecksum(algorithm, partChecksumsBase64) { if (!COMPOSITE_ALGOS.has(algorithm)) { return { checksum: null, error: { code: ChecksumError.MPUAlgoNotSupported, details: { algorithm } } }; } const concat = Buffer.concat(partChecksumsBase64.map(c => Buffer.from(c, 'base64'))); - const digest = algorithms[algorithm].digest(concat); + const digest = await algorithms[algorithm].digest(concat); return { checksum: `${digest}-${partChecksumsBase64.length}`, error: null, @@ -619,4 +736,5 @@ module.exports = { getChecksumDataFromMPUHeaders, computeCompositeMPUChecksum, computeFullObjectMPUChecksum, + validateCompleteMultipartUploadChecksum, }; diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 5c82a2132a..5ee2fea851 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -26,7 +26,13 @@ const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); -const { algorithms: checksumAlgorithms } = require('./apiUtils/integrity/validateChecksums'); +const { + algorithms: checksumAlgorithms, + arsenalErrorFromChecksumError, + computeCompositeMPUChecksum, + computeFullObjectMPUChecksum, + validateCompleteMultipartUploadChecksum, +} = require('./apiUtils/integrity/validateChecksums'); const versionIdUtils = versioning.VersionID; @@ -108,6 +114,83 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu return null; } +/** + * Compute the final-object checksum for a CompleteMultipartUpload from the + * stored MPU configuration and per-part checksums. Returns null when the MPU + * has no checksum configured, when any part is missing its stored + * ChecksumValue (defensive — should not happen in steady state), or when the + * compute primitive itself reports an error. Errors are logged; the caller + * proceeds without a final-object checksum so the response simply omits the + * checksum fields rather than failing the whole CompleteMPU. + * + * @param {array} storedParts - parts from services.getMPUparts (carry ChecksumValue) + * @param {array} filteredPartList - validated, ordered subset matching jsonList + * @param {object} storedMetadata - MPU overview metadata + * @param {string} mpuSplitter - splitter used in part keys + * @param {string} uploadId - for log context + * @param {object} log - werelogs logger + * @returns {object|null} { algorithm, type, value } or null + */ +async function computeFinalChecksum(storedParts, filteredPartList, storedMetadata, mpuSplitter, uploadId, log) { + const algorithm = storedMetadata.checksumAlgorithm; + const type = storedMetadata.checksumType; + if (!algorithm || !type) { + return null; + } + + const storedByKey = new Map(); + storedParts.forEach(p => storedByKey.set(p.key, p)); + + const partInputs = []; + const missingPartNumbers = []; + for (const fp of filteredPartList) { + const stored = storedByKey.get(fp.key); + const value = stored && stored.value && stored.value.ChecksumValue; + if (!value) { + missingPartNumbers.push(fp.key.split(mpuSplitter)[1]); + continue; + } + partInputs.push({ value, length: Number.parseInt(fp.size, 10) }); + } + + if (missingPartNumbers.length > 0) { + log.error('one or more MPU parts missing checksum value; ' + 'skipping final-object checksum computation', { + uploadId, + algorithm, + type, + missingPartNumbers, + }); + return null; + } + + let result; + if (type === 'COMPOSITE') { + result = await computeCompositeMPUChecksum( + algorithm, + partInputs.map(p => p.value), + ); + } else if (type === 'FULL_OBJECT') { + result = computeFullObjectMPUChecksum(algorithm, partInputs); + } else { + log.error('unknown MPU checksumType; skipping final-object checksum computation', { + uploadId, + checksumType: type, + }); + return null; + } + + if (result.error) { + log.error('final-object checksum computation failed', { + uploadId, + checksumErrorCode: result.error.code, + checksumErrorDetails: result.error.details, + }); + return null; + } + + return { algorithm, type, value: result.checksum }; +} + /* Format of xml request: @@ -169,6 +252,7 @@ function completeMultipartUpload(authInfo, request, log, callback) { hostname, }; let oldByteLength = null; + let finalChecksum = null; const responseHeaders = {}; let versionId; @@ -490,56 +574,130 @@ function completeMultipartUpload(authInfo, request, log, callback) { totalMPUSize, next, ) { - // if mpu was completed on backend that stored mpu MD externally, - // skip MD processing steps - if (completeObjData && skipMpuPartProcessing(completeObjData)) { - const dataLocations = [ - { - key: completeObjData.key, - size: completeObjData.contentLength, - start: 0, - dataStoreVersionId: completeObjData.dataStoreVersionId, - dataStoreName: storedMetadata.dataStoreName, - dataStoreETag: completeObjData.eTag, - dataStoreType: completeObjData.dataStoreType, - }, - ]; - const calculatedSize = completeObjData.contentLength; - return next( - null, - destBucket, - objMD, - mpuBucket, - storedMetadata, - completeObjData.eTag, - calculatedSize, - dataLocations, - [mpuOverviewKey], - null, - completeObjData, - totalMPUSize, - ); + let nextCalled = false; + const callNext = (...args) => { + if (nextCalled) { + log.error('processParts: swallowed late callNext after next already invoked', { + uploadId, + error: args[0], + }); + return undefined; + } + nextCalled = true; + return next(...args); + }; + // External-handled MPUs (ingestion / external backends) come in + // with completeObjData set and no filteredPartsObj — the data + // store already aggregated the parts, and we have no per-part + // info to feed the compute step. Skip in that case. + if (!filteredPartsObj) { + return continueProcessParts(null); } + computeFinalChecksum( + storedParts, + filteredPartsObj.partList, + storedMetadata, + splitter, + uploadId, + log, + ).then( + fc => { + try { + finalChecksum = fc; + const checksumErr = validateCompleteMultipartUploadChecksum(request.headers, finalChecksum); + if (checksumErr) { + log.debug('x-amz-checksum- header validation failed on CompleteMPU', { + uploadId, + error: checksumErr.error, + details: checksumErr.details, + }); + return callNext(arsenalErrorFromChecksumError(checksumErr), destBucket); + } + return continueProcessParts(null); + } catch (resolveErr) { + log.error('unexpected throw after final-checksum compute', { + uploadId, + error: resolveErr, + }); + return callNext(resolveErr, destBucket); + } + }, + computeErr => { + log.error('final-object checksum compute threw', { + uploadId, + error: computeErr, + }); + return callNext(computeErr, destBucket); + }, + ); + return undefined; - const partsInfo = generateMpuPartStorageInfo(filteredPartsObj.partList); - if (partsInfo.error) { - return next(partsInfo.error, destBucket); - } - const { keysToDelete, extraPartLocations } = filteredPartsObj; - const { aggregateETag, dataLocations, calculatedSize } = partsInfo; + function continueProcessParts() { + // if mpu was completed on backend that stored mpu MD externally, + // skip MD processing steps + if (completeObjData && skipMpuPartProcessing(completeObjData)) { + const dataLocations = [ + { + key: completeObjData.key, + size: completeObjData.contentLength, + start: 0, + dataStoreVersionId: completeObjData.dataStoreVersionId, + dataStoreName: storedMetadata.dataStoreName, + dataStoreETag: completeObjData.eTag, + dataStoreType: completeObjData.dataStoreType, + }, + ]; + const calculatedSize = completeObjData.contentLength; + return callNext( + null, + destBucket, + objMD, + mpuBucket, + storedMetadata, + completeObjData.eTag, + calculatedSize, + dataLocations, + [mpuOverviewKey], + null, + completeObjData, + totalMPUSize, + ); + } - if (completeObjData) { - const dataLocations = [ - { - key: completeObjData.key, - size: calculatedSize, - start: 0, - dataStoreName: storedMetadata.dataStoreName, - dataStoreETag: aggregateETag, - dataStoreType: completeObjData.dataStoreType, - }, - ]; - return next( + const partsInfo = generateMpuPartStorageInfo(filteredPartsObj.partList); + if (partsInfo.error) { + return callNext(partsInfo.error, destBucket); + } + const { keysToDelete, extraPartLocations } = filteredPartsObj; + const { aggregateETag, dataLocations, calculatedSize } = partsInfo; + + if (completeObjData) { + const dataLocations = [ + { + key: completeObjData.key, + size: calculatedSize, + start: 0, + dataStoreName: storedMetadata.dataStoreName, + dataStoreETag: aggregateETag, + dataStoreType: completeObjData.dataStoreType, + }, + ]; + return callNext( + null, + destBucket, + objMD, + mpuBucket, + storedMetadata, + aggregateETag, + calculatedSize, + dataLocations, + keysToDelete, + extraPartLocations, + completeObjData, + totalMPUSize, + ); + } + return callNext( null, destBucket, objMD, @@ -550,24 +708,10 @@ function completeMultipartUpload(authInfo, request, log, callback) { dataLocations, keysToDelete, extraPartLocations, - completeObjData, + null, totalMPUSize, ); } - return next( - null, - destBucket, - objMD, - mpuBucket, - storedMetadata, - aggregateETag, - calculatedSize, - dataLocations, - keysToDelete, - extraPartLocations, - null, - totalMPUSize, - ); }, function prepForStoring( destBucket, @@ -1009,3 +1153,4 @@ function completeMultipartUpload(authInfo, request, log, callback) { module.exports = completeMultipartUpload; module.exports.validatePerPartChecksums = validatePerPartChecksums; +module.exports.computeFinalChecksum = computeFinalChecksum; diff --git a/tests/functional/raw-node/test/xAmzChecksum.js b/tests/functional/raw-node/test/xAmzChecksum.js index 691aedcd98..956a529663 100644 --- a/tests/functional/raw-node/test/xAmzChecksum.js +++ b/tests/functional/raw-node/test/xAmzChecksum.js @@ -24,13 +24,11 @@ describe('Test x-amz-checksums', () => { validWrong: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', }, ]; + // CompleteMultipartUpload intentionally not listed here: its + // x-amz-checksum- header is the expected final-object checksum, + // not a body digest, so it's not part of the buffered-body validator + // path tested below. const methods = [ - { - Name: 'CompleteMultipartupload', - Query: 'uploadId=77a4ce46b9bf4ea69d9e0cc3f0bb1aae', - Key: objectKey, - HTTPMethod: 'POST', - }, { Name: 'DeleteObjects', Query: 'delete', diff --git a/tests/unit/api/apiUtils/integrity/computeMpuChecksums.js b/tests/unit/api/apiUtils/integrity/computeMpuChecksums.js index 3640238dc5..67ff80a9b5 100644 --- a/tests/unit/api/apiUtils/integrity/computeMpuChecksums.js +++ b/tests/unit/api/apiUtils/integrity/computeMpuChecksums.js @@ -27,34 +27,34 @@ describe('computeCompositeMPUChecksum', () => { COMPOSITE_ALGOS.forEach(algo => { const label = algo.toUpperCase(); - it(`should match ${label}(decode(c1) || ... || decode(cN)) + "-N"`, () => { + it(`should match ${label}(decode(c1) || ... || decode(cN)) + "-N"`, async () => { const partChecksums = parts.map(p => algorithms[algo].digest(p)); const expectedConcat = Buffer.concat(partChecksums.map(c => Buffer.from(c, 'base64'))); const expected = `${algorithms[algo].digest(expectedConcat)}-3`; - const got = computeCompositeMPUChecksum(algo, partChecksums); + const got = await computeCompositeMPUChecksum(algo, partChecksums); assert.strictEqual(got.error, null); assert.strictEqual(got.checksum, expected); }); }); - it('should return N=1 for a single part', () => { + it('should return N=1 for a single part', async () => { const partChecksums = [algorithms.sha256.digest(parts[0])]; - const got = computeCompositeMPUChecksum('sha256', partChecksums); + const got = await computeCompositeMPUChecksum('sha256', partChecksums); assert.strictEqual(got.error, null); assert(got.checksum.endsWith('-1')); }); - it('should return an error object on unsupported algorithm', () => { - const got = computeCompositeMPUChecksum('md5', ['AAAA']); + it('should return an error object on unsupported algorithm', async () => { + const got = await computeCompositeMPUChecksum('md5', ['AAAA']); assert.strictEqual(got.checksum, null); assert(got.error); assert.strictEqual(got.error.code, 'MPUAlgoNotSupported'); assert.deepStrictEqual(got.error.details, { algorithm: 'md5' }); }); - it('should return an error object for crc64nvme (not allowed for COMPOSITE)', () => { - const got = computeCompositeMPUChecksum('crc64nvme', ['AQIDBAUGBwg=']); + it('should return an error object for crc64nvme (not allowed for COMPOSITE)', async () => { + const got = await computeCompositeMPUChecksum('crc64nvme', ['AQIDBAUGBwg=']); assert.strictEqual(got.checksum, null); assert.strictEqual(got.error.code, 'MPUAlgoNotSupported'); }); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index 01bebac4a5..3d95036f66 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -10,6 +10,7 @@ const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, getChecksumDataFromMPUHeaders, + validateCompleteMultipartUploadChecksum, } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); const { config } = require('../../../../../lib/Config'); @@ -467,6 +468,74 @@ describe('validateMethodChecksumNoChunking', () => { assert.strictEqual(result, null); }); }); + + // CompleteMPU is not in `checksumedMethods` (x-amz-checksum-* is the + // final-object checksum, not a body digest) but it still must validate + // Content-MD5 against the XML body when present. + describe('completeMultipartUpload (md5-only path)', () => { + const body = 'Hello, World!'; + const correctMd5 = crypto.createHash('md5').update(body, 'utf8').digest('base64'); + + it('should return null when Content-MD5 matches the body', async () => { + config.integrityChecks.completeMultipartUpload = true; + const request = { + apiMethod: 'completeMultipartUpload', + headers: { 'content-md5': correctMd5 }, + }; + const log = { debug: sandbox.stub() }; + const result = await validateMethodChecksumNoChunking(request, body, log); + assert.strictEqual(result, null); + }); + + it('should return BadDigest when Content-MD5 does not match the body', async () => { + config.integrityChecks.completeMultipartUpload = true; + const request = { + apiMethod: 'completeMultipartUpload', + headers: { 'content-md5': '1B2M2Y8AsgTpgAmY7PhCfg==' }, + }; + const log = { debug: sandbox.stub() }; + const result = await validateMethodChecksumNoChunking(request, body, log); + assert.deepStrictEqual(result, ArsenalErrors.BadDigest); + assert(log.debug.calledOnce); + }); + + it('should return InvalidDigest when Content-MD5 is malformed', async () => { + config.integrityChecks.completeMultipartUpload = true; + const request = { + apiMethod: 'completeMultipartUpload', + headers: { 'content-md5': 'wrongchecksum123=' }, + }; + const log = { debug: sandbox.stub() }; + const result = await validateMethodChecksumNoChunking(request, body, log); + assert.deepStrictEqual(result, ArsenalErrors.InvalidDigest); + }); + + it('should return null when no Content-MD5 header is present', async () => { + config.integrityChecks.completeMultipartUpload = true; + const request = { + apiMethod: 'completeMultipartUpload', + headers: {}, + }; + const log = { debug: sandbox.stub() }; + const result = await validateMethodChecksumNoChunking(request, body, log); + assert.strictEqual(result, null); + }); + + it('should NOT validate x-amz-checksum-* as a body digest (final-object semantics)', async () => { + // If we routed CompleteMPU through `defaultValidationFunc` like + // the other methods, this wrong x-amz-checksum-sha256 (treated + // as a body digest) would return BadDigest. The md5-only path + // must ignore it — the final-object validator handles it later. + config.integrityChecks.completeMultipartUpload = true; + const request = { + apiMethod: 'completeMultipartUpload', + headers: { 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, + }; + const log = { debug: sandbox.stub() }; + const result = await validateMethodChecksumNoChunking(request, body, log); + assert.strictEqual(result, null); + }); + }); }); describe('getChecksumDataFromHeaders', () => { @@ -930,3 +999,192 @@ describe('getChecksumDataFromMPUHeaders', () => { } }); }); + +describe('validateCompleteMultipartUploadChecksum', () => { + // Real-length placeholder digests that pass `isValidDigest`. + const SHA256_A = 'ypeBEsobvcr6wjGzmiPcTaeG7/gUfE5yuYB3ha/uSLs='; + const SHA256_B = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + const CRC32_A = 'AAAAAA=='; + + it('should return null when no x-amz-checksum- header is present', () => { + const err = validateCompleteMultipartUploadChecksum( + { host: 'example.com' }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert.strictEqual(err, null); + }); + + it('should ignore x-amz-checksum-type and x-amz-checksum-algorithm headers', () => { + const err = validateCompleteMultipartUploadChecksum( + { + 'x-amz-checksum-type': 'COMPOSITE', + 'x-amz-checksum-algorithm': 'SHA256', + }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert.strictEqual(err, null); + }); + + it('should return null when COMPOSITE header value matches (digest-N form)', () => { + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-sha256': `${SHA256_A}-3` }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert.strictEqual(err, null); + }); + + it('should return null when FULL_OBJECT header value matches (no suffix)', () => { + // Use crc32 (a dual-form algorithm) for the FULL_OBJECT case — sha256 + // is COMPOSITE-only, so a no-suffix sha256 value is shape-malformed + // regardless of the MPU's `type`. + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-crc32': CRC32_A }, + { algorithm: 'crc32', type: 'FULL_OBJECT', value: CRC32_A }, + ); + assert.strictEqual(err, null); + }); + + it('should return XAmzMismatch when header value differs', () => { + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-sha256': `${SHA256_B}-3` }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.XAmzMismatch); + assert.strictEqual(err.details.algorithm, 'sha256'); + }); + + it('should return XAmzMismatch when header algorithm differs from MPU', () => { + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-crc32': CRC32_A }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.XAmzMismatch); + assert.strictEqual(err.details.algorithm, 'crc32'); + }); + + it('should return XAmzMismatch when header is present but finalChecksum is null', () => { + // Use a shape-valid value (sha256 is COMPOSITE-only, so it requires + // the `-N` suffix) so we exercise the finalChecksum=null path + // rather than the shape-mismatch path. + const err = validateCompleteMultipartUploadChecksum({ 'x-amz-checksum-sha256': `${SHA256_A}-3` }, null); + assert(err); + assert.strictEqual(err.error, ChecksumError.XAmzMismatch); + }); + + it('should return null when finalChecksum is null and no header present', () => { + const err = validateCompleteMultipartUploadChecksum({ host: 'example.com' }, null); + assert.strictEqual(err, null); + }); + + it('should return MultipleChecksumTypes when multiple x-amz-checksum-* headers are sent', () => { + const err = validateCompleteMultipartUploadChecksum( + { + 'x-amz-checksum-sha256': SHA256_A, + 'x-amz-checksum-crc32': CRC32_A, + }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.MultipleChecksumTypes); + assert.deepStrictEqual(err.details.algorithms.sort(), ['x-amz-checksum-crc32', 'x-amz-checksum-sha256']); + }); + + it('should return MalformedChecksum when the header value is not a valid digest', () => { + // AWS S3 returns InvalidRequest "Value for x-amz-checksum-sha256 + // header is invalid." for a malformed value (verified us-east-1, + // 2026-05-13). Falling through to a misleading "did not match" + // BadDigest would be wrong. + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-sha256': '!!!not-base64!!!' }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.MalformedChecksum); + assert.strictEqual(err.details.algorithm, 'sha256'); + assert.strictEqual(err.details.expected, '!!!not-base64!!!'); + }); + + it('should return MalformedChecksum when the digest-N prefix is the wrong length', () => { + // 'abc' is valid base64 chars but not a 44-char SHA256 digest. + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-sha256': 'abc-3' }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.MalformedChecksum); + }); + + it('should return MalformedChecksum when a FULL_OBJECT-only algorithm carries a `-N` suffix', () => { + // crc64nvme only exists as FULL_OBJECT — a `-N` value is + // shape-malformed, not a mismatch. Without the shape check the + // suffix would be silently stripped and the digest would validate, + // causing this to fall through as XAmzMismatch. + const CRC64NVME_VALID = 'AAAAAAAAAAAA'; // 12 chars + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-crc64nvme': `${CRC64NVME_VALID}-5` }, + { algorithm: 'crc64nvme', type: 'FULL_OBJECT', value: CRC64NVME_VALID }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.MalformedChecksum); + assert.strictEqual(err.details.algorithm, 'crc64nvme'); + assert.strictEqual(err.details.expected, `${CRC64NVME_VALID}-5`); + }); + + it('should return MalformedChecksum when a COMPOSITE-only algorithm lacks the `-N` suffix', () => { + // sha1 only exists as COMPOSITE — a bare `` value is + // shape-malformed. + const SHA1_VALID = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAA'; // 28 chars + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-sha1': SHA1_VALID }, + { algorithm: 'sha1', type: 'COMPOSITE', value: `${SHA1_VALID}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.MalformedChecksum); + assert.strictEqual(err.details.algorithm, 'sha1'); + }); + + it('should return XAmzMismatch (not MalformedChecksum) when a dual-form algorithm sends the wrong shape', () => { + // crc32 supports both FULL_OBJECT and COMPOSITE, so `-N` + // against a FULL_OBJECT MPU is shape-valid; the type mismatch + // is a regular value mismatch, not malformed. + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-crc32': `${CRC32_A}-3` }, + { algorithm: 'crc32', type: 'FULL_OBJECT', value: CRC32_A }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.XAmzMismatch); + }); + + it('should return XAmzMismatch (not MalformedChecksum) when a dual-form algorithm omits a required suffix', () => { + // Symmetric: bare `` against a COMPOSITE crc32 MPU is + // shape-valid (crc32 also supports FULL_OBJECT) but value-mismatched. + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-crc32': CRC32_A }, + { algorithm: 'crc32', type: 'COMPOSITE', value: `${CRC32_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.XAmzMismatch); + }); + + it('should return AlgoNotSupported when x-amz-checksum- uses an unsupported algorithm', () => { + // Must scan ALL x-amz-checksum-* headers, not just the supported + // algorithms — otherwise a bogus header is silently ignored and the + // request proceeds. + const err = validateCompleteMultipartUploadChecksum( + { 'x-amz-checksum-bad': 'anything' }, + { algorithm: 'sha256', type: 'COMPOSITE', value: `${SHA256_A}-3` }, + ); + assert(err); + assert.strictEqual(err.error, ChecksumError.AlgoNotSupported); + assert.strictEqual(err.details.algorithm, 'bad'); + }); + + it('should reject an unsupported x-amz-checksum- header even when finalChecksum is null', () => { + const err = validateCompleteMultipartUploadChecksum({ 'x-amz-checksum-md5': 'anything' }, null); + assert(err); + assert.strictEqual(err.error, ChecksumError.AlgoNotSupported); + assert.strictEqual(err.details.algorithm, 'md5'); + }); +}); diff --git a/tests/unit/api/completeMultipartUpload.js b/tests/unit/api/completeMultipartUpload.js index ae4f9ad05d..a03a07fa60 100644 --- a/tests/unit/api/completeMultipartUpload.js +++ b/tests/unit/api/completeMultipartUpload.js @@ -7,8 +7,11 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const objectPutPart = require('../../../lib/api/objectPutPart'); const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); -const { validatePerPartChecksums } = completeMultipartUpload; -const { validateMethodChecksumNoChunking } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const { validatePerPartChecksums, computeFinalChecksum } = completeMultipartUpload; +const { + validateMethodChecksumNoChunking, + algorithms, +} = require('../../../lib/api/apiUtils/integrity/validateChecksums'); const DummyRequest = require('../DummyRequest'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); @@ -463,7 +466,7 @@ describe('CompleteMultipartUpload body-checksum bypass', () => { const log = new DummyRequestLogger(); it( - 'validateMethodChecksumNoChunking returns null for completeMultipartUpload ' + + 'should skip body-checksum validation for completeMultipartUpload ' + 'even when x-amz-checksum-sha256 does not match the body digest', async () => { const body = Buffer.from( @@ -485,8 +488,7 @@ describe('CompleteMultipartUpload body-checksum bypass', () => { ); it( - 'validateMethodChecksumNoChunking still rejects body mismatch for methods ' + - 'that remain in checksumedMethods (sanity check)', + 'should still reject body mismatch for methods that remain in checksumedMethods ' + '(sanity check)', async () => { const body = Buffer.from('{"Objects":[]}'); const finalObjectChecksum = '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU='; @@ -500,3 +502,208 @@ describe('CompleteMultipartUpload body-checksum bypass', () => { }, ); }); + +describe('computeFinalChecksum', () => { + const log = new DummyRequestLogger(); + const uploadId = UPLOAD_ID; + + function partListFromStored(stored) { + return stored.map(s => ({ + key: s.key, + ETag: `"${s.value.ETag}"`, + size: s.value.Size, + locations: s.value.partLocations, + })); + } + + it('should return null when MPU has no checksumAlgorithm', async () => { + const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; + const got = await computeFinalChecksum(stored, partListFromStored(stored), {}, SPLITTER, uploadId, log); + assert.strictEqual(got, null); + }); + + it('should return null when MPU has no checksumType', async () => { + const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'sha256' }, + SPLITTER, + uploadId, + log, + ); + assert.strictEqual(got, null); + }); + + it('should return COMPOSITE checksum with -N suffix for SHA256 MPU', async () => { + const [d1, d2, d3] = [SAMPLE_DIGESTS.sha256[0], SAMPLE_DIGESTS.sha256[1], SAMPLE_DIGESTS.sha256[0]]; + const stored = [ + makeStoredPart(1, { algorithm: 'sha256', value: d1 }), + makeStoredPart(2, { algorithm: 'sha256', value: d2 }), + makeStoredPart(3, { algorithm: 'sha256', value: d3 }), + ]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE' }, + SPLITTER, + uploadId, + log, + ); + assert(got); + assert.strictEqual(got.algorithm, 'sha256'); + assert.strictEqual(got.type, 'COMPOSITE'); + assert(got.value.endsWith('-3'), `expected -N suffix, got ${got.value}`); + // computeCompositeMPUChecksum's deterministic output for these + // exact placeholder digests: + const expected = crypto + .createHash('sha256') + .update(Buffer.concat([d1, d2, d3].map(x => Buffer.from(x, 'base64')))) + .digest('base64'); + assert.strictEqual(got.value, `${expected}-3`); + }); + + ['sha1', 'crc32', 'crc32c'].forEach(algo => { + it(`should compute COMPOSITE checksum for ${algo.toUpperCase()}`, async () => { + const [d1, d2] = SAMPLE_DIGESTS[algo]; + const stored = [ + makeStoredPart(1, { algorithm: algo, value: d1 }), + makeStoredPart(2, { algorithm: algo, value: d2 }), + ]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: algo, checksumType: 'COMPOSITE' }, + SPLITTER, + uploadId, + log, + ); + assert(got); + assert.strictEqual(got.algorithm, algo); + assert.strictEqual(got.type, 'COMPOSITE'); + assert(got.value.endsWith('-2')); + }); + }); + + it('should return FULL_OBJECT checksum without -N suffix for CRC64NVME', async () => { + // Real CRCs over real bytes so we can verify against the equivalent + // direct CRC of the concatenation. + const a = crypto.randomBytes(1024); + const b = crypto.randomBytes(2048); + const dA = await algorithms.crc64nvme.digest(a); + const dB = await algorithms.crc64nvme.digest(b); + const stored = [ + { + key: `${UPLOAD_ID}${SPLITTER}1`, + value: { + ETag: 'e', + Size: a.length, + ChecksumAlgorithm: 'crc64nvme', + ChecksumValue: dA, + partLocations: [], + }, + }, + { + key: `${UPLOAD_ID}${SPLITTER}2`, + value: { + ETag: 'e', + Size: b.length, + ChecksumAlgorithm: 'crc64nvme', + ChecksumValue: dB, + partLocations: [], + }, + }, + ]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'crc64nvme', checksumType: 'FULL_OBJECT' }, + SPLITTER, + uploadId, + log, + ); + assert(got); + assert.strictEqual(got.algorithm, 'crc64nvme'); + assert.strictEqual(got.type, 'FULL_OBJECT'); + assert(!got.value.includes('-'), `FULL_OBJECT should have no -N suffix, got ${got.value}`); + const expected = await algorithms.crc64nvme.digest(Buffer.concat([a, b])); + assert.strictEqual(got.value, expected); + }); + + it('should return null and log when a part is missing ChecksumValue', async () => { + const stored = [ + makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] }), + makeStoredPart(2, null), + makeStoredPart(3, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[1] }), + ]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE' }, + SPLITTER, + uploadId, + log, + ); + assert.strictEqual(got, null); + }); + + it('should return null when checksumType is unknown', async () => { + const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'sha256', checksumType: 'WEIRD' }, + SPLITTER, + uploadId, + log, + ); + assert.strictEqual(got, null); + }); + + it( + 'should return null when underlying compute reports an error ' + '(crc64nvme COMPOSITE is not allowed)', + async () => { + const stored = [makeStoredPart(1, { algorithm: 'crc64nvme', value: SAMPLE_DIGESTS.crc64nvme[0] })]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'crc64nvme', checksumType: 'COMPOSITE' }, + SPLITTER, + uploadId, + log, + ); + assert.strictEqual(got, null); + }, + ); + + it('should compute over filteredPartList (subset), not all storedParts', async () => { + const [d1, d2, d3] = [SAMPLE_DIGESTS.sha256[0], SAMPLE_DIGESTS.sha256[1], SAMPLE_DIGESTS.sha256[0]]; + const stored = [ + makeStoredPart(1, { algorithm: 'sha256', value: d1 }), + makeStoredPart(2, { algorithm: 'sha256', value: d2 }), + makeStoredPart(3, { algorithm: 'sha256', value: d3 }), + ]; + // User completes only parts 1 and 3, dropping 2 (orphan). + const filtered = [stored[0], stored[2]].map(s => ({ + key: s.key, + ETag: `"${s.value.ETag}"`, + size: s.value.Size, + locations: s.value.partLocations, + })); + const got = await computeFinalChecksum( + stored, + filtered, + { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE' }, + SPLITTER, + uploadId, + log, + ); + assert(got); + assert(got.value.endsWith('-2'), `should reflect 2 completed parts, got ${got.value}`); + const expected = crypto + .createHash('sha256') + .update(Buffer.concat([d1, d3].map(x => Buffer.from(x, 'base64')))) + .digest('base64'); + assert.strictEqual(got.value, `${expected}-2`); + }); +}); From ea974510f68d51f990ad2b92c83211ff1fb5ec4b Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 7 May 2026 22:09:52 +0200 Subject: [PATCH 3/7] CLDSRV-898: CompleteMPU store FULL_OBJECT checksum in object metadata --- lib/api/completeMultipartUpload.js | 9 + .../aws-node-sdk/test/object/mpuVersion.js | 19 +- tests/unit/api/completeMultipartUpload.js | 182 ++++++++++++++++++ 3 files changed, 202 insertions(+), 8 deletions(-) diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 5ee2fea851..167d0d660e 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -743,6 +743,9 @@ function completeMultipartUpload(authInfo, request, log, callback) { 'expires', 'eventualStorageBucket', 'dataStoreName', + 'checksumAlgorithm', + 'checksumType', + 'checksumIsDefault', ]; const metadataKeysToPull = Object.keys(storedMetadata).filter( item => keysNotNeeded.indexOf(item) === -1, @@ -780,6 +783,12 @@ function completeMultipartUpload(authInfo, request, log, callback) { overheadField: constants.overheadField, log, }; + // Persist FULL_OBJECT final-object checksum on the new ObjectMD. + // COMPOSITE is intentionally skipped to prevent metadata bloat, + // to be done in S3C-10399. + if (finalChecksum && finalChecksum.type === 'FULL_OBJECT') { + metaStoreParams.checksum = finalChecksum; + } // If key already exists if (objMD) { // Re-use creation-time if we can diff --git a/tests/functional/aws-node-sdk/test/object/mpuVersion.js b/tests/functional/aws-node-sdk/test/object/mpuVersion.js index 3bb476ecb8..259a4a1537 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuVersion.js +++ b/tests/functional/aws-node-sdk/test/object/mpuVersion.js @@ -142,14 +142,6 @@ function checkObjMdAndUpdate(objMDBefore, objMDAfter, props) { // eslint-disable-next-line no-param-reassign delete objMDBefore['content-type']; } - if (objMDBefore.checksum && !objMDAfter.checksum) { - // The initial PutObject stores a checksum, but the MPU restore path does not - // (CompleteMultipartUpload checksum storage is not yet implemented). - // Once it is, the restored object should carry a checksum and this workaround - // should be removed. - // eslint-disable-next-line no-param-reassign - delete objMDBefore.checksum; - } } function clearUploadIdAndRestoreStatusFromVersions(versions) { @@ -353,6 +345,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'archive', 'dataStoreName', 'originOp', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); @@ -393,6 +386,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); @@ -438,6 +432,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -482,6 +477,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -524,6 +520,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -569,6 +566,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -619,6 +617,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -666,6 +665,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -711,6 +711,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -764,6 +765,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert.deepStrictEqual(objMDAfter, objMDBefore); }); @@ -807,6 +809,7 @@ describe('MPU with x-scal-s3-version-id header', () => { 'x-amz-restore', 'archive', 'dataStoreName', + 'checksum', ]); assert(isDeepStrictEqual(objMDAfter, objMDBefore), 'Objects should be deeply equal'); diff --git a/tests/unit/api/completeMultipartUpload.js b/tests/unit/api/completeMultipartUpload.js index a03a07fa60..e71f253c9c 100644 --- a/tests/unit/api/completeMultipartUpload.js +++ b/tests/unit/api/completeMultipartUpload.js @@ -7,6 +7,7 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const objectPutPart = require('../../../lib/api/objectPutPart'); const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); +const metadata = require('../../../lib/metadata/wrapper'); const { validatePerPartChecksums, computeFinalChecksum } = completeMultipartUpload; const { validateMethodChecksumNoChunking, @@ -707,3 +708,184 @@ describe('computeFinalChecksum', () => { assert.strictEqual(got.value, `${expected}-2`); }); }); + +describe('CompleteMultipartUpload final-object checksum storage', () => { + const log = new DummyRequestLogger(); + const authInfo = makeAuthInfo('accessKey1'); + const namespace = 'default'; + const bucketName = 'bucketname-final-checksum'; + const objectKey = 'testObject'; + const partBody = Buffer.from('I am a part\n', 'utf8'); + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + + const bucketPutRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + 'scality-internal-mem' + + '', + actionImplicitDenies: false, + }; + + // (algorithm, type) pairs valid for an MPU per AWS rules. + // shouldStore reflects Part 3's gating: only FULL_OBJECT is persisted. + const STORAGE_MATRIX = [ + { algorithm: 'crc32', type: 'FULL_OBJECT', shouldStore: true }, + { algorithm: 'crc32c', type: 'FULL_OBJECT', shouldStore: true }, + { algorithm: 'crc64nvme', type: 'FULL_OBJECT', shouldStore: true }, + { algorithm: 'crc32', type: 'COMPOSITE', shouldStore: false }, + { algorithm: 'crc32c', type: 'COMPOSITE', shouldStore: false }, + { algorithm: 'sha1', type: 'COMPOSITE', shouldStore: false }, + { algorithm: 'sha256', type: 'COMPOSITE', shouldStore: false }, + ]; + + function bucketPutP() { + return new Promise((resolve, reject) => + bucketPut(authInfo, bucketPutRequest, log, err => (err ? reject(err) : resolve())), + ); + } + + function initiateMpuP(headers) { + return new Promise((resolve, reject) => { + initiateMultipartUpload( + authInfo, + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: `/${objectKey}?uploads`, + actionImplicitDenies: false, + }, + log, + (err, xml) => { + if (err) { + return reject(err); + } + return parseString(xml, (parseErr, json) => + parseErr ? reject(parseErr) : resolve(json.InitiateMultipartUploadResult.UploadId[0]), + ); + }, + ); + }); + } + + function uploadPartP(uploadId, headers = {}) { + return new Promise((resolve, reject) => { + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { partNumber: '1', uploadId }, + partHash, + actionImplicitDenies: false, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => (err ? reject(err) : resolve())); + }); + } + + function completeMpuP(uploadId, partChecksumXml = '') { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"${partChecksumXml}` + + '' + + ''; + return new Promise((resolve, reject) => { + completeMultipartUpload( + authInfo, + { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId }, + post: completeBody, + actionImplicitDenies: false, + }, + log, + err => (err ? reject(err) : resolve()), + ); + }); + } + + function fetchObjectMDP() { + return new Promise((resolve, reject) => + metadata.getObjectMD(bucketName, objectKey, {}, log, (err, md) => (err ? reject(err) : resolve(md))), + ); + } + + beforeEach(() => cleanup()); + + STORAGE_MATRIX.forEach(({ algorithm, type, shouldStore }) => { + const upper = algorithm.toUpperCase(); + const verb = shouldStore ? 'should persist' : 'should not persist'; + const tag = TAG_BY_ALGO[algorithm]; + + it(`${verb} ${type} ${upper} checksum on the ObjectMD`, async () => { + await bucketPutP(); + const uploadId = await initiateMpuP({ + 'x-amz-checksum-algorithm': upper, + 'x-amz-checksum-type': type, + }); + // Pre-compute the part's checksum so we can supply it on + // UploadPart and (for COMPOSITE non-default) in the Complete body. + const partChecksum = await algorithms[algorithm].digest(partBody); + const uploadHeaders = type === 'COMPOSITE' ? { [`x-amz-checksum-${algorithm}`]: partChecksum } : {}; + await uploadPartP(uploadId, uploadHeaders); + const partChecksumXml = type === 'COMPOSITE' ? `<${tag}>${partChecksum}` : ''; + await completeMpuP(uploadId, partChecksumXml); + const md = await fetchObjectMDP(); + if (shouldStore) { + assert(md.checksum, `expected ${type} ${upper} checksum on ObjectMD`); + assert.strictEqual(md.checksum.checksumAlgorithm, algorithm); + assert.strictEqual(md.checksum.checksumType, type); + assert(typeof md.checksum.checksumValue === 'string'); + assert(md.checksum.checksumValue.length > 0); + } else { + assert.strictEqual(md.checksum, undefined, `${type} ${upper} should not persist on ObjectMD`); + } + }); + }); + + it('should persist FULL_OBJECT CRC64NVME checksum for default MPU (no checksum headers)', async () => { + // No x-amz-checksum-algorithm / x-amz-checksum-type headers — AWS + // defaults to crc64nvme/FULL_OBJECT and still persists the result. + await bucketPutP(); + const uploadId = await initiateMpuP({}); + await uploadPartP(uploadId); + await completeMpuP(uploadId); + const md = await fetchObjectMDP(); + assert(md.checksum, 'default MPU should still persist a checksum'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + }); + + it('should not leak checksumAlgorithm/Type/IsDefault into ObjectMD top-level fields', async () => { + // keysNotNeeded keeps these MPU-overview-only keys out of metaHeaders, + // which prevents them from sticking around on the final ObjectMD. + await bucketPutP(); + const uploadId = await initiateMpuP({ + 'x-amz-checksum-algorithm': 'CRC32', + 'x-amz-checksum-type': 'FULL_OBJECT', + }); + await uploadPartP(uploadId); + await completeMpuP(uploadId); + const md = await fetchObjectMDP(); + assert.strictEqual(md.checksumAlgorithm, undefined); + assert.strictEqual(md.checksumType, undefined); + assert.strictEqual(md.checksumIsDefault, undefined); + }); +}); From 4a95779035018d71dabef56e628ab868a12c234a Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 7 May 2026 23:08:02 +0200 Subject: [PATCH 4/7] CLDSRV-898: CompleteMPU set checksum value and type in response XML body --- lib/api/completeMultipartUpload.js | 5 + tests/unit/api/completeMultipartUpload.js | 171 ++++++++++++++++++++++ 2 files changed, 176 insertions(+) diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 167d0d660e..2ac33fa843 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -1144,6 +1144,11 @@ function completeMultipartUpload(authInfo, request, log, callback) { const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; xmlParams.eTag = `"${aggregateETag}"`; + if (finalChecksum) { + xmlParams.checksumAlgorithm = finalChecksum.algorithm; + xmlParams.checksumValue = finalChecksum.value; + xmlParams.checksumType = finalChecksum.type; + } const xml = convertToXml('completeMultipartUpload', xmlParams); pushMetric('completeMultipartUpload', log, { oldByteLength: isVersionedObj ? null : oldByteLength, diff --git a/tests/unit/api/completeMultipartUpload.js b/tests/unit/api/completeMultipartUpload.js index e71f253c9c..1e6044c82e 100644 --- a/tests/unit/api/completeMultipartUpload.js +++ b/tests/unit/api/completeMultipartUpload.js @@ -889,3 +889,174 @@ describe('CompleteMultipartUpload final-object checksum storage', () => { assert.strictEqual(md.checksumIsDefault, undefined); }); }); + +describe('CompleteMultipartUpload final-object checksum response', () => { + const log = new DummyRequestLogger(); + const authInfo = makeAuthInfo('accessKey1'); + const namespace = 'default'; + const bucketName = 'bucketname-final-checksum-resp'; + const objectKey = 'testObject'; + const partBody = Buffer.from('I am a part\n', 'utf8'); + const partHash = crypto.createHash('md5').update(partBody).digest('hex'); + + const bucketPutRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + 'scality-internal-mem' + + '', + actionImplicitDenies: false, + }; + + const RESPONSE_MATRIX = [ + { algorithm: 'crc32', type: 'FULL_OBJECT' }, + { algorithm: 'crc32c', type: 'FULL_OBJECT' }, + { algorithm: 'crc64nvme', type: 'FULL_OBJECT' }, + { algorithm: 'crc32', type: 'COMPOSITE' }, + { algorithm: 'crc32c', type: 'COMPOSITE' }, + { algorithm: 'sha1', type: 'COMPOSITE' }, + { algorithm: 'sha256', type: 'COMPOSITE' }, + ]; + + function bucketPutP() { + return new Promise((resolve, reject) => + bucketPut(authInfo, bucketPutRequest, log, err => (err ? reject(err) : resolve())), + ); + } + + function initiateMpuP(headers) { + return new Promise((resolve, reject) => { + initiateMultipartUpload( + authInfo, + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: `/${objectKey}?uploads`, + actionImplicitDenies: false, + }, + log, + (err, xml) => { + if (err) { + return reject(err); + } + return parseString(xml, (parseErr, json) => + parseErr ? reject(parseErr) : resolve(json.InitiateMultipartUploadResult.UploadId[0]), + ); + }, + ); + }); + } + + function uploadPartP(uploadId, headers = {}) { + return new Promise((resolve, reject) => { + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { partNumber: '1', uploadId }, + partHash, + actionImplicitDenies: false, + }, + partBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => (err ? reject(err) : resolve())); + }); + } + + // Resolves with { xml, headers } so callers can inspect both the + // response body and the response headers. + function completeMpuP(uploadId, partChecksumXml = '') { + const completeBody = + '' + + '' + + '1' + + `"${partHash}"${partChecksumXml}` + + '' + + ''; + return new Promise((resolve, reject) => { + completeMultipartUpload( + authInfo, + { + bucketName, + namespace, + objectKey, + parsedHost: 's3.amazonaws.com', + url: `/${objectKey}?uploadId=${uploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { uploadId }, + post: completeBody, + actionImplicitDenies: false, + }, + log, + (err, xml, headers) => (err ? reject(err) : resolve({ xml, headers })), + ); + }); + } + + function parseXmlP(xmlStr) { + return new Promise((resolve, reject) => + parseString(xmlStr, (err, json) => (err ? reject(err) : resolve(json))), + ); + } + + beforeEach(() => cleanup()); + + RESPONSE_MATRIX.forEach(({ algorithm, type }) => { + const upper = algorithm.toUpperCase(); + const tag = TAG_BY_ALGO[algorithm]; + + it(`should emit ${type} ${upper} in response XML`, async () => { + await bucketPutP(); + const uploadId = await initiateMpuP({ + 'x-amz-checksum-algorithm': upper, + 'x-amz-checksum-type': type, + }); + const partChecksum = await algorithms[algorithm].digest(partBody); + const uploadHeaders = type === 'COMPOSITE' ? { [`x-amz-checksum-${algorithm}`]: partChecksum } : {}; + await uploadPartP(uploadId, uploadHeaders); + const partChecksumXml = type === 'COMPOSITE' ? `<${tag}>${partChecksum}` : ''; + const { xml, headers } = await completeMpuP(uploadId, partChecksumXml); + const json = await parseXmlP(xml); + const result = json.CompleteMultipartUploadResult; + assert(result[tag], `expected ${tag} in response XML`); + const xmlValue = result[tag][0]; + assert(typeof xmlValue === 'string' && xmlValue.length > 0); + assert.strictEqual(result.ChecksumType[0], type); + // COMPOSITE values carry the "-N" suffix; FULL_OBJECT do not. + if (type === 'COMPOSITE') { + assert(xmlValue.endsWith('-1'), `expected -1 suffix for 1-part COMPOSITE, got ${xmlValue}`); + } else { + assert(!xmlValue.includes('-'), `FULL_OBJECT value should have no suffix, got ${xmlValue}`); + } + // AWS-verified: CompleteMPU does NOT emit + // x-amz-checksum-* / x-amz-checksum-type response headers. + assert.strictEqual(headers[`x-amz-checksum-${algorithm}`], undefined); + assert.strictEqual(headers['x-amz-checksum-type'], undefined); + }); + }); + + it('should emit FULL_OBJECT CRC64NVME for default MPU (no checksum headers)', async () => { + // AWS-verified: a default MPU still surfaces the CRC64NVME + // checksum and ChecksumType=FULL_OBJECT in the CompleteMPU response + // BODY (not headers). + await bucketPutP(); + const uploadId = await initiateMpuP({}); + await uploadPartP(uploadId); + const { xml, headers } = await completeMpuP(uploadId); + const json = await parseXmlP(xml); + const result = json.CompleteMultipartUploadResult; + assert(result.ChecksumCRC64NVME, 'default MPU should emit ChecksumCRC64NVME'); + assert.strictEqual(result.ChecksumType[0], 'FULL_OBJECT'); + assert.strictEqual(headers['x-amz-checksum-crc64nvme'], undefined); + assert.strictEqual(headers['x-amz-checksum-type'], undefined); + }); +}); From 06330052dcf86277b3f24c2d3c75c979fdb2d8bc Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 7 May 2026 23:33:11 +0200 Subject: [PATCH 5/7] CLDSRV-898: CompleteMPU checksum functional tests --- .../test/object/completeMpuChecksum.js | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js diff --git a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js new file mode 100644 index 0000000000..0b3b8869e8 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js @@ -0,0 +1,186 @@ +'use strict'; + +const assert = require('assert'); +const { + CreateBucketCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + HeadObjectCommand, + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); +const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); + +const bucket = `mpu-complete-checksum-${Date.now()}`; +const partBody = Buffer.from('I am a part body for complete-MPU testing', 'utf8'); + +// All AWS-valid (algorithm, type) pairs for an MPU. CompleteMPU should +// surface the resulting final-object checksum and ChecksumType in the +// response for every combination here, plus the implicit default. +const COMBOS = [ + { algo: 'CRC32', type: 'FULL_OBJECT' }, + { algo: 'CRC32', type: 'COMPOSITE' }, + { algo: 'CRC32C', type: 'FULL_OBJECT' }, + { algo: 'CRC32C', type: 'COMPOSITE' }, + { algo: 'CRC64NVME', type: 'FULL_OBJECT' }, + { algo: 'SHA1', type: 'COMPOSITE' }, + { algo: 'SHA256', type: 'COMPOSITE' }, +]; + +const tagField = algo => `Checksum${algo}`; + +describe('CompleteMultipartUpload final-object checksum', () => + withV4(sigCfg => { + let bucketUtil; + let s3; + + before(async () => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + }); + + after(async () => { + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + COMBOS.forEach(({ algo, type }) => { + const field = tagField(algo); + it(`should return ${algo}/${type} on CompleteMPU response`, async () => { + const key = `complete-${algo.toLowerCase()}-${type.toLowerCase()}-${Date.now()}`; + const partChecksum = await algorithms[algo.toLowerCase()].digest(partBody); + + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + ChecksumType: type, + }), + ); + + const uploadPart = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + [field]: partChecksum, + }), + ); + + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: uploadPart.ETag, + [field]: partChecksum, + }, + ], + }, + }), + ); + + assert(complete[field], `expected ${field} in CompleteMPU response, got: ${JSON.stringify(complete)}`); + assert.strictEqual(complete.ChecksumType, type); + if (type === 'COMPOSITE') { + assert( + complete[field].endsWith('-1'), + `expected -1 suffix for 1-part COMPOSITE, got ${complete[field]}`, + ); + } else { + assert( + !complete[field].includes('-'), + `FULL_OBJECT value should have no suffix, got ${complete[field]}`, + ); + } + + // HeadObject with ChecksumMode=ENABLED must surface the same + // value that CompleteMPU returned for FULL_OBJECT MPUs. + // COMPOSITE storage is deferred, so HeadObject leaves the field absent — matching + // cloudserver's current intentional skip. + const head = await s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + ChecksumMode: 'ENABLED', + }), + ); + if (type === 'FULL_OBJECT') { + assert.strictEqual( + head[field], + complete[field], + `HeadObject ${field} should match CompleteMPU response`, + ); + assert.strictEqual(head.ChecksumType, type); + } else { + assert.strictEqual( + head[field], + undefined, + `COMPOSITE storage is deferred; HeadObject should not surface ${field}`, + ); + assert.strictEqual(head.ChecksumType, undefined); + } + }); + }); + + it('should return CRC64NVME/FULL_OBJECT on CompleteMPU response when CreateMPU sent no checksum headers', async () => { + const key = `complete-default-${Date.now()}`; + + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); + + const uploadPart = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + }), + ); + + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + MultipartUpload: { + Parts: [{ PartNumber: 1, ETag: uploadPart.ETag }], + }, + }), + ); + + assert( + complete.ChecksumCRC64NVME, + `expected ChecksumCRC64NVME for default MPU, got: ${JSON.stringify(complete)}`, + ); + assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT'); + + // Default MPU is FULL_OBJECT — checksum is persisted, so + // HeadObject must return the same value. + const head = await s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + ChecksumMode: 'ENABLED', + }), + ); + assert.strictEqual(head.ChecksumCRC64NVME, complete.ChecksumCRC64NVME); + assert.strictEqual(head.ChecksumType, 'FULL_OBJECT'); + }); + })); From d9e5260fdf03e48a3518e744c992cefd48f0b180 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 12 May 2026 20:09:30 +0200 Subject: [PATCH 6/7] CLDSRV-898: reject Checksum field on a default MPU checksum --- lib/api/completeMultipartUpload.js | 12 ++ .../test/object/completeMpuChecksum.js | 169 +++++++++++++----- tests/unit/api/completeMultipartUpload.js | 82 +++++++-- 3 files changed, 202 insertions(+), 61 deletions(-) diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 2ac33fa843..991c12f908 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -83,6 +83,18 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu const presentTags = allChecksumXmlTags.filter(tag => part[tag]); + // AWS rejects any per-part Checksum field on a default MPU + // (one created without an explicit ChecksumAlgorithm) with + // InvalidPart — even when the value is correct and even when the + // field matches the implicit default algorithm. + if (mpuChecksum.isDefault && presentTags.length > 0) { + return errorInstances.InvalidPart.customizeDescription( + 'One or more of the specified parts could not be found. ' + + 'The part may not have been uploaded, or the specified ' + + "entity tag may not match the part's entity tag.", + ); + } + for (const tag of presentTags) { if (tag !== expectedTag) { const algoLabel = tag.replace(/^Checksum/, '').toLowerCase(); diff --git a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js index 0b3b8869e8..912b9d6fca 100644 --- a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js @@ -134,53 +134,126 @@ describe('CompleteMultipartUpload final-object checksum', () => }); }); - it('should return CRC64NVME/FULL_OBJECT on CompleteMPU response when CreateMPU sent no checksum headers', async () => { - const key = `complete-default-${Date.now()}`; - - const create = await s3.send( - new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - }), - ); - - const uploadPart = await s3.send( - new UploadPartCommand({ - Bucket: bucket, - Key: key, - UploadId: create.UploadId, - PartNumber: 1, - Body: partBody, - }), - ); - - const complete = await s3.send( - new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: create.UploadId, - MultipartUpload: { - Parts: [{ PartNumber: 1, ETag: uploadPart.ETag }], - }, - }), - ); - - assert( - complete.ChecksumCRC64NVME, - `expected ChecksumCRC64NVME for default MPU, got: ${JSON.stringify(complete)}`, - ); - assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT'); - - // Default MPU is FULL_OBJECT — checksum is persisted, so - // HeadObject must return the same value. - const head = await s3.send( - new HeadObjectCommand({ - Bucket: bucket, - Key: key, - ChecksumMode: 'ENABLED', - }), - ); - assert.strictEqual(head.ChecksumCRC64NVME, complete.ChecksumCRC64NVME); - assert.strictEqual(head.ChecksumType, 'FULL_OBJECT'); + it( + 'should return CRC64NVME/FULL_OBJECT on CompleteMPU response ' + 'when CreateMPU sent no checksum headers', + async () => { + const key = `complete-default-${Date.now()}`; + + const create = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); + + const uploadPart = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + }), + ); + + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + MultipartUpload: { + Parts: [{ PartNumber: 1, ETag: uploadPart.ETag }], + }, + }), + ); + + assert( + complete.ChecksumCRC64NVME, + `expected ChecksumCRC64NVME for default MPU, got: ${JSON.stringify(complete)}`, + ); + assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT'); + + // Default MPU is FULL_OBJECT — checksum is persisted, so + // HeadObject must return the same value. + const head = await s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + ChecksumMode: 'ENABLED', + }), + ); + assert.strictEqual(head.ChecksumCRC64NVME, complete.ChecksumCRC64NVME); + assert.strictEqual(head.ChecksumType, 'FULL_OBJECT'); + }, + ); + + // AWS S3 rejects any per-part + // Checksum field on a default MPU (one created without an + // explicit ChecksumAlgorithm) with InvalidPart — even when the + // field matches the implicit CRC64NVME algorithm and value. + describe('default MPU rejects per-part Checksum fields', () => { + async function setupDefaultMpu() { + const key = `complete-default-rejects-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const create = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })); + const uploadPart = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + }), + ); + return { key, uploadId: create.UploadId, eTag: uploadPart.ETag }; + } + + async function assertInvalidPart(promise) { + let caught; + try { + await promise; + } catch (err) { + caught = err; + } + assert(caught, 'expected CompleteMPU to reject'); + assert.strictEqual( + caught.name, + 'InvalidPart', + `expected InvalidPart, got ${caught.name}: ${caught.message}`, + ); + } + + it('should return InvalidPart when Part includes matching ChecksumCRC64NVME (correct value)', async () => { + const { key, uploadId, eTag } = await setupDefaultMpu(); + const crc64 = await algorithms.crc64nvme.digest(partBody); + await assertInvalidPart( + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ PartNumber: 1, ETag: eTag, ChecksumCRC64NVME: crc64 }], + }, + }), + ), + ); + }); + + it('should return InvalidPart when Part includes a non-matching algorithm field', async () => { + const { key, uploadId, eTag } = await setupDefaultMpu(); + const crc32 = await algorithms.crc32.digest(partBody); + await assertInvalidPart( + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ PartNumber: 1, ETag: eTag, ChecksumCRC32: crc32 }], + }, + }), + ), + ); + }); }); })); diff --git a/tests/unit/api/completeMultipartUpload.js b/tests/unit/api/completeMultipartUpload.js index 1e6044c82e..e9157e0dce 100644 --- a/tests/unit/api/completeMultipartUpload.js +++ b/tests/unit/api/completeMultipartUpload.js @@ -40,17 +40,19 @@ const SAMPLE_DIGESTS = { sha256: ['YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=', 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI='], }; -// Every AWS-valid (algorithm, type) combination, plus the implicit default. +// Every AWS-valid (algorithm, type) combination for an explicit-algorithm MPU. // See validateChecksums.getChecksumDataFromMPUHeaders for the source of truth. +// The implicit-default MPU (isDefault=true) is tested separately because AWS +// rejects any per-part Checksum field on a default MPU with InvalidPart, +// regardless of value or algorithm. const MATRIX = [ - { algorithm: 'crc32', type: 'COMPOSITE', isDefault: false }, - { algorithm: 'crc32', type: 'FULL_OBJECT', isDefault: false }, - { algorithm: 'crc32c', type: 'COMPOSITE', isDefault: false }, - { algorithm: 'crc32c', type: 'FULL_OBJECT', isDefault: false }, - { algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: false }, - { algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: true }, - { algorithm: 'sha1', type: 'COMPOSITE', isDefault: false }, - { algorithm: 'sha256', type: 'COMPOSITE', isDefault: false }, + { algorithm: 'crc32', type: 'COMPOSITE' }, + { algorithm: 'crc32', type: 'FULL_OBJECT' }, + { algorithm: 'crc32c', type: 'COMPOSITE' }, + { algorithm: 'crc32c', type: 'FULL_OBJECT' }, + { algorithm: 'crc64nvme', type: 'FULL_OBJECT' }, + { algorithm: 'sha1', type: 'COMPOSITE' }, + { algorithm: 'sha256', type: 'COMPOSITE' }, ]; function makeStoredPart(partNumber, checksum) { @@ -88,11 +90,11 @@ function pickWrongAlgo(algo) { describe('validatePerPartChecksums', () => { describe('AWS combination matrix', () => { - MATRIX.forEach(({ algorithm, type, isDefault }) => { - const label = `${algorithm}/${type}${isDefault ? ' (default)' : ''}`; + MATRIX.forEach(({ algorithm, type }) => { + const label = `${algorithm}/${type}`; const tag = TAG_BY_ALGO[algorithm]; const [d1, d2] = SAMPLE_DIGESTS[algorithm]; - const mpuChecksum = { algorithm, type, isDefault }; + const mpuChecksum = { algorithm, type, isDefault: false }; const stored = [makeStoredPart(1, { algorithm, value: d1 }), makeStoredPart(2, { algorithm, value: d2 })]; @@ -143,7 +145,7 @@ describe('validatePerPartChecksums', () => { ); }); - const requiresPerPart = type === 'COMPOSITE' && !isDefault; + const requiresPerPart = type === 'COMPOSITE'; const missingLabel = requiresPerPart ? 'should return InvalidRequest when a part is missing its checksum' : 'should accept a parts list missing per-part checksums'; @@ -165,6 +167,60 @@ describe('validatePerPartChecksums', () => { }); }); + describe('default MPU (isDefault=true)', () => { + // AWS S3 rejects any per-part + // Checksum field on a default MPU with InvalidPart — even when + // the field matches the implicit CRC64NVME algorithm and the value + // is the same one the part was stored with. + const mpuChecksum = { algorithm: 'crc64nvme', type: 'FULL_OBJECT', isDefault: true }; + const [d1, d2] = SAMPLE_DIGESTS.crc64nvme; + const stored = [ + makeStoredPart(1, { algorithm: 'crc64nvme', value: d1 }), + makeStoredPart(2, { algorithm: 'crc64nvme', value: d2 }), + ]; + const invalidPartMessage = + 'One or more of the specified parts could not be ' + + 'found. The part may not have been uploaded, or ' + + 'the specified entity tag may not match the ' + + "part's entity tag."; + + it('should accept when no parts include a checksum field', () => { + const jsonList = { Part: [makeJsonPart(1, 'etag1'), makeJsonPart(2, 'etag2')] }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert.strictEqual(err, null); + }); + + it('should return InvalidPart when a part includes the matching field (correct value)', () => { + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { ChecksumCRC64NVME: d1 }), makeJsonPart(2, 'etag2')], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert(err); + assert.strictEqual(err.is.InvalidPart, true); + assert.strictEqual(err.description, invalidPartMessage); + }); + + it('should return InvalidPart when a part includes the matching field (wrong value)', () => { + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { ChecksumCRC64NVME: d2 }), makeJsonPart(2, 'etag2')], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert(err); + assert.strictEqual(err.is.InvalidPart, true); + assert.strictEqual(err.description, invalidPartMessage); + }); + + it('should return InvalidPart when a part includes a non-matching algorithm field', () => { + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { ChecksumCRC32: SAMPLE_DIGESTS.crc32[0] }), makeJsonPart(2, 'etag2')], + }; + const err = validatePerPartChecksums(jsonList, stored, SPLITTER, mpuChecksum); + assert(err); + assert.strictEqual(err.is.InvalidPart, true); + assert.strictEqual(err.description, invalidPartMessage); + }); + }); + describe('legacy MPU (no algorithm configured)', () => { // Pre-feature MPUs have storedMetadata.checksumAlgorithm === undefined. // Pre-PR CompleteMPU silently ignored any per-part Checksum body From 27b4a43d3e3097377f68865db59576c3c9aabb18 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 12 May 2026 22:33:13 +0200 Subject: [PATCH 7/7] CLDSRV-898: reject CompleteMPU if explicit checksum but checksum missing in part metadata --- lib/api/completeMultipartUpload.js | 77 +++++++----- tests/unit/api/completeMultipartUpload.js | 142 +++++++++++++++------- 2 files changed, 141 insertions(+), 78 deletions(-) diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 991c12f908..56b490ad46 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -128,12 +128,21 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu /** * Compute the final-object checksum for a CompleteMultipartUpload from the - * stored MPU configuration and per-part checksums. Returns null when the MPU - * has no checksum configured, when any part is missing its stored - * ChecksumValue (defensive — should not happen in steady state), or when the - * compute primitive itself reports an error. Errors are logged; the caller - * proceeds without a final-object checksum so the response simply omits the - * checksum fields rather than failing the whole CompleteMPU. + * stored MPU configuration and per-part checksums. + * + * Returns `{ result, error }`: + * - Success: `{ result: { algorithm, type, value }, error: null }` + * - No MPU checksum configured: `{ result: null, error: null }` + * - "Should-not-happen" branch (missing stored ChecksumValue, unknown + * checksumType, compute primitive error): behavior depends on whether + * the client opted in to checksums — + * - Default MPU (`checksumIsDefault === true`): the client never + * asked for a checksum; soft-fail with `{ result: null, error: null }` + * so the response simply omits the checksum field. + * - Explicit MPU (`checksumIsDefault === false`): the client asked + * for a checksum at CreateMPU; silently dropping it would violate + * the contract. Return `{ result: null, error: }` + * for the caller to fail the CompleteMPU. * * @param {array} storedParts - parts from services.getMPUparts (carry ChecksumValue) * @param {array} filteredPartList - validated, ordered subset matching jsonList @@ -141,13 +150,22 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu * @param {string} mpuSplitter - splitter used in part keys * @param {string} uploadId - for log context * @param {object} log - werelogs logger - * @returns {object|null} { algorithm, type, value } or null + * @returns {Promise<{ result: ({algorithm, type, value}|null), error: (ArsenalError|null) }>} */ async function computeFinalChecksum(storedParts, filteredPartList, storedMetadata, mpuSplitter, uploadId, log) { const algorithm = storedMetadata.checksumAlgorithm; const type = storedMetadata.checksumType; if (!algorithm || !type) { - return null; + return { result: null, error: null }; + } + const isDefault = !!storedMetadata.checksumIsDefault; + + function failOrSkip(message, extra) { + log.error(message, { uploadId, algorithm, type, ...extra }); + if (isDefault) { + return { result: null, error: null }; + } + return { result: null, error: errorInstances.InternalError.customizeDescription(message) }; } const storedByKey = new Map(); @@ -166,41 +184,29 @@ async function computeFinalChecksum(storedParts, filteredPartList, storedMetadat } if (missingPartNumbers.length > 0) { - log.error('one or more MPU parts missing checksum value; ' + 'skipping final-object checksum computation', { - uploadId, - algorithm, - type, - missingPartNumbers, - }); - return null; + return failOrSkip('one or more MPU parts missing checksum value', { missingPartNumbers }); } - let result; + let computed; if (type === 'COMPOSITE') { - result = await computeCompositeMPUChecksum( + computed = await computeCompositeMPUChecksum( algorithm, partInputs.map(p => p.value), ); } else if (type === 'FULL_OBJECT') { - result = computeFullObjectMPUChecksum(algorithm, partInputs); + computed = computeFullObjectMPUChecksum(algorithm, partInputs); } else { - log.error('unknown MPU checksumType; skipping final-object checksum computation', { - uploadId, - checksumType: type, - }); - return null; + return failOrSkip('unknown MPU checksumType', { checksumType: type }); } - if (result.error) { - log.error('final-object checksum computation failed', { - uploadId, - checksumErrorCode: result.error.code, - checksumErrorDetails: result.error.details, + if (computed.error) { + return failOrSkip('final-object checksum computation failed', { + checksumErrorCode: computed.error.code, + checksumErrorDetails: computed.error.details, }); - return null; } - return { algorithm, type, value: result.checksum }; + return { result: { algorithm, type, value: computed.checksum }, error: null }; } /* @@ -613,9 +619,16 @@ function completeMultipartUpload(authInfo, request, log, callback) { uploadId, log, ).then( - fc => { + ({ result, error }) => { + if (error) { + log.error('failing CompleteMPU due to final-object checksum error', { + uploadId, + error, + }); + return callNext(error, destBucket); + } try { - finalChecksum = fc; + finalChecksum = result; const checksumErr = validateCompleteMultipartUploadChecksum(request.headers, finalChecksum); if (checksumErr) { log.debug('x-amz-checksum- header validation failed on CompleteMPU', { diff --git a/tests/unit/api/completeMultipartUpload.js b/tests/unit/api/completeMultipartUpload.js index e9157e0dce..b22be6fbe9 100644 --- a/tests/unit/api/completeMultipartUpload.js +++ b/tests/unit/api/completeMultipartUpload.js @@ -573,13 +573,23 @@ describe('computeFinalChecksum', () => { })); } - it('should return null when MPU has no checksumAlgorithm', async () => { + function assertSoftNull(got) { + assert.deepStrictEqual(got, { result: null, error: null }); + } + + function assertInternalError(got) { + assert.strictEqual(got.result, null); + assert(got.error, 'expected an error on the result'); + assert.strictEqual(got.error.is.InternalError, true); + } + + it('should return { result: null, error: null } when MPU has no checksumAlgorithm', async () => { const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; const got = await computeFinalChecksum(stored, partListFromStored(stored), {}, SPLITTER, uploadId, log); - assert.strictEqual(got, null); + assertSoftNull(got); }); - it('should return null when MPU has no checksumType', async () => { + it('should return { result: null, error: null } when MPU has no checksumType', async () => { const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; const got = await computeFinalChecksum( stored, @@ -589,7 +599,7 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assert.strictEqual(got, null); + assertSoftNull(got); }); it('should return COMPOSITE checksum with -N suffix for SHA256 MPU', async () => { @@ -599,7 +609,7 @@ describe('computeFinalChecksum', () => { makeStoredPart(2, { algorithm: 'sha256', value: d2 }), makeStoredPart(3, { algorithm: 'sha256', value: d3 }), ]; - const got = await computeFinalChecksum( + const { result, error } = await computeFinalChecksum( stored, partListFromStored(stored), { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE' }, @@ -607,17 +617,18 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assert(got); - assert.strictEqual(got.algorithm, 'sha256'); - assert.strictEqual(got.type, 'COMPOSITE'); - assert(got.value.endsWith('-3'), `expected -N suffix, got ${got.value}`); + assert.strictEqual(error, null); + assert(result); + assert.strictEqual(result.algorithm, 'sha256'); + assert.strictEqual(result.type, 'COMPOSITE'); + assert(result.value.endsWith('-3'), `expected -N suffix, got ${result.value}`); // computeCompositeMPUChecksum's deterministic output for these // exact placeholder digests: const expected = crypto .createHash('sha256') .update(Buffer.concat([d1, d2, d3].map(x => Buffer.from(x, 'base64')))) .digest('base64'); - assert.strictEqual(got.value, `${expected}-3`); + assert.strictEqual(result.value, `${expected}-3`); }); ['sha1', 'crc32', 'crc32c'].forEach(algo => { @@ -627,7 +638,7 @@ describe('computeFinalChecksum', () => { makeStoredPart(1, { algorithm: algo, value: d1 }), makeStoredPart(2, { algorithm: algo, value: d2 }), ]; - const got = await computeFinalChecksum( + const { result, error } = await computeFinalChecksum( stored, partListFromStored(stored), { checksumAlgorithm: algo, checksumType: 'COMPOSITE' }, @@ -635,10 +646,11 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assert(got); - assert.strictEqual(got.algorithm, algo); - assert.strictEqual(got.type, 'COMPOSITE'); - assert(got.value.endsWith('-2')); + assert.strictEqual(error, null); + assert(result); + assert.strictEqual(result.algorithm, algo); + assert.strictEqual(result.type, 'COMPOSITE'); + assert(result.value.endsWith('-2')); }); }); @@ -671,7 +683,7 @@ describe('computeFinalChecksum', () => { }, }, ]; - const got = await computeFinalChecksum( + const { result, error } = await computeFinalChecksum( stored, partListFromStored(stored), { checksumAlgorithm: 'crc64nvme', checksumType: 'FULL_OBJECT' }, @@ -679,15 +691,39 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assert(got); - assert.strictEqual(got.algorithm, 'crc64nvme'); - assert.strictEqual(got.type, 'FULL_OBJECT'); - assert(!got.value.includes('-'), `FULL_OBJECT should have no -N suffix, got ${got.value}`); + assert.strictEqual(error, null); + assert(result); + assert.strictEqual(result.algorithm, 'crc64nvme'); + assert.strictEqual(result.type, 'FULL_OBJECT'); + assert(!result.value.includes('-'), `FULL_OBJECT should have no -N suffix, got ${result.value}`); const expected = await algorithms.crc64nvme.digest(Buffer.concat([a, b])); - assert.strictEqual(got.value, expected); + assert.strictEqual(result.value, expected); + }); + + // Soft-null (`{ result: null, error: null }`) is intentional only for + // default MPUs — the client didn't opt in to a checksum, so missing it + // on the response is graceful degradation. Explicit MPUs return + // `{ result: null, error: InternalError }` because silently dropping + // a checksum the client asked for would violate the CreateMPU contract. + + it('should soft-null when a default-MPU part is missing ChecksumValue', async () => { + const stored = [ + makeStoredPart(1, { algorithm: 'crc64nvme', value: SAMPLE_DIGESTS.crc64nvme[0] }), + makeStoredPart(2, null), + makeStoredPart(3, { algorithm: 'crc64nvme', value: SAMPLE_DIGESTS.crc64nvme[1] }), + ]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'crc64nvme', checksumType: 'FULL_OBJECT', checksumIsDefault: true }, + SPLITTER, + uploadId, + log, + ); + assertSoftNull(got); }); - it('should return null and log when a part is missing ChecksumValue', async () => { + it('should return InternalError when an explicit-MPU part is missing ChecksumValue', async () => { const stored = [ makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] }), makeStoredPart(2, null), @@ -696,42 +732,55 @@ describe('computeFinalChecksum', () => { const got = await computeFinalChecksum( stored, partListFromStored(stored), - { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE' }, + { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE', checksumIsDefault: false }, + SPLITTER, + uploadId, + log, + ); + assertInternalError(got); + }); + + it('should soft-null when checksumType is unknown on a default MPU', async () => { + const stored = [makeStoredPart(1, { algorithm: 'crc64nvme', value: SAMPLE_DIGESTS.crc64nvme[0] })]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'crc64nvme', checksumType: 'WEIRD', checksumIsDefault: true }, SPLITTER, uploadId, log, ); - assert.strictEqual(got, null); + assertSoftNull(got); }); - it('should return null when checksumType is unknown', async () => { + it('should return InternalError when checksumType is unknown on an explicit MPU', async () => { const stored = [makeStoredPart(1, { algorithm: 'sha256', value: SAMPLE_DIGESTS.sha256[0] })]; const got = await computeFinalChecksum( stored, partListFromStored(stored), - { checksumAlgorithm: 'sha256', checksumType: 'WEIRD' }, + { checksumAlgorithm: 'sha256', checksumType: 'WEIRD', checksumIsDefault: false }, SPLITTER, uploadId, log, ); - assert.strictEqual(got, null); + assertInternalError(got); }); - it( - 'should return null when underlying compute reports an error ' + '(crc64nvme COMPOSITE is not allowed)', - async () => { - const stored = [makeStoredPart(1, { algorithm: 'crc64nvme', value: SAMPLE_DIGESTS.crc64nvme[0] })]; - const got = await computeFinalChecksum( - stored, - partListFromStored(stored), - { checksumAlgorithm: 'crc64nvme', checksumType: 'COMPOSITE' }, - SPLITTER, - uploadId, - log, - ); - assert.strictEqual(got, null); - }, - ); + it('should return InternalError when underlying compute reports an error on an explicit MPU', async () => { + // crc64nvme + COMPOSITE is not allowed by computeCompositeMPUChecksum. + // Reaching here on an explicit MPU means upstream validation failed, + // which is exactly the kind of internal-state bug we want to surface. + const stored = [makeStoredPart(1, { algorithm: 'crc64nvme', value: SAMPLE_DIGESTS.crc64nvme[0] })]; + const got = await computeFinalChecksum( + stored, + partListFromStored(stored), + { checksumAlgorithm: 'crc64nvme', checksumType: 'COMPOSITE', checksumIsDefault: false }, + SPLITTER, + uploadId, + log, + ); + assertInternalError(got); + }); it('should compute over filteredPartList (subset), not all storedParts', async () => { const [d1, d2, d3] = [SAMPLE_DIGESTS.sha256[0], SAMPLE_DIGESTS.sha256[1], SAMPLE_DIGESTS.sha256[0]]; @@ -747,7 +796,7 @@ describe('computeFinalChecksum', () => { size: s.value.Size, locations: s.value.partLocations, })); - const got = await computeFinalChecksum( + const { result, error } = await computeFinalChecksum( stored, filtered, { checksumAlgorithm: 'sha256', checksumType: 'COMPOSITE' }, @@ -755,13 +804,14 @@ describe('computeFinalChecksum', () => { uploadId, log, ); - assert(got); - assert(got.value.endsWith('-2'), `should reflect 2 completed parts, got ${got.value}`); + assert.strictEqual(error, null); + assert(result); + assert(result.value.endsWith('-2'), `should reflect 2 completed parts, got ${result.value}`); const expected = crypto .createHash('sha256') .update(Buffer.concat([d1, d3].map(x => Buffer.from(x, 'base64')))) .digest('base64'); - assert.strictEqual(got.value, `${expected}-2`); + assert.strictEqual(result.value, `${expected}-2`); }); });