Skip to content

CLDSRV-903: lint files and bump arsenal #6168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leif-scality wants to merge 3 commits into
base: development/9.4
Choose a base branch
from
Open

CLDSRV-903: lint files and bump arsenal #6168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d27b1c
CLDSRV-903: prettier format files
leif-scality May 13, 2026
8e80eb0
CLDSRV-903: bump arsenal to 8.4.2
leif-scality May 13, 2026
93efed1
CLDSRV-903: rename getObjectAttributesXMLTag to xmlTag
leif-scality May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 65 additions & 52 deletions lib/api/apiUtils/integrity/validateChecksums.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,59 @@ const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
const { errors: ArsenalErrors, errorInstances } = require('arsenal');
const { config } = require('../../../Config');

const defaultChecksumData = Object.freeze(
{ algorithm: 'crc64nvme', isTrailer: false, expected: undefined });
const defaultChecksumData = Object.freeze({ algorithm: 'crc64nvme', isTrailer: false, expected: undefined });

const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
'The algorithm type you specified in x-amz-checksum- header is invalid.');
'The algorithm type you specified in x-amz-checksum- header is invalid.',
);
const errAlgoNotSupportedSDK = errorInstances.InvalidRequest.customizeDescription(
'Value for x-amz-sdk-checksum-algorithm header is invalid.');
'Value for x-amz-sdk-checksum-algorithm header is invalid.',
);
const errMissingCorresponding = errorInstances.InvalidRequest.customizeDescription(
'x-amz-sdk-checksum-algorithm specified, but no corresponding x-amz-checksum-* ' +
'or x-amz-trailer headers were found.');
'or x-amz-trailer headers were found.',
);
const errMultipleChecksumTypes = errorInstances.InvalidRequest.customizeDescription(
'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.');
'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.',
);
const errTrailerAndChecksum = errorInstances.InvalidRequest.customizeDescription(
'Expecting a single x-amz-checksum- header');
'Expecting a single x-amz-checksum- header',
);
const errTrailerNotSupported = errorInstances.InvalidRequest.customizeDescription(
'The value specified in the x-amz-trailer header is not supported');
'The value specified in the x-amz-trailer header is not supported',
);
const errMPUAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
'Checksum algorithm provided is unsupported. ' +
'Please try again with any of the valid types: ' +
'[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]');
'Please try again with any of the valid types: ' +
'[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]',
);
const errMPUTypeInvalid = errorInstances.InvalidRequest.customizeDescription(
'Value for x-amz-checksum-type header is invalid.');
'Value for x-amz-checksum-type header is invalid.',
);
const errMPUTypeWithoutAlgo = errorInstances.InvalidRequest.customizeDescription(
'The x-amz-checksum-type header can only be used ' +
'with the x-amz-checksum-algorithm header.');
'The x-amz-checksum-type header can only be used ' + 'with the x-amz-checksum-algorithm header.',
Comment thread
leif-scality marked this conversation as resolved.
);

const checksumedMethods = Object.freeze({
'completeMultipartUpload': true,
'multiObjectDelete': true,
'bucketPutACL': true,
'bucketPutCors': true,
'bucketPutEncryption': true,
'bucketPutLifecycle': true,
'bucketPutLogging': true,
'bucketPutNotification': true,
'bucketPutPolicy': true,
'bucketPutReplication': true,
'bucketPutTagging': true,
'bucketPutVersioning': true,
'bucketPutWebsite': true,
'objectPutACL': true,
'objectPutLegalHold': true,
'bucketPutObjectLock': true, // PutObjectLockConfiguration
'objectPutRetention': true,
'objectPutTagging': true,
'objectRestore': true,
completeMultipartUpload: true,
multiObjectDelete: true,
bucketPutACL: true,
bucketPutCors: true,
bucketPutEncryption: true,
bucketPutLifecycle: true,
bucketPutLogging: true,
bucketPutNotification: true,
bucketPutPolicy: true,
bucketPutReplication: true,
bucketPutTagging: true,
bucketPutVersioning: true,
bucketPutWebsite: true,
objectPutACL: true,
objectPutLegalHold: true,
bucketPutObjectLock: true, // PutObjectLockConfiguration
objectPutRetention: true,
objectPutTagging: true,
objectRestore: true,
});

const ChecksumError = Object.freeze({
Expand Down Expand Up @@ -85,7 +92,7 @@ function uint32ToBase64(num) {

const algorithms = Object.freeze({
crc64nvme: {
getObjectAttributesXMLTag: 'ChecksumCRC64NVME',
xmlTag: 'ChecksumCRC64NVME',
digest: async data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
const crc = new CrtCrc64Nvme();
Expand All @@ -98,10 +105,10 @@ const algorithms = Object.freeze({
return Buffer.from(result).toString('base64');
},
isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected),
createHash: () => new CrtCrc64Nvme()
createHash: () => new CrtCrc64Nvme(),
},
crc32: {
getObjectAttributesXMLTag: 'ChecksumCRC32',
xmlTag: 'ChecksumCRC32',
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
Expand All @@ -111,38 +118,38 @@ const algorithms = Object.freeze({
return uint32ToBase64(result >>> 0);
},
isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
createHash: () => new Crc32()
createHash: () => new Crc32(),
},
crc32c: {
getObjectAttributesXMLTag: 'ChecksumCRC32C',
xmlTag: 'ChecksumCRC32C',
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
},
digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0),
isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
createHash: () => new Crc32c()
createHash: () => new Crc32c(),
},
sha1: {
getObjectAttributesXMLTag: 'ChecksumSHA1',
xmlTag: 'ChecksumSHA1',
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return crypto.createHash('sha1').update(input).digest('base64');
},
digestFromHash: hash => hash.digest('base64'),
isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected),
createHash: () => crypto.createHash('sha1')
createHash: () => crypto.createHash('sha1'),
},
sha256: {
getObjectAttributesXMLTag: 'ChecksumSHA256',
xmlTag: 'ChecksumSHA256',
digest: data => {
const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
return crypto.createHash('sha256').update(input).digest('base64');
},
digestFromHash: hash => hash.digest('base64'),
isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected),
createHash: () => crypto.createHash('sha256')
}
createHash: () => crypto.createHash('sha256'),
},
});

async function validateXAmzChecksums(headers, body) {
Expand All @@ -155,7 +162,7 @@ async function validateXAmzChecksums(headers, body) {
if (xAmzChecksumCnt === 0 && 'x-amz-sdk-checksum-algorithm' in headers) {
return {
error: ChecksumError.MissingCorresponding,
details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
details: { expected: headers['x-amz-sdk-checksum-algorithm'] },
};
} else if (xAmzChecksumCnt === 0) {
return { error: ChecksumError.MissingChecksum, details: null };
Expand All @@ -164,7 +171,7 @@ async function validateXAmzChecksums(headers, body) {
// No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
const algo = checksumHeaders[0].slice('x-amz-checksum-'.length);
if (!(algo in algorithms)) {
return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };;
return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
}

const expected = headers[`x-amz-checksum-${algo}`];
Expand Down Expand Up @@ -232,7 +239,7 @@ function getChecksumDataFromHeaders(headers) {
if (checksumHeader === undefined && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) {
return {
error: ChecksumError.MissingCorresponding,
details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
details: { expected: headers['x-amz-sdk-checksum-algorithm'] },
};
}

Expand Down Expand Up @@ -335,7 +342,8 @@ function arsenalErrorFromChecksumError(err) {
case ChecksumError.XAmzMismatch: {
const algoUpper = err.details.algorithm.toUpperCase();
return errorInstances.BadDigest.customizeDescription(
`The ${algoUpper} you specified did not match the calculated checksum.`);
`The ${algoUpper} you specified did not match the calculated checksum.`,
);
}
case ChecksumError.AlgoNotSupported:
return errAlgoNotSupported;
Expand All @@ -347,7 +355,8 @@ function arsenalErrorFromChecksumError(err) {
return errMultipleChecksumTypes;
case ChecksumError.MalformedChecksum:
return errorInstances.InvalidRequest.customizeDescription(
`Value for x-amz-checksum-${err.details.algorithm} header is invalid.`);
`Value for x-amz-checksum-${err.details.algorithm} header is invalid.`,
);
case ChecksumError.MD5Invalid:
return ArsenalErrors.InvalidDigest;
case ChecksumError.TrailerAlgoMismatch:
Expand All @@ -358,7 +367,8 @@ function arsenalErrorFromChecksumError(err) {
return ArsenalErrors.MalformedTrailerError;
case ChecksumError.TrailerChecksumMalformed:
return errorInstances.InvalidRequest.customizeDescription(
`Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`);
`Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`,
);
case ChecksumError.TrailerAndChecksum:
return errTrailerAndChecksum;
case ChecksumError.TrailerNotSupported:
Expand All @@ -372,7 +382,8 @@ function arsenalErrorFromChecksumError(err) {
case ChecksumError.MPUInvalidCombination:
return errorInstances.InvalidRequest.customizeDescription(
`The ${err.details.type} checksum type cannot be used ` +
`with the ${err.details.algorithm.toUpperCase()} checksum algorithm.`);
`with the ${err.details.algorithm.toUpperCase()} checksum algorithm.`,
);
default:
return ArsenalErrors.BadDigest;
}
Expand Down Expand Up @@ -464,8 +475,10 @@ function getChecksumDataFromMPUHeaders(headers) {
}

// Validate algorithm + type combination
if ((type === 'FULL_OBJECT' && !fullObjectAlgorithms.has(algo)) ||
(type === 'COMPOSITE' && !compositeAlgorithms.has(algo))) {
if (
(type === 'FULL_OBJECT' && !fullObjectAlgorithms.has(algo)) ||
(type === 'COMPOSITE' && !compositeAlgorithms.has(algo))
) {
return { error: ChecksumError.MPUInvalidCombination, details: { algorithm: algo, type } };
}

Expand Down
8 changes: 2 additions & 6 deletions lib/api/apiUtils/object/objectAttributes.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,7 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml, log) {
case 'ObjectParts': {
const partCount = getPartCountFromMd5(objectMD);
if (partCount) {
xml.push(
'<ObjectParts>',
`<PartsCount>${partCount}</PartsCount>`,
'</ObjectParts>',
);
xml.push('<ObjectParts>', `<PartsCount>${partCount}</PartsCount>`, '</ObjectParts>');
}
break;
}
Expand All @@ -93,7 +89,7 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml, log) {
});
break;
}
const tag = algo.getObjectAttributesXMLTag;
const tag = algo.xmlTag;
xml.push(
'<Checksum>',
`<${tag}>${checksum.checksumValue}</${tag}>`,
Expand Down
Loading
Loading