diff --git a/lib/api/apiUtils/bucket/bucketCreation.js b/lib/api/apiUtils/bucket/bucketCreation.js index ca43c09a82..b5612d31b7 100644 --- a/lib/api/apiUtils/bucket/bucketCreation.js +++ b/lib/api/apiUtils/bucket/bucketCreation.js @@ -18,7 +18,6 @@ const oldUsersBucket = constants.oldUsersBucket; const zenkoSeparator = constants.zenkoSeparator; const userBucketOwner = 'admin'; - function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) { // BACKWARD: Simplify once do not have to deal with old // usersbucket name and old splitter @@ -28,8 +27,7 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) { if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) { return cb(err); } - const splitter = usersBucketAttrs ? - constants.splitter : constants.oldSplitter; + const splitter = usersBucketAttrs ? constants.splitter : constants.oldSplitter; let key = createKeyForUserBucket(canonicalID, splitter, bucketName); const omVal = { creationDate: new Date().toJSON(), @@ -38,46 +36,44 @@ function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) { // If the new format usersbucket does not exist, try to put the // key in the old usersBucket using the old splitter. // Otherwise put the key in the new format usersBucket - const usersBucketBeingCalled = usersBucketAttrs ? - usersBucket : oldUsersBucket; - return metadata.putObjectMD(usersBucketBeingCalled, key, - omVal, {}, log, err => { - if (err?.is?.NoSuchBucket) { - // There must be no usersBucket so createBucket - // one using the new format - log.trace('users bucket does not exist, ' + - 'creating users bucket'); - key = `${canonicalID}${constants.splitter}` + - `${bucketName}`; - const creationDate = new Date().toJSON(); - const freshBucket = new BucketInfo(usersBucket, - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion()); - return metadata.createBucket(usersBucket, - freshBucket, log, err => { - // Note: In the event that two - // users' requests try to create the - // usersBucket at the same time, - // this will prevent one of the users - // from getting a BucketAlreadyExists - // error with respect - // to the usersBucket. - // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver - if (err && !err.BucketAlreadyExists) { - log.error('error from metadata', { - error: err, - }); - return cb(err); - } - log.trace('Users bucket created'); - // Finally put the key in the new format - // usersBucket - return metadata.putObjectMD(usersBucket, - key, omVal, {}, log, cb); + const usersBucketBeingCalled = usersBucketAttrs ? usersBucket : oldUsersBucket; + return metadata.putObjectMD(usersBucketBeingCalled, key, omVal, {}, log, err => { + if (err?.is?.NoSuchBucket) { + // There must be no usersBucket so createBucket + // one using the new format + log.trace('users bucket does not exist, ' + 'creating users bucket'); + key = `${canonicalID}${constants.splitter}` + `${bucketName}`; + const creationDate = new Date().toJSON(); + const freshBucket = new BucketInfo( + usersBucket, + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), + ); + return metadata.createBucket(usersBucket, freshBucket, log, err => { + // Note: In the event that two + // users' requests try to create the + // usersBucket at the same time, + // this will prevent one of the users + // from getting a BucketAlreadyExists + // error with respect + // to the usersBucket. + // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver + if (err && !err.BucketAlreadyExists) { + log.error('error from metadata', { + error: err, }); - } - return cb(err); - }); + return cb(err); + } + log.trace('Users bucket created'); + // Finally put the key in the new format + // usersBucket + return metadata.putObjectMD(usersBucket, key, omVal, {}, log, cb); + }); + } + return cb(err); + }); }); } @@ -89,6 +85,18 @@ function removeTransientOrDeletedLabel(bucket, log, callback) { return metadata.updateBucket(bucketName, bucket, log, callback); } +function seedBucketQuotaCapacity(bucket, log, done) { + if (!config.isQuotaEnabled()) { + return done(); + } + return metadata.initializeBucketCapacity(bucket.getName(), bucket.getCreationDate(), log, err => { + if (err) { + log.error('error seeding bucket quota capacity metric', { error: err }); + } + return done(); + }); +} + function freshStartCreateBucket(bucket, canonicalID, log, callback) { const bucketName = bucket.getName(); metadata.createBucket(bucketName, bucket, log, err => { @@ -101,7 +109,12 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) { if (err) { return callback(err); } - return removeTransientOrDeletedLabel(bucket, log, callback); + return removeTransientOrDeletedLabel(bucket, log, labelErr => { + if (labelErr) { + return callback(labelErr); + } + return seedBucketQuotaCapacity(bucket, log, () => callback(null)); + }); }); }); } @@ -124,7 +137,12 @@ function cleanUpBucket(bucketMD, canonicalID, log, callback) { if (err) { return callback(err); } - return removeTransientOrDeletedLabel(bucketMD, log, callback); + return removeTransientOrDeletedLabel(bucketMD, log, labelErr => { + if (labelErr) { + return callback(labelErr); + } + return seedBucketQuotaCapacity(bucketMD, log, () => callback(null)); + }); }); } @@ -139,16 +157,15 @@ function cleanUpBucket(bucketMD, canonicalID, log, callback) { * @callback called with (err, sseInfo: object) */ function bucketLevelServerSideEncryption(bucket, headers, log, cb) { - kms.bucketLevelEncryption( - bucket, headers, log, (err, sseInfo) => { - if (err) { - log.debug('error getting bucket encryption info', { - error: err, - }); - return cb(err); - } - return cb(null, sseInfo); - }); + kms.bucketLevelEncryption(bucket, headers, log, (err, sseInfo) => { + if (err) { + log.debug('error getting bucket encryption info', { + error: err, + }); + return cb(err); + } + return cb(null, sseInfo); + }); } /** @@ -163,28 +180,44 @@ function bucketLevelServerSideEncryption(bucket, headers, log, cb) { * @param {function} cb - callback to bucketPut * @return {undefined} */ -function createBucket(authInfo, bucketName, headers, - locationConstraint, log, cb) { +function createBucket(authInfo, bucketName, headers, locationConstraint, log, cb) { // Prefer using undefined instead of null for unused properties so they are not present in metadata payload log.trace('Creating bucket'); assert.strictEqual(typeof bucketName, 'string'); const canonicalID = authInfo.getCanonicalID(); - const ownerDisplayName = - authInfo.getAccountDisplayName(); + const ownerDisplayName = authInfo.getAccountDisplayName(); const creationDate = new Date().toJSON(); const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true' || undefined; const headerObjectLock = headers['x-amz-bucket-object-lock-enabled']; - const objectLockEnabled - = headerObjectLock && headerObjectLock.toLowerCase() === 'true'; - const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName, - creationDate, BucketInfo.currentModelVersion(), undefined, undefined, undefined, undefined, - undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, isNFSEnabled, - undefined, undefined, objectLockEnabled); + const objectLockEnabled = headerObjectLock && headerObjectLock.toLowerCase() === 'true'; + const bucket = new BucketInfo( + bucketName, + canonicalID, + ownerDisplayName, + creationDate, + BucketInfo.currentModelVersion(), + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + isNFSEnabled, + undefined, + undefined, + objectLockEnabled, + ); let locationConstraintVal = null; if (locationConstraint) { - const [locationConstraintStr, ingestion] = - locationConstraint.split(zenkoSeparator); + const [locationConstraintStr, ingestion] = locationConstraint.split(zenkoSeparator); if (locationConstraintStr) { locationConstraintVal = locationConstraintStr; bucket.setLocationConstraint(locationConstraintStr); @@ -210,88 +243,91 @@ function createBucket(authInfo, bucketName, headers, acl: bucket.acl, log, }; - async.parallel({ - prepareNewBucketMD: function prepareNewBucketMD(callback) { - acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => { - if (err) { - log.debug('error parsing acl from headers', { - error: err, - }); - return callback(err); - } - bucket.setFullAcl(parsedACL); - return callback(null, bucket); - }); - }, - getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) { - metadata.getBucket(bucketName, log, (err, data) => { - // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver - if (err && err.NoSuchBucket) { - return callback(null, 'NoBucketYet'); - } - if (err) { - return callback(err); - } - return callback(null, data); - }); + async.parallel( + { + prepareNewBucketMD: function prepareNewBucketMD(callback) { + acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => { + if (err) { + log.debug('error parsing acl from headers', { + error: err, + }); + return callback(err); + } + bucket.setFullAcl(parsedACL); + return callback(null, bucket); + }); + }, + getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) { + metadata.getBucket(bucketName, log, (err, data) => { + // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver + if (err && err.NoSuchBucket) { + return callback(null, 'NoBucketYet'); + } + if (err) { + return callback(err); + } + return callback(null, data); + }); + }, }, - }, - // Function to run upon finishing both parallel requests - (err, results) => { - if (err) { - return cb(err); - } - const existingBucketMD = results.getAnyExistingBucketInfo; - if (existingBucketMD instanceof BucketInfo && - existingBucketMD.getOwner() !== canonicalID && - !isServiceAccount(canonicalID)) { - // return existingBucketMD to collect cors headers - return cb(errors.BucketAlreadyExists, existingBucketMD); - } - const newBucketMD = results.prepareNewBucketMD; - if (existingBucketMD === 'NoBucketYet') { - const bucketSseConfig = parseBucketEncryptionHeaders(headers); + // Function to run upon finishing both parallel requests + (err, results) => { + if (err) { + return cb(err); + } + const existingBucketMD = results.getAnyExistingBucketInfo; + if ( + existingBucketMD instanceof BucketInfo && + existingBucketMD.getOwner() !== canonicalID && + !isServiceAccount(canonicalID) + ) { + // return existingBucketMD to collect cors headers + return cb(errors.BucketAlreadyExists, existingBucketMD); + } + const newBucketMD = results.prepareNewBucketMD; + if (existingBucketMD === 'NoBucketYet') { + const bucketSseConfig = parseBucketEncryptionHeaders(headers); - // Apply global SSE configuration when global encryption is enabled - // and no SSE settings were specified during bucket creation. - // Bucket-specific SSE headers override the default encryption. - const sseConfig = config.globalEncryptionEnabled && !bucketSseConfig.algorithm - ? { - algorithm: 'AES256', - mandatory: true, - } : bucketSseConfig; + // Apply global SSE configuration when global encryption is enabled + // and no SSE settings were specified during bucket creation. + // Bucket-specific SSE headers override the default encryption. + const sseConfig = + config.globalEncryptionEnabled && !bucketSseConfig.algorithm + ? { + algorithm: 'AES256', + mandatory: true, + } + : bucketSseConfig; - return bucketLevelServerSideEncryption( - bucket, sseConfig, log, - (err, sseInfo) => { + return bucketLevelServerSideEncryption(bucket, sseConfig, log, (err, sseInfo) => { if (err) { return cb(err); } newBucketMD.setServerSideEncryption(sseInfo); - log.trace( - 'new bucket without flags; adding transient label'); + log.trace('new bucket without flags; adding transient label'); newBucketMD.addTransientFlag(); - return freshStartCreateBucket(newBucketMD, canonicalID, - log, cb); + return freshStartCreateBucket(newBucketMD, canonicalID, log, cb); }); - } - if (existingBucketMD.hasTransientFlag() || - existingBucketMD.hasDeletedFlag()) { - log.trace('bucket has transient flag or deleted flag. cleaning up'); - return cleanUpBucket(newBucketMD, canonicalID, log, cb); - } - // If bucket already exists in non-transient and non-deleted - // state and owned by requester, then return BucketAlreadyOwnedByYou - // error unless old AWS behavior (us-east-1) - // Existing locationConstraint must have legacyAwsBehavior === true - // New locationConstraint should have legacyAwsBehavior === true - if (isLegacyAWSBehavior(locationConstraintVal) && - isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) { - log.trace('returning 200 instead of 409 to mirror us-east-1'); - return cb(null, existingBucketMD); - } - return cb(errors.BucketAlreadyOwnedByYou, existingBucketMD); - }); + } + if (existingBucketMD.hasTransientFlag() || existingBucketMD.hasDeletedFlag()) { + log.trace('bucket has transient flag or deleted flag. cleaning up'); + return cleanUpBucket(newBucketMD, canonicalID, log, cb); + } + // If bucket already exists in non-transient and non-deleted + // state and owned by requester, then return BucketAlreadyOwnedByYou + // error unless old AWS behavior (us-east-1) + // Existing locationConstraint must have legacyAwsBehavior === true + // New locationConstraint should have legacyAwsBehavior === true + if ( + isLegacyAWSBehavior(locationConstraintVal) && + isLegacyAWSBehavior(existingBucketMD.getLocationConstraint()) + ) { + log.trace('returning 200 instead of 409 to mirror us-east-1'); + return cb(null, existingBucketMD); + } + return cb(errors.BucketAlreadyOwnedByYou, existingBucketMD); + }, + ); } module.exports = { diff --git a/lib/api/bucketUpdateQuota.js b/lib/api/bucketUpdateQuota.js index e1d62c7e0c..17fbc90842 100644 --- a/lib/api/bucketUpdateQuota.js +++ b/lib/api/bucketUpdateQuota.js @@ -1,4 +1,4 @@ -const { waterfall } = require('async'); +const { promisify } = require('util'); const { errorInstances } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); @@ -6,48 +6,75 @@ const metadata = require('../metadata/wrapper'); const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); const { parseString } = require('xml2js'); +const { config } = require('../Config'); -function validateBucketQuotaProperty(requestBody, next) { +function validateBucketQuotaProperty(requestBody) { let quota = requestBody.quota; if (quota === undefined) { quota = requestBody.QuotaConfiguration?.Quota; } const quotaValue = parseInt(quota, 10); if (Number.isNaN(quotaValue)) { - return next(errorInstances.InvalidArgument.customizeDescription('Quota Value should be a number')); + throw errorInstances.InvalidArgument.customizeDescription('Quota Value should be a number'); } if (quotaValue <= 0) { - return next(errorInstances.InvalidArgument.customizeDescription('Quota value must be a positive number')); + throw errorInstances.InvalidArgument.customizeDescription('Quota value must be a positive number'); } - return next(null, quotaValue); + return quotaValue; } -function parseRequestBody(requestBody, contentType, next) { - switch (contentType) { - case 'application/xml': - return parseString(requestBody, { explicitArray: false }, (xmlError, xmlData) => { - if (xmlError) { - return next(errorInstances.InvalidArgument.customizeDescription('Invalid XML format')); - } - return next(null, xmlData); - }); - case 'application/json': - default: - try { - const jsonData = JSON.parse(requestBody); - if (typeof jsonData !== 'object') { - throw new Error('Invalid JSON'); - } - return next(null, jsonData); - } catch { - return next(errorInstances.InvalidArgument.customizeDescription('Request body must be a JSON object')); - } +async function parseRequestBody(requestBody, contentType) { + if (contentType === 'application/xml') { + try { + return await promisify(parseString)(requestBody, { explicitArray: false }); + } catch { + throw errorInstances.InvalidArgument.customizeDescription('Invalid XML format'); + } + } + try { + const jsonData = JSON.parse(requestBody); + if (typeof jsonData !== 'object') { + throw new Error('Invalid JSON'); + } + return jsonData; + } catch { + throw errorInstances.InvalidArgument.customizeDescription('Request body must be a JSON object'); } } -function bucketUpdateQuota(authInfo, request, log, callback) { - log.debug('processing request', { method: 'bucketUpdateQuota' }); +async function seedEmptyBucketCapacity(bucket, log) { + if (!config.isQuotaEnabled()) { + return; + } + const bucketName = bucket.getName(); + try { + const list = await promisify(metadata.listObject.bind(metadata))( + bucketName, + { + maxKeys: 1, + listingType: 'DelimiterVersions', + }, + log, + ); + const isEmpty = + (list.Versions ? list.Versions.length : 0) + (list.DeleteMarkers ? list.DeleteMarkers.length : 0) === 0; + if (!isEmpty) { + return; + } + await promisify(metadata.initializeBucketCapacity.bind(metadata))(bucketName, bucket.getCreationDate(), log); + } catch (err) { + log.error('error seeding bucket quota capacity metric', { error: err }); + } +} +async function bucketUpdateQuota(authInfo, request, log, callback) { + if (callback) { + return bucketUpdateQuota(authInfo, request, log) + .then(corsHeaders => callback(null, corsHeaders)) + .catch(err => callback(err, err.code, err.additionalResHeaders)); + } + + log.debug('processing request', { method: 'bucketUpdateQuota' }); const { bucketName } = request; const metadataValParams = { authInfo, @@ -55,41 +82,33 @@ function bucketUpdateQuota(authInfo, request, log, callback) { requestType: request.apiMethods || 'bucketUpdateQuota', request, }; - let bucket = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - bucket = b; - return next(err, bucket); - }), - (bucket, next) => parseRequestBody(request.post, request.headers['content-type'], (err, requestBody) => - next(err, bucket, requestBody)), - (bucket, requestBody, next) => validateBucketQuotaProperty(requestBody, (err, quotaValue) => - next(err, bucket, quotaValue)), - (bucket, quotaValue, next) => { - bucket.setQuota(quotaValue); - return metadata.updateBucket(bucket.getName(), bucket, log, next); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketUpdateQuota' - }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'updateBucketQuota'); - return callback(err, err.code, corsHeaders); - } - monitoring.promMetrics( - 'PUT', bucketName, '200', 'updateBucketQuota'); - pushMetric('updateBucketQuota', log, { - authInfo, - bucket: bucketName, + + let bucket; + try { + bucket = await promisify(standardMetadataValidateBucket)(metadataValParams, request.actionImplicitDenies, log); + const requestBody = await parseRequestBody(request.post, request.headers['content-type']); + const quotaValue = validateBucketQuotaProperty(requestBody); + bucket.setQuota(quotaValue); + await promisify(metadata.updateBucket.bind(metadata))(bucket.getName(), bucket, log); + await seedEmptyBucketCapacity(bucket, log); + } catch (err) { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + log.debug('error processing request', { + error: err, + method: 'bucketUpdateQuota', }); - return callback(null, corsHeaders); + monitoring.promMetrics('PUT', bucketName, err.code, 'updateBucketQuota'); + err.additionalResHeaders = err.additionalResHeaders || corsHeaders; + throw err; + } + + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + monitoring.promMetrics('PUT', bucketName, '200', 'updateBucketQuota'); + pushMetric('updateBucketQuota', log, { + authInfo, + bucket: bucketName, }); + return corsHeaders; } module.exports = bucketUpdateQuota; diff --git a/package.json b/package.json index c0e3dfaf94..09bb9afae5 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@opentelemetry/instrumentation-ioredis": "~0.64.0", "@opentelemetry/instrumentation-mongodb": "~0.69.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/arsenal#8.5.6", + "arsenal": "git+https://github.com/scality/arsenal#43a1d0577dffd6e77a712ab813abaff57ace43ae", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/tests/unit/api/bucketPut.js b/tests/unit/api/bucketPut.js index 529f2947cb..7c778deab5 100644 --- a/tests/unit/api/bucketPut.js +++ b/tests/unit/api/bucketPut.js @@ -12,10 +12,7 @@ const metadata = require('../metadataswitch'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const originalLCs = Object.assign({}, config.locationConstraints); -const { - LOCATION_NAME_DMF, - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_DMF, LOCATION_NAME_CRR } = require('../../constants'); const log = new DummyRequestLogger(); const accessKey = 'accessKey1'; @@ -100,27 +97,27 @@ describe('checkLocationConstraint function', () => { config.backends.data = initialConfigData; }); testChecks.forEach(testCheck => { - const returnText = testCheck.isError ? `${testCheck.expectedError} error` - : 'the appropriate location constraint'; - it(`with data backend: "${testCheck.data}", ` + - `location: "${testCheck.locationSent}",` + - ` and host: "${testCheck.parsedHost}", should return ${returnText} `, - done => { - config.backends.data = testCheck.data; - request.parsedHost = testCheck.parsedHost; - const checkLocation = checkLocationConstraint(request, - testCheck.locationSent, log); - if (testCheck.isError) { - assert.notEqual(checkLocation.error, null, - 'Expected failure but got success'); - assert(checkLocation.error.is[testCheck.expectedError]); - } else { - assert.ifError(checkLocation.error); - assert.strictEqual(checkLocation.locationConstraint, - testCheck.locationReturn); - } - done(); - }); + const returnText = testCheck.isError + ? `${testCheck.expectedError} error` + : 'the appropriate location constraint'; + it( + `with data backend: "${testCheck.data}", ` + + `location: "${testCheck.locationSent}",` + + ` and host: "${testCheck.parsedHost}", should return ${returnText} `, + done => { + config.backends.data = testCheck.data; + request.parsedHost = testCheck.parsedHost; + const checkLocation = checkLocationConstraint(request, testCheck.locationSent, log); + if (testCheck.isError) { + assert.notEqual(checkLocation.error, null, 'Expected failure but got success'); + assert(checkLocation.error.is[testCheck.expectedError]); + } else { + assert.ifError(checkLocation.error); + assert.strictEqual(checkLocation.locationConstraint, testCheck.locationReturn); + } + done(); + }, + ); }); }); @@ -132,11 +129,10 @@ describe('bucketPut API', () => { it('should return an error if bucket already exists', done => { const otherAuthInfo = makeAuthInfo('accessKey2'); bucketPut(authInfo, testRequest, log, () => { - bucketPut(otherAuthInfo, testRequest, - log, err => { - assert.strictEqual(err.is.BucketAlreadyExists, true); - done(); - }); + bucketPut(otherAuthInfo, testRequest, log, err => { + assert.strictEqual(err.is.BucketAlreadyExists, true); + done(); + }); }); }); @@ -149,12 +145,10 @@ describe('bucketPut API', () => { assert.strictEqual(md.getName(), bucketName); assert.strictEqual(md.getOwner(), canonicalID); const prefix = `${canonicalID}${splitter}`; - metadata.listObject(usersBucket, { prefix }, - log, (err, listResponse) => { - assert.strictEqual(listResponse.Contents[0].key, - `${canonicalID}${splitter}${bucketName}`); - done(); - }); + metadata.listObject(usersBucket, { prefix }, log, (err, listResponse) => { + assert.strictEqual(listResponse.Contents[0].key, `${canonicalID}${splitter}${bucketName}`); + done(); + }); }); }); }); @@ -165,7 +159,7 @@ describe('bucketPut API', () => { url: '/', post: '', headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': `${status}`, }, }); @@ -202,16 +196,13 @@ describe('bucketPut API', () => { }); }); - it('should return an error if ACL set in header ' + - 'with an invalid group URI', done => { + it('should return an error if ACL set in header ' + 'with an invalid group URI', done => { const testRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': - 'uri="http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP"', + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP"', }, url: '/', post: '', @@ -225,13 +216,12 @@ describe('bucketPut API', () => { }); }); - it('should return an error if ACL set in header ' + - 'with an invalid canned ACL', done => { + it('should return an error if ACL set in header ' + 'with an invalid canned ACL', done => { const testRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'not-valid-option', }, url: '/', @@ -246,15 +236,13 @@ describe('bucketPut API', () => { }); }); - it('should return an error if ACL set in header ' + - 'with an invalid email address', done => { + it('should return an error if ACL set in header ' + 'with an invalid email address', done => { const testRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-read': - 'emailaddress="fake@faking.com"', + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-read': 'emailaddress="fake@faking.com"', }, url: '/', post: '', @@ -268,15 +256,13 @@ describe('bucketPut API', () => { }); }); - it('should set a canned ACL while creating bucket' + - ' if option set out in header', done => { + it('should set a canned ACL while creating bucket' + ' if option set out in header', done => { const testRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-acl': - 'public-read', + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-acl': 'public-read', }, url: '/', post: '', @@ -291,45 +277,33 @@ describe('bucketPut API', () => { }); }); - it('should set specific ACL grants while creating bucket' + - ' if options set out in header', done => { + it('should set specific ACL grants while creating bucket' + ' if options set out in header', done => { const testRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: '/', post: '', }; - const canonicalIDforSample1 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; - const canonicalIDforSample2 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; + const canonicalIDforSample1 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; + const canonicalIDforSample2 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; bucketPut(authInfo, testRequest, log, err => { assert.strictEqual(err, null, 'Error creating bucket'); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getAcl().READ[0], constants.logId); assert.strictEqual(md.getAcl().WRITE[0], constants.publicId); - assert(md.getAcl() - .FULL_CONTROL.indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl() - .FULL_CONTROL.indexOf(canonicalIDforSample2) > -1); - assert(md.getAcl() - .READ_ACP.indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl() - .WRITE_ACP.indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().READ_ACP.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().WRITE_ACP.indexOf(canonicalIDforSample2) > -1); done(); }); }); @@ -361,8 +335,7 @@ describe('bucketPut API', () => { assert.deepStrictEqual(err, null); metadata.getBucket(bucketName, log, (err, bucketInfo) => { assert.deepStrictEqual(err, null); - assert.deepStrictEqual(newLocation, - bucketInfo.getLocationConstraint()); + assert.deepStrictEqual(newLocation, bucketInfo.getLocationConstraint()); done(); }); }); @@ -427,21 +400,25 @@ describe('bucketPut API', () => { const newLCs = Object.assign({}, config.locationConstraints, newLC); const req = Object.assign({}, testRequest, { bucketName, - post: '' + + post: + '' + '' + - `${newLCKey}` + + `${newLCKey}` + '', }); afterEach(() => config.setLocationConstraints(originalLCs)); - it('should return error if location constraint config is not updated', - done => bucketPut(authInfo, req, log, err => { + it('should return error if location constraint config is not updated', done => + bucketPut(authInfo, req, log, err => { assert.strictEqual(err.is.InvalidLocationConstraint, true); - assert.strictEqual(err.description, 'value of the location you are ' + - `attempting to set - ${newLCKey} - is not listed in the ` + - 'locationConstraint config'); + assert.strictEqual( + err.description, + 'value of the location you are ' + + `attempting to set - ${newLCKey} - is not listed in the ` + + 'locationConstraint config', + ); done(); })); @@ -472,12 +449,7 @@ describe('bucketPut API', () => { { description: 'many allowed auth', error: undefined, - results: [ - { isAllowed: true }, - { isAllowed: true }, - { isAllowed: true }, - { isAllowed: true }, - ], + results: [{ isAllowed: true }, { isAllowed: true }, { isAllowed: true }, { isAllowed: true }], calledWith: [null, constraint], }, { @@ -511,20 +483,17 @@ describe('bucketPut API', () => { { description: 'one not allowed auth of many', error: undefined, - results: [ - { isAllowed: true }, - { isAllowed: true }, - { isAllowed: false }, - { isAllowed: true }, - ], + results: [{ isAllowed: true }, { isAllowed: true }, { isAllowed: false }, { isAllowed: true }], calledWith: [errors.AccessDenied], }, - ].forEach(tc => it(tc.description, () => { - const cb = sinon.fake(); - const handler = _handleAuthResults(constraint, log, cb); - handler(tc.error, tc.results); - assert.deepStrictEqual(cb.getCalls()[0].args, tc.calledWith); - })); + ].forEach(tc => + it(tc.description, () => { + const cb = sinon.fake(); + const handler = _handleAuthResults(constraint, log, cb); + handler(tc.error, tc.results); + assert.deepStrictEqual(cb.getCalls()[0].args, tc.calledWith); + }), + ); }); }); @@ -544,7 +513,7 @@ describe('bucketPut API with bucket-level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'AES256', }, }; @@ -568,7 +537,7 @@ describe('bucketPut API with bucket-level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'aws:kms', }, }; @@ -593,7 +562,7 @@ describe('bucketPut API with bucket-level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'aws:kms', 'x-amz-scal-server-side-encryption-aws-kms-key-id': keyId, }, @@ -622,7 +591,7 @@ describe('bucketPut API with bucket-level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'AES256', 'x-amz-scal-server-side-encryption-aws-kms-key-id': keyId, }, @@ -652,7 +621,7 @@ describe('bucketPut API with account level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'AES256', }, }; @@ -677,7 +646,7 @@ describe('bucketPut API with account level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'aws:kms', }, }; @@ -703,7 +672,7 @@ describe('bucketPut API with account level encryption', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'aws:kms', 'x-amz-scal-server-side-encryption-aws-kms-key-id': keyId, }, @@ -739,7 +708,7 @@ describe('bucketPut API with failed encryption service', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'AES256', }, }; @@ -753,8 +722,9 @@ describe('bucketPut API with failed encryption service', () => { describe('bucketPut API with failed vault service', () => { beforeEach(() => { sinon.stub(inMemory, 'supportsDefaultKeyPerAccount').value(true); - sinon.stub(vault, 'getOrCreateEncryptionKeyId').callsFake((accountCanonicalId, log, cb) => - cb(errors.ServiceFailure)); + sinon + .stub(vault, 'getOrCreateEncryptionKeyId') + .callsFake((accountCanonicalId, log, cb) => cb(errors.ServiceFailure)); }); afterEach(() => { @@ -766,7 +736,7 @@ describe('bucketPut API with failed vault service', () => { const testRequestWithEncryption = { ...testRequest, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-scal-server-side-encryption': 'AES256', }, }; @@ -934,3 +904,50 @@ describe('bucketPut API with SSE Configurations', () => { }); }); }); + +describe('bucketPut API quota metric seeding', () => { + afterEach(() => { + sinon.restore(); + cleanup(); + }); + + it('should seed a zero-value quota metric on bucket creation when quotas are enabled', done => { + sinon.stub(config, 'isQuotaEnabled').returns(true); + const initStub = sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => cb(null)); + bucketPut(authInfo, testRequest, log, err => { + assert.ifError(err); + assert(initStub.calledOnce, 'expected initializeBucketCapacity to be called once'); + assert.strictEqual(initStub.firstCall.args[0], bucketName); + done(); + }); + }); + + it('should still create the bucket if seeding the quota metric fails', done => { + sinon.stub(config, 'isQuotaEnabled').returns(true); + sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => cb(errors.InternalError)); + bucketPut(authInfo, testRequest, log, err => { + assert.ifError(err); + return metadata.getBucket(bucketName, log, (getErr, md) => { + assert.ifError(getErr); + assert.strictEqual(md.getName(), bucketName); + done(); + }); + }); + }); + + it('should not seed a quota metric when quotas are disabled', done => { + sinon.stub(config, 'isQuotaEnabled').returns(false); + const initStub = sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => cb(null)); + bucketPut(authInfo, testRequest, log, err => { + assert.ifError(err); + assert(initStub.notCalled, 'expected no seeding when quotas are disabled'); + done(); + }); + }); +}); diff --git a/tests/unit/api/bucketUpdateQuota.js b/tests/unit/api/bucketUpdateQuota.js new file mode 100644 index 0000000000..1f4fa0fdc7 --- /dev/null +++ b/tests/unit/api/bucketUpdateQuota.js @@ -0,0 +1,122 @@ +const assert = require('assert'); +const sinon = require('sinon'); +const { errors } = require('arsenal'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const bucketUpdateQuota = require('../../../lib/api/bucketUpdateQuota'); +const objectPut = require('../../../lib/api/objectPut'); +const { config } = require('../../../lib/Config'); +const DummyRequest = require('../DummyRequest'); +const metadata = require('../metadataswitch'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); + +const log = new DummyRequestLogger(); +const authInfo = makeAuthInfo('accessKey1'); +const namespace = 'default'; +const bucketName = 'quotabucketname'; + +const bucketPutRequest = { + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + actionImplicitDenies: false, +}; + +const updateQuotaRequest = { + bucketName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-type': 'application/json', + }, + post: '{"quota": 1000}', + actionImplicitDenies: false, +}; + +function putObject(objectName, cb) { + const request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + Buffer.from('body content', 'utf8'), + ); + return objectPut(authInfo, request, undefined, log, cb); +} + +describe('bucketUpdateQuota API quota metric seeding', () => { + beforeEach(() => { + sinon.stub(config, 'isQuotaEnabled').returns(true); + }); + + afterEach(() => { + sinon.restore(); + cleanup(); + }); + + it('should seed a zero-value metric when a quota is enabled on an empty bucket', done => { + const initStub = sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => process.nextTick(() => cb(null))); + bucketPut(authInfo, bucketPutRequest, log, err => { + assert.ifError(err); + initStub.resetHistory(); + return bucketUpdateQuota(authInfo, updateQuotaRequest, log, err => { + assert.ifError(err); + assert(initStub.calledOnce, 'expected seeding for an empty bucket'); + assert.strictEqual(initStub.firstCall.args[0], bucketName); + done(); + }); + }); + }); + + it('should not seed a metric when the bucket is not empty', done => { + const initStub = sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => process.nextTick(() => cb(null))); + bucketPut(authInfo, bucketPutRequest, log, err => { + assert.ifError(err); + return putObject('someobject', err => { + assert.ifError(err); + initStub.resetHistory(); + return bucketUpdateQuota(authInfo, updateQuotaRequest, log, err => { + assert.ifError(err); + assert(initStub.notCalled, 'expected no seeding for a non-empty bucket'); + done(); + }); + }); + }); + }); + + it('should still update the quota if seeding the metric fails', done => { + bucketPut(authInfo, bucketPutRequest, log, err => { + assert.ifError(err); + sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => process.nextTick(() => cb(errors.InternalError))); + return bucketUpdateQuota(authInfo, updateQuotaRequest, log, err => { + assert.ifError(err); + done(); + }); + }); + }); + + it('should not seed a metric when quotas are disabled', done => { + config.isQuotaEnabled.restore(); + sinon.stub(config, 'isQuotaEnabled').returns(false); + const initStub = sinon + .stub(metadata, 'initializeBucketCapacity') + .callsFake((name, creationDate, l, cb) => process.nextTick(() => cb(null))); + bucketPut(authInfo, bucketPutRequest, log, err => { + assert.ifError(err); + initStub.resetHistory(); + return bucketUpdateQuota(authInfo, updateQuotaRequest, log, err => { + assert.ifError(err); + assert(initStub.notCalled, 'expected no seeding when quotas are disabled'); + done(); + }); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index e465d55da2..64d8789102 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6595,9 +6595,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#8.5.6": - version "8.5.6" - resolved "git+https://github.com/scality/arsenal#0db557930c7d13204167188a7503e6e00d154df5" +"arsenal@git+https://github.com/scality/arsenal#43a1d0577dffd6e77a712ab813abaff57ace43ae": + version "8.5.7" + resolved "git+https://github.com/scality/arsenal#43a1d0577dffd6e77a712ab813abaff57ace43ae" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0"