diff --git a/lib/api/apiUtils/quotas/quotaUtils.js b/lib/api/apiUtils/quotas/quotaUtils.js index 6c967bec94..78601eb45f 100644 --- a/lib/api/apiUtils/quotas/quotaUtils.js +++ b/lib/api/apiUtils/quotas/quotaUtils.js @@ -1,11 +1,7 @@ const async = require('async'); const { errors } = require('arsenal'); const monitoring = require('../../../utilities/monitoringHandler'); -const { - actionNeedQuotaCheckCopy, - actionNeedQuotaCheck, - actionWithDataDeletion, -} = require('arsenal').policies; +const { actionNeedQuotaCheckCopy, actionNeedQuotaCheck, actionWithDataDeletion } = require('arsenal').policies; const { config } = require('../../../Config'); const QuotaService = require('../../../utilization/instance'); @@ -44,7 +40,7 @@ function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, // but it also replaces the target, which decreases storage bytes -= getHotContentLength(destObjMD); } - } else if (!bucket.isVersioningEnabled() || bucket.isVersioningEnabled() && versionId) { + } else if (!bucket.isVersioningEnabled() || (bucket.isVersioningEnabled() && versionId)) { // object is being deleted (non versioned) or hard-deleted (versioned, as indicated by // the `versionId` field) bytes = -getHotContentLength(objMD); @@ -68,8 +64,7 @@ function processBytesToWrite(apiMethod, bucket, versionId, contentLength, objMD, * @returns {boolean} Returns true if the metric is stale, false otherwise. */ function isMetricStale(metric, resourceType, resourceName, action, inflight, log) { - if (metric.date && Date.now() - new Date(metric.date).getTime() > - QuotaService.maxStaleness) { + if (metric.date && Date.now() - new Date(metric.date).getTime() > QuotaService.maxStaleness) { log.warn('Stale metrics from the quota service, allowing the request', { resourceType, resourceName, @@ -110,70 +105,89 @@ function _evaluateQuotas( let bucketQuotaExceeded = false; let accountQuotaExceeded = false; const creationDate = new Date(bucket.getCreationDate()).getTime(); - return async.parallel({ - bucketQuota: parallelDone => { - if (bucketQuota > 0) { - return QuotaService.getUtilizationMetrics('bucket', - `${bucket.getName()}_${creationDate}`, null, { - action, - inflight, - }, (err, bucketMetrics) => { - if (err || inflight < 0) { - return parallelDone(err); - } - if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) && - BigInt(bucketMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > bucketQuota) { - log.debug('Bucket quota exceeded', { - bucket: bucket.getName(), + return async.parallel( + { + bucketQuota: parallelDone => { + if (bucketQuota > 0) { + return QuotaService.getUtilizationMetrics( + 'bucket', + `${bucket.getName()}_${creationDate}`, + null, + { action, inflight, - quota: bucketQuota, - bytesTotal: bucketMetrics.bytesTotal, - }); - bucketQuotaExceeded = true; - } - return parallelDone(); - }); - } - return parallelDone(); - }, - accountQuota: parallelDone => { - if (accountQuota > 0 && account?.account) { - return QuotaService.getUtilizationMetrics('account', - account.account, null, { - action, - inflight, - }, (err, accountMetrics) => { - if (err || inflight < 0) { - return parallelDone(err); - } - // Metrics are served as BigInt strings - if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) && - BigInt(accountMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > accountQuota) { - log.debug('Account quota exceeded', { - accountId: account.account, + }, + log, + (err, bucketMetrics) => { + if (err || inflight < 0) { + return parallelDone(err); + } + if ( + !isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) && + BigInt(bucketMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > bucketQuota + ) { + log.debug('Bucket quota exceeded', { + bucket: bucket.getName(), + action, + inflight, + quota: bucketQuota, + bytesTotal: bucketMetrics.bytesTotal, + }); + bucketQuotaExceeded = true; + } + return parallelDone(); + }, + ); + } + return parallelDone(); + }, + accountQuota: parallelDone => { + if (accountQuota > 0 && account?.account) { + return QuotaService.getUtilizationMetrics( + 'account', + account.account, + null, + { action, inflight, - quota: accountQuota, - bytesTotal: accountMetrics.bytesTotal, - }); - accountQuotaExceeded = true; - } - return parallelDone(); + }, + log, + (err, accountMetrics) => { + if (err || inflight < 0) { + return parallelDone(err); + } + // Metrics are served as BigInt strings + if ( + !isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) && + BigInt(accountMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) > accountQuota + ) { + log.debug('Account quota exceeded', { + accountId: account.account, + action, + inflight, + quota: accountQuota, + bytesTotal: accountMetrics.bytesTotal, + }); + accountQuotaExceeded = true; + } + return parallelDone(); + }, + ); + } + return parallelDone(); + }, + }, + err => { + if (err) { + log.warn('Error evaluating quotas', { + error: err.name, + description: err.message, + isInflightDeletion: inflight < 0, }); } - return parallelDone(); + return callback(err, bucketQuotaExceeded, accountQuotaExceeded); }, - }, err => { - if (err) { - log.warn('Error evaluating quotas', { - error: err.name, - description: err.message, - isInflightDeletion: inflight < 0, - }); - } - return callback(err, bucketQuotaExceeded, accountQuotaExceeded); - }); + ); } /** @@ -186,11 +200,13 @@ function _evaluateQuotas( * @returns {undefined} - Returns nothing. */ function monitorQuotaEvaluationDuration(apiMethod, type, code, duration) { - monitoring.quotaEvaluationDuration.labels({ - action: apiMethod, - type, - code, - }).observe(duration / 1e9); + monitoring.quotaEvaluationDuration + .labels({ + action: apiMethod, + type, + code, + }) + .observe(duration / 1e9); } /** @@ -248,76 +264,103 @@ function validateQuotas(request, bucket, account, apiNames, apiMethod, inflight, inflight = 0; } - return async.forEach(apiNames, (apiName, done) => { - // Object copy operations first check the target object, - // meaning the source object, containing the current bytes, - // is checked second. This logic handles these APIs calls by - // ensuring the bytes are positives (i.e., not an object - // replacement). - if (actionNeedQuotaCheckCopy(apiName, apiMethod)) { - // eslint-disable-next-line no-param-reassign - inflight = Math.abs(inflight); - } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) { - return done(); - } - // When inflights are disabled, the sum of the current utilization metrics - // and the current bytes are compared with the quota. The current bytes - // are not sent to the utilization service. When inflights are enabled, - // the sum of the current utilization metrics only are compared with the - // quota. They include the current inflight bytes sent in the request. - let _inflights = shouldSendInflights ? inflight : undefined; - const inflightForCheck = shouldSendInflights ? 0 : inflight; - return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights, - inflightForCheck, apiName, log, - (err, _bucketQuotaExceeded, _accountQuotaExceeded) => { - if (err) { - return done(err); - } + return async.forEach( + apiNames, + (apiName, done) => { + // Object copy operations first check the target object, + // meaning the source object, containing the current bytes, + // is checked second. This logic handles these APIs calls by + // ensuring the bytes are positives (i.e., not an object + // replacement). + if (actionNeedQuotaCheckCopy(apiName, apiMethod)) { + // eslint-disable-next-line no-param-reassign + inflight = Math.abs(inflight); + } else if (!actionNeedQuotaCheck[apiName] && !actionWithDataDeletion[apiName]) { + return done(); + } + // When inflights are disabled, the sum of the current utilization metrics + // and the current bytes are compared with the quota. The current bytes + // are not sent to the utilization service. When inflights are enabled, + // the sum of the current utilization metrics only are compared with the + // quota. They include the current inflight bytes sent in the request. + let _inflights = shouldSendInflights ? inflight : undefined; + const inflightForCheck = shouldSendInflights ? 0 : inflight; + return _evaluateQuotas( + bucketQuota, + accountQuota, + bucket, + account, + _inflights, + inflightForCheck, + apiName, + log, + (err, _bucketQuotaExceeded, _accountQuotaExceeded) => { + if (err) { + return done(err); + } - bucketQuotaExceeded = _bucketQuotaExceeded; - accountQuotaExceeded = _accountQuotaExceeded; + bucketQuotaExceeded = _bucketQuotaExceeded; + accountQuotaExceeded = _accountQuotaExceeded; - // Inflights are inverted: in case of cleanup, we just re-issue - // the same API call. - if (_inflights) { - _inflights = -_inflights; - } + // Inflights are inverted: in case of cleanup, we just re-issue + // the same API call. + if (_inflights) { + _inflights = -_inflights; + } - request.finalizerHooks.push((errorFromAPI, _done) => { - const code = (bucketQuotaExceeded || accountQuotaExceeded) ? 429 : 200; - const quotaCleanUpStartTime = process.hrtime.bigint(); - // Quotas are cleaned only in case of error in the API - async.waterfall([ - cb => { - if (errorFromAPI) { - return _evaluateQuotas(bucketQuota, accountQuota, bucket, account, _inflights, - null, apiName, log, cb); - } - return cb(); - }, - ], () => { - monitorQuotaEvaluationDuration(apiMethod, type, code, quotaEvaluationDuration + - Number(process.hrtime.bigint() - quotaCleanUpStartTime)); - return _done(); + request.finalizerHooks.push((errorFromAPI, _done) => { + const code = bucketQuotaExceeded || accountQuotaExceeded ? 429 : 200; + const quotaCleanUpStartTime = process.hrtime.bigint(); + // Quotas are cleaned only in case of error in the API + async.waterfall( + [ + cb => { + if (errorFromAPI) { + return _evaluateQuotas( + bucketQuota, + accountQuota, + bucket, + account, + _inflights, + null, + apiName, + log, + cb, + ); + } + return cb(); + }, + ], + () => { + monitorQuotaEvaluationDuration( + apiMethod, + type, + code, + quotaEvaluationDuration + Number(process.hrtime.bigint() - quotaCleanUpStartTime), + ); + return _done(); + }, + ); }); - }); - return done(); - }); - }, err => { - quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime); - if (err) { - log.warn('Error getting metrics from the quota service, allowing the request', { - error: err.name, - description: err.message, - }); - } - if (!actionWithDataDeletion[apiMethod] && - (bucketQuotaExceeded || accountQuotaExceeded)) { - return callback(errors.QuotaExceeded); - } - return callback(); - }); + return done(); + }, + ); + }, + err => { + quotaEvaluationDuration = Number(process.hrtime.bigint() - requestStartTime); + if (err) { + log.warn('Error getting metrics from the quota service, allowing the request', { + error: err.name, + description: err.message, + }); + } + if (!actionWithDataDeletion[apiMethod] && (bucketQuotaExceeded || accountQuotaExceeded)) { + return callback(errors.QuotaExceeded); + } + return callback(); + }, + ); } module.exports = { diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js index 3676669435..4f4e22618e 100644 --- a/lib/routes/veeam/utils.js +++ b/lib/routes/veeam/utils.js @@ -37,12 +37,15 @@ async function receiveData(request, log) { // Prevent memory overloads by limiting the size of the // received data. if (parsedContentLength > ContentLengthThreshold) { - throw errorInstances.InvalidInput - .customizeDescription(`maximum allowed content-length is ${ContentLengthThreshold} bytes`); + throw errorInstances.InvalidInput.customizeDescription( + `maximum allowed content-length is ${ContentLengthThreshold} bytes`, + ); } return await new Promise((resolve, reject) => { const settle = jsutil.once((err, result) => { - if (err) { return reject(err); } + if (err) { + return reject(err); + } return resolve(result); }); let totalLength = 0; @@ -51,8 +54,7 @@ async function receiveData(request, log) { write(chunk, _enc, cb) { totalLength += chunk.length; if (totalLength > parsedContentLength) { - log.error('data stream exceed announced size', - { parsedContentLength, overflow: totalLength }); + log.error('data stream exceed announced size', { parsedContentLength, overflow: totalLength }); return cb(errors.InternalError); } chunks.push(chunk); @@ -88,17 +90,18 @@ function buildHeadXML(xmlContent) { * @returns {object} - response headers */ function getResponseHeader(request, bucket, dataBuffer, lastModified, log) { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - const responseMetaHeaders = collectResponseHeaders({ - 'last-modified': lastModified || new Date().toISOString(), - 'content-md5': crypto - .createHash('md5') - .update(dataBuffer) - .digest('hex'), - 'content-length': dataBuffer.byteLength, - 'content-type': 'text/xml', - }, corsHeaders, null, false); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + const responseMetaHeaders = collectResponseHeaders( + { + 'last-modified': lastModified || new Date().toISOString(), + 'content-md5': crypto.createHash('md5').update(dataBuffer).digest('hex'), + 'content-length': dataBuffer.byteLength, + 'content-type': 'text/xml', + }, + corsHeaders, + null, + false, + ); responseMetaHeaders.versionId = 'null'; responseMetaHeaders['x-amz-id-2'] = log.getSerializedUids(); responseMetaHeaders['x-amz-request-id'] = log.getSerializedUids(); @@ -136,10 +139,10 @@ async function respondWithData(request, response, log, bucket, data, lastModifie try { response.setHeader(key, responseMetaHeaders[key]); } catch (e) { - log.debug('header can not be added ' + - 'to the response', { + log.debug('header can not be added ' + 'to the response', { header: responseMetaHeaders[key], - error: e.stack, method: 'routeVeeam/respondWithData' + error: e.stack, + method: 'routeVeeam/respondWithData', }); } } @@ -199,7 +202,7 @@ function getFileToBuild(request, data, inlineLastModified = false) { return { error: errors.NoSuchKey }; } - const modified = fileToBuild.LastModified || (new Date()).toISOString(); + const modified = fileToBuild.LastModified || new Date().toISOString(); const fieldName = _isSystemXML ? 'SystemInfo' : 'CapacityInfo'; if (inlineLastModified) { @@ -231,7 +234,7 @@ function getFileToBuild(request, data, inlineLastModified = false) { async function fetchCapacityMetrics(bucketMd, request, log) { const bucketKey = `${bucketMd._name}_${new Date(bucketMd._creationDate).getTime()}`; try { - return await getUtilizationMetrics('bucket', bucketKey, null, {}); + return await getUtilizationMetrics('bucket', bucketKey, null, {}, log); } catch (err) { const statusCode = err.response?.status || err.statusCode || err.code; if (statusCode === 404) { @@ -286,9 +289,11 @@ async function buildVeeamFileData(request, bucketMd, log) { } const modified = bucketMetrics.date; - if (bucketMetrics.bytesTotal !== undefined - && fileToBuild.value.CapacityInfo - && !fileToBuild.value.CapacityInfo.Used) { + if ( + bucketMetrics.bytesTotal !== undefined && + fileToBuild.value.CapacityInfo && + !fileToBuild.value.CapacityInfo.Used + ) { fileToBuild.value.CapacityInfo.Used = Number(bucketMetrics.bytesTotal); fileToBuild.value.CapacityInfo.Available = Number(fileToBuild.value.CapacityInfo.Capacity) - Number(bucketMetrics.bytesTotal); diff --git a/lib/utilization/scuba/wrapper.js b/lib/utilization/scuba/wrapper.js index 0cde306868..e5d8aeb2c6 100644 --- a/lib/utilization/scuba/wrapper.js +++ b/lib/utilization/scuba/wrapper.js @@ -27,28 +27,30 @@ class ScubaClientImpl extends ScubaClient { } _healthCheck() { - return this.healthCheck().then(data => { - if (data?.date) { - const date = new Date(data.date); - if (Date.now() - date.getTime() > this.maxStaleness) { - throw new Error('Data is stale, disabling quotas'); + return this.healthCheck() + .then(data => { + if (data?.date) { + const date = new Date(data.date); + if (Date.now() - date.getTime() > this.maxStaleness) { + throw new Error('Data is stale, disabling quotas'); + } } - } - if (!this.enabled) { - this._log.info('Scuba health check passed, enabling quotas'); - } - monitoring.utilizationServiceAvailable.set(1); - this.enabled = true; - }).catch(err => { - if (this.enabled) { - this._log.warn('Scuba health check failed, disabling quotas', { - err: err.name, - description: err.message, - }); - } - monitoring.utilizationServiceAvailable.set(0); - this.enabled = false; - }); + if (!this.enabled) { + this._log.info('Scuba health check passed, enabling quotas'); + } + monitoring.utilizationServiceAvailable.set(1); + this.enabled = true; + }) + .catch(err => { + if (this.enabled) { + this._log.warn('Scuba health check failed, disabling quotas', { + err: err.name, + description: err.message, + }); + } + monitoring.utilizationServiceAvailable.set(0); + this.enabled = false; + }); } periodicHealthCheck() { @@ -56,22 +58,38 @@ class ScubaClientImpl extends ScubaClient { clearInterval(this._healthCheckTimer); } this._healthCheck(); - this._healthCheckTimer = setInterval(async () => { - this._healthCheck(); - }, Number(process.env.SCUBA_HEALTHCHECK_FREQUENCY) - || externalBackendHealthCheckInterval); + this._healthCheckTimer = setInterval( + async () => { + this._healthCheck(); + }, + Number(process.env.SCUBA_HEALTHCHECK_FREQUENCY) || externalBackendHealthCheckInterval, + ); } - getUtilizationMetrics(metricsClass, resourceName, options, body, callback) { + async getUtilizationMetrics(metricsClass, resourceName, options, body, log, callback) { + if (typeof callback === 'function') { + return this.getUtilizationMetrics(metricsClass, resourceName, options, body, log) + .then(data => callback(null, data)) + .catch(callback); + } const requestStartTime = process.hrtime.bigint(); - return this._getLatestMetricsCallback(metricsClass, resourceName, options, body, (err, data) => { + const reqUids = log && typeof log.getSerializedUids === 'function' ? log.getSerializedUids() : undefined; + const mergedOptions = reqUids + ? { ...options, headers: { ...(options && options.headers), 'X-Scal-Request-Uids': reqUids } } + : options; + const getLatestMetrics = util.promisify((...args) => this._getLatestMetricsCallback(...args)); + let code = 200; + try { + return await getLatestMetrics(metricsClass, resourceName, mergedOptions, body); + } catch (err) { + code = err.statusCode || 500; + throw err; + } finally { const responseTimeInNs = Number(process.hrtime.bigint() - requestStartTime); - monitoring.utilizationMetricsRetrievalDuration.labels({ - code: err ? (err.statusCode || 500) : 200, - class: metricsClass, - }).observe(responseTimeInNs / 1e9); - return callback(err, data); - }); + monitoring.utilizationMetricsRetrievalDuration + .labels({ code, class: metricsClass }) + .observe(responseTimeInNs / 1e9); + } } } diff --git a/tests/unit/api/apiUtils/quotas/quotaUtils.js b/tests/unit/api/apiUtils/quotas/quotaUtils.js index 67b4750c43..53ff98e64c 100644 --- a/tests/unit/api/apiUtils/quotas/quotaUtils.js +++ b/tests/unit/api/apiUtils/quotas/quotaUtils.js @@ -75,15 +75,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectPut', inflight: 1, - } - ), true); + }), + true, + ); done(); }); }); @@ -102,15 +100,13 @@ describe('validateQuotas (buckets)', () => { assert.strictEqual(err.is.QuotaExceeded, true); assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); assert.strictEqual(request.finalizerHooks.length, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectPut', inflight: 1, - } - ), true); + }), + true, + ); done(); }); }); @@ -128,15 +124,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', 0, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: 0, - } - ), true); + }), + true, + ); done(); }); }); @@ -154,15 +148,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: -50, - } - ), true); + }), + true, + ); done(); }); }); @@ -180,15 +172,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDeleteVersion', -50, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: -50, - } - ), true); + }), + true, + ); done(); }); }); @@ -206,15 +196,13 @@ describe('validateQuotas (buckets)', () => { validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -5000, false, mockLog, err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectDelete', inflight: -5000, - } - ), true); + }), + true, + ); done(); }); }); @@ -229,21 +217,28 @@ describe('validateQuotas (buckets)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore', - true, false, mockLog, err => { + validateQuotas( + request, + mockBucket, + {}, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectRestore', inflight: true, - } - ), true); + }), + true, + ); done(); - }); + }, + ); }); it('should not include the inflights in the request if they are disabled', done => { @@ -257,21 +252,28 @@ describe('validateQuotas (buckets)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, {}, ['objectRestore', 'objectPut'], 'objectRestore', - true, false, mockLog, err => { + validateQuotas( + request, + mockBucket, + {}, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { assert.ifError(err); assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { action: 'objectRestore', inflight: undefined, - } - ), true); - done(); - }); + }), + true, + ); + done(); + }, + ); }); it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => { @@ -284,21 +286,18 @@ describe('validateQuotas (buckets)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, {}, ['objectPut'], 'objectPut', - true, true, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: 0, - } - ), true); - done(); - }); + validateQuotas(request, mockBucket, {}, ['objectPut'], 'objectPut', true, true, mockLog, err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: 0, + }), + true, + ); + done(); + }); }); it('should handle numbers above MAX_SAFE_INTEGER when quota is not exceeded', done => { @@ -308,23 +307,31 @@ describe('validateQuotas (buckets)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, { - ...mockBucket, - getQuota: () => 9007199254740993n, - }, {}, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: 1, - }, - ), true); - done(); - }); + validateQuotas( + request, + { + ...mockBucket, + getQuota: () => 9007199254740993n, + }, + {}, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should handle numbers above MAX_SAFE_INTEGER when quota is exceeded', done => { @@ -334,15 +341,25 @@ describe('validateQuotas (buckets)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, { - ...mockBucket, - getQuota: () => 9007199254740991n, - }, {}, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(request.finalizerHooks.length, 1); - done(); - }); + validateQuotas( + request, + { + ...mockBucket, + getQuota: () => 9007199254740991n, + }, + {}, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual(request.finalizerHooks.length, 1); + done(); + }, + ); }); it('should handle numbers above MAX_SAFE_INTEGER with disabled inflights when quota is not exceeded', done => { @@ -353,23 +370,31 @@ describe('validateQuotas (buckets)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, { - ...mockBucket, - getQuota: () => 9007199254740993n, - }, {}, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: undefined, - }, - ), true); - done(); - }); + validateQuotas( + request, + { + ...mockBucket, + getQuota: () => 9007199254740993n, + }, + {}, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: undefined, + }), + true, + ); + done(); + }, + ); }); }); @@ -398,37 +423,67 @@ describe('validateQuotas (with accounts)', () => { }); it('should return null if quota is <= 0', done => { - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 0n, - }, [], '', false, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 0n, + }, + [], + '', + false, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); + done(); + }, + ); }); it('should not return null if bucket quota is <= 0 but account quota is > 0', done => { - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 1000n, - }, [], '', false, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 1000n, + }, + [], + '', + false, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); + done(); + }, + ); }); it('should return null if scuba is disabled', done => { QuotaService.enabled = false; - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, [], '', false, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + [], + '', + false, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.called, false); + done(); + }, + ); }); it('should return null if metrics retrieval fails', done => { @@ -436,23 +491,32 @@ describe('validateQuotas (with accounts)', () => { const error = new Error('Failed to get metrics'); QuotaService._getLatestMetricsCallback.yields(error); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'bucket', - 'bucketName_1640995200000', - null, - { - action: 'objectPut', - inflight: 1, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectPut', 'getObject'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + // Scuba resolves async in production, so the account branch runs even when the bucket branch errors. + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('bucket', 'bucketName_1640995200000', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should return errors.QuotaExceeded if quota is exceeded', done => { @@ -465,24 +529,32 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 100n, - }, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); - assert.strictEqual(request.finalizerHooks.length, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: 1, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 100n, + }, + ['objectPut', 'getObject'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); + assert.strictEqual(request.finalizerHooks.length, 1); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should not return QuotaExceeded if the quotas are exceeded but operation is a delete', done => { @@ -495,23 +567,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 1000n, - }, ['objectDelete'], 'objectDelete', -50, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectDelete', - inflight: -50, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 1000n, + }, + ['objectDelete'], + 'objectDelete', + -50, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectDelete', + inflight: -50, + }), + true, + ); + done(); + }, + ); }); it('should decrease the inflights by deleting data, and go below 0 to unblock operations', done => { @@ -524,23 +604,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 1000n, - }, ['objectDelete'], 'objectDelete', -5000, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectDelete', - inflight: -5000, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 1000n, + }, + ['objectDelete'], + 'objectDelete', + -5000, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectDelete', + inflight: -5000, + }), + true, + ); + done(); + }, + ); }); it('should return null if quota is not exceeded', done => { @@ -553,23 +641,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectRestore', - inflight: true, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectRestore', + inflight: true, + }), + true, + ); + done(); + }, + ); }); it('should return quota exceeded if account and bucket quotas are different', done => { @@ -582,15 +678,25 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectPut', 'getObject'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 2); - assert.strictEqual(request.finalizerHooks.length, 1); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectPut', 'getObject'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 2); + assert.strictEqual(request.finalizerHooks.length, 1); + done(); + }, + ); }); it('should update the request with one function per action to clear quota updates', done => { @@ -603,23 +709,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectRestore', 'objectPut'], 'objectRestore', true, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectRestore', - inflight: true, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectRestore', 'objectPut'], + 'objectRestore', + true, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 4); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectRestore', + inflight: true, + }), + true, + ); + done(); + }, + ); }); it('should evaluate the quotas and not update the inflights when isStorageReserved is true', done => { @@ -632,23 +746,31 @@ describe('validateQuotas (with accounts)', () => { QuotaService._getLatestMetricsCallback.yields(null, result1); QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2); - validateQuotas(request, mockBucket, { - account: 'test_1', - quota: 1000n, - }, ['objectPut'], 'objectPut', true, true, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: 0, - } - ), true); - done(); - }); + validateQuotas( + request, + mockBucket, + { + account: 'test_1', + quota: 1000n, + }, + ['objectPut'], + 'objectPut', + true, + true, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledTwice, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: 0, + }), + true, + ); + done(); + }, + ); }); it('should handle account numbers above MAX_SAFE_INTEGER when quota is not exceeded', done => { @@ -658,23 +780,31 @@ describe('validateQuotas (with accounts)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 9007199254740993n, - }, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: 1, - }, - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 9007199254740993n, + }, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: 1, + }), + true, + ); + done(); + }, + ); }); it('should handle account numbers above MAX_SAFE_INTEGER when quota is exceeded', done => { @@ -684,15 +814,25 @@ describe('validateQuotas (with accounts)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 9007199254740991n, - }, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.strictEqual(err.is.QuotaExceeded, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(request.finalizerHooks.length, 1); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 9007199254740991n, + }, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.strictEqual(err.is.QuotaExceeded, true); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual(request.finalizerHooks.length, 1); + done(); + }, + ); }); it('should handle account numbers above MAX_SAFE_INTEGER with disabled inflights', done => { @@ -703,23 +843,31 @@ describe('validateQuotas (with accounts)', () => { }; QuotaService._getLatestMetricsCallback.yields(null, result1); - validateQuotas(request, mockBucketNoQuota, { - account: 'test_1', - quota: 9007199254740993n, - }, ['objectPut'], 'objectPut', 1, false, mockLog, err => { - assert.ifError(err); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); - assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith( - 'account', - 'test_1', - null, - { - action: 'objectPut', - inflight: undefined, - }, - ), true); - done(); - }); + validateQuotas( + request, + mockBucketNoQuota, + { + account: 'test_1', + quota: 9007199254740993n, + }, + ['objectPut'], + 'objectPut', + 1, + false, + mockLog, + err => { + assert.ifError(err); + assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true); + assert.strictEqual( + QuotaService._getLatestMetricsCallback.calledWith('account', 'test_1', null, { + action: 'objectPut', + inflight: undefined, + }), + true, + ); + done(); + }, + ); }); }); @@ -746,7 +894,7 @@ describe('processBytesToWrite', () => { ...hotObject, dataStoreName: 'glacier', archive: { - archiveInfo: '{archiveID,archiveVersion}' + archiveInfo: '{archiveID,archiveVersion}', }, }; const restoringObject = { diff --git a/tests/unit/quotas/scuba/wrapper.js b/tests/unit/quotas/scuba/wrapper.js index 57065ff96a..1792fcc97d 100644 --- a/tests/unit/quotas/scuba/wrapper.js +++ b/tests/unit/quotas/scuba/wrapper.js @@ -44,7 +44,7 @@ describe('ScubaClientImpl', () => { }); it('should disable Scuba if health check returns non-stale data', async () => { - sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - (12 * 60 * 60 * 1000) }); + sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - 12 * 60 * 60 * 1000 }); await client._healthCheck(); @@ -52,7 +52,7 @@ describe('ScubaClientImpl', () => { }); it('should disable Scuba if health check returns stale data', async () => { - sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - (48 * 60 * 60 * 1000) }); + sinon.stub(client, 'healthCheck').resolves({ date: Date.now() - 48 * 60 * 60 * 1000 }); await client._healthCheck(); @@ -99,4 +99,55 @@ describe('ScubaClientImpl', () => { assert(clearIntervalStub.calledOnceWith(123)); }); }); + + describe('getUtilizationMetrics', () => { + it('should forward the werelogs req_id chain as the X-Scal-Request-Uids header', done => { + const metricsStub = sinon + .stub(client, '_getLatestMetricsCallback') + .callsArgWith(4, null, { bytesTotal: 0 }); + const reqLog = { getSerializedUids: () => 'req1:req2' }; + + client.getUtilizationMetrics( + 'bucket', + 'k', + null, + { action: 'objectPut', inflight: 1 }, + reqLog, + (err, data) => { + assert.ifError(err); + assert.deepStrictEqual(data, { bytesTotal: 0 }); + const forwardedOptions = metricsStub.getCall(0).args[2]; + assert.strictEqual(forwardedOptions.headers['X-Scal-Request-Uids'], 'req1:req2'); + done(); + }, + ); + }); + + it('should not set X-Scal-Request-Uids when log lacks getSerializedUids', done => { + const metricsStub = sinon.stub(client, '_getLatestMetricsCallback').callsArgWith(4, null, {}); + + client.getUtilizationMetrics('bucket', 'k', null, { action: 'objectPut', inflight: 1 }, {}, err => { + assert.ifError(err); + const forwardedOptions = metricsStub.getCall(0).args[2]; + assert.strictEqual(forwardedOptions, null); + done(); + }); + }); + + it('should merge X-Scal-Request-Uids with existing options headers', done => { + const metricsStub = sinon + .stub(client, '_getLatestMetricsCallback') + .callsArgWith(4, null, { bytesTotal: 0 }); + const reqLog = { getSerializedUids: () => 'req1:req2' }; + const options = { headers: { 'X-Existing': 'v' } }; + + client.getUtilizationMetrics('bucket', 'k', options, { action: 'objectPut', inflight: 1 }, reqLog, err => { + assert.ifError(err); + const forwardedOptions = metricsStub.getCall(0).args[2]; + assert.strictEqual(forwardedOptions.headers['X-Existing'], 'v'); + assert.strictEqual(forwardedOptions.headers['X-Scal-Request-Uids'], 'req1:req2'); + done(); + }); + }); + }); }); diff --git a/tests/unit/routes/veeam-routes.js b/tests/unit/routes/veeam-routes.js index f3eb3d9039..e3eafbd4c6 100644 --- a/tests/unit/routes/veeam-routes.js +++ b/tests/unit/routes/veeam-routes.js @@ -83,7 +83,8 @@ describe('Veeam routes - comprehensive unit tests', () => { response.end.called = true; response.headersSent = true; // Emit finish event when end is called - const finishHandlers = response.on.getCalls() + const finishHandlers = response.on + .getCalls() .filter(call => call.args[0] === 'finish') .map(call => call.args[1]); finishHandlers.forEach(handler => handler()); @@ -104,7 +105,7 @@ describe('Veeam routes - comprehensive unit tests', () => { it('should handle 404 error from UtilizationService and return 200', async () => { const error404 = new Error('Not Found'); error404.response = { status: 404 }; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const request = createRequest(); const response = createResponse(); @@ -113,41 +114,43 @@ describe('Veeam routes - comprehensive unit tests', () => { assert(logWarnSpy.calledOnce, 'log.warn should have been called once'); const warnCall = logWarnSpy.getCall(0); - assert(warnCall.args[0].includes('UtilizationService returned 404'), - 'warning message should mention 404'); + assert(warnCall.args[0].includes('UtilizationService returned 404'), 'warning message should mention 404'); assert.strictEqual(warnCall.args[1].bucket, 'test-bucket'); - assert(response.writeHead.calledWith(200), - 'should return 200 despite 404 from UtilizationService'); + assert(response.writeHead.calledWith(200), 'should return 200 despite 404 from UtilizationService'); assert(response.end.called, 'response should be ended'); }); it('should handle 500 error from UtilizationService and return 500', async () => { const error500 = new Error('Internal Server Error'); error500.response = { status: 500 }; - utilizationStub.callsArgWith(4, error500); + utilizationStub.callsArgWith(5, error500); const request = createRequest(); const response = createResponse(); await getVeeamFile(request, response, bucketMd, log); - assert(response.headersSent || response.write.called || response.writeHead.called, - 'should send error response for 500 errors'); + assert( + response.headersSent || response.write.called || response.writeHead.called, + 'should send error response for 500 errors', + ); }); it('should handle connection error from UtilizationService and return 500', async () => { const errorConn = new Error('Connection refused'); errorConn.code = 'ECONNREFUSED'; - utilizationStub.callsArgWith(4, errorConn); + utilizationStub.callsArgWith(5, errorConn); const request = createRequest(); const response = createResponse(); await getVeeamFile(request, response, bucketMd, log); - assert(response.headersSent || response.write.called || response.writeHead.called, - 'should send error response for connection errors'); + assert( + response.headersSent || response.write.called || response.writeHead.called, + 'should send error response for connection errors', + ); }); it('should successfully use metrics when UtilizationService returns data', async () => { @@ -156,7 +159,7 @@ describe('Veeam routes - comprehensive unit tests', () => { bytesTotal: 123456789, date: metricsDate, }; - utilizationStub.callsArgWith(4, null, bucketMetrics); + utilizationStub.callsArgWith(5, null, bucketMetrics); const request = createRequest(); const response = createResponse(); @@ -168,8 +171,7 @@ describe('Veeam routes - comprehensive unit tests', () => { assert(utilizationStub.calledOnce, 'should call UtilizationService once'); assert(response.end.called, 'response should be ended'); - const lastModifiedCall = response.setHeader.getCalls() - .find(call => call.args[0] === 'Last-Modified'); + const lastModifiedCall = response.setHeader.getCalls().find(call => call.args[0] === 'Last-Modified'); assert(lastModifiedCall, 'Last-Modified header should be set'); assert.strictEqual( @@ -209,7 +211,7 @@ describe('Veeam routes - comprehensive unit tests', () => { // because no metrics are available yet const error404 = new Error('Not Found'); error404.response = { status: 404 }; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const request = createRequest(); const response = createResponse(); @@ -217,8 +219,7 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); assert(logWarnSpy.calledOnce, 'should log warning for 404'); - assert(response.writeHead.calledWith(200), - 'should return 200 with static capacity data for 404'); + assert(response.writeHead.calledWith(200), 'should return 200 with static capacity data for 404'); assert(response.end.called, 'response should be ended'); const warnCall = logWarnSpy.getCall(0); assert(warnCall.args[0].includes('404'), 'warning should mention 404'); @@ -233,8 +234,10 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); - assert(response.headersSent || response.write.called || response.writeHead.called, - 'should send response for metadata errors'); + assert( + response.headersSent || response.write.called || response.writeHead.called, + 'should send response for metadata errors', + ); }); it('should handle tagging query parameter', async () => { @@ -244,8 +247,7 @@ describe('Veeam routes - comprehensive unit tests', () => { await getVeeamFile(request, response, bucketMd, log); - assert(response.writeHead.calledWith(200), - 'should return 200 for tagging query'); + assert(response.writeHead.calledWith(200), 'should return 200 for tagging query'); assert(response.end.called, 'response should be ended'); }); }); @@ -350,7 +352,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => it('should call UtilizationService for capacity.xml and use metrics date', async () => { const metricsDate = '2026-03-26T19:00:08.996Z'; - utilizationStub.callsArgWith(4, null, { bytesTotal: 123456789, date: metricsDate }); + utilizationStub.callsArgWith(5, null, { bytesTotal: 123456789, date: metricsDate }); const request = createRequest('.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml'); const response = createResponse(); @@ -361,8 +363,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => assert(response.setHeader.called, 'should set headers'); assert(response.end.called, 'response should be ended'); - const lastModifiedCall = response.setHeader.getCalls() - .find(call => call.args[0] === 'Last-Modified'); + const lastModifiedCall = response.setHeader.getCalls().find(call => call.args[0] === 'Last-Modified'); assert(lastModifiedCall, 'Last-Modified header should be set'); assert.strictEqual( lastModifiedCall.args[1], @@ -374,7 +375,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => it('should handle 404 from UtilizationService on HEAD and return 200', async () => { const error404 = new Error('Not Found'); error404.response = { status: 404 }; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const request = createRequest('.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml'); const response = createResponse(); @@ -383,8 +384,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => assert(logWarnSpy.calledOnce, 'log.warn should have been called once'); const warnCall = logWarnSpy.getCall(0); - assert(warnCall.args[0].includes('UtilizationService returned 404'), - 'warning message should mention 404'); + assert(warnCall.args[0].includes('UtilizationService returned 404'), 'warning message should mention 404'); assert(response.setHeader.called, 'should set headers'); assert(response.end.called, 'response should be ended'); }); @@ -392,7 +392,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => it('should handle non-404 error from UtilizationService on HEAD and return 500', async () => { const error500 = new Error('Internal Server Error'); error500.response = { status: 500 }; - utilizationStub.callsArgWith(4, error500); + utilizationStub.callsArgWith(5, error500); const request = createRequest('.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml'); const response = createResponse(); @@ -408,7 +408,7 @@ describe('Veeam routes - HEAD request UtilizationService error handling', () => _capabilities: {}, }; metadataStub.callsArgWith(2, null, bucketMdWithoutVeeam); - utilizationStub.callsArgWith(4, null, {}); + utilizationStub.callsArgWith(5, null, {}); const request = createRequest(); const response = createResponse(); @@ -496,7 +496,8 @@ describe('Veeam routes - LIST request handling', () => { response.once.returns(response); response.end.callsFake(() => { response.end.called = true; - const finishHandlers = response.on.getCalls() + const finishHandlers = response.on + .getCalls() .filter(call => call.args[0] === 'finish') .map(call => call.args[1]); finishHandlers.forEach(handler => handler()); @@ -506,7 +507,7 @@ describe('Veeam routes - LIST request handling', () => { it('should list both system.xml and capacity.xml when both are present', async () => { const metricsDate = '2026-03-26T19:00:08.996Z'; - utilizationStub.callsArgWith(4, null, { bytesTotal: 123456789, date: metricsDate }); + utilizationStub.callsArgWith(5, null, { bytesTotal: 123456789, date: metricsDate }); const request = createRequest(); const response = createResponse(); @@ -519,7 +520,7 @@ describe('Veeam routes - LIST request handling', () => { }); it('should emit LastModified in ISO 8601 format in XML body', async () => { - utilizationStub.callsArgWith(4, null, { bytesTotal: 0, date: new Date().toISOString() }); + utilizationStub.callsArgWith(5, null, { bytesTotal: 0, date: new Date().toISOString() }); const request = createRequest(); const response = createResponse(); @@ -544,19 +545,18 @@ describe('Veeam routes - LIST request handling', () => { matches.push(match[1]); } - assert.strictEqual(matches.length, 3, - 'should have 3 LastModified entries (system.xml, capacity.xml, folder)'); + assert.strictEqual(matches.length, 3, 'should have 3 LastModified entries (system.xml, capacity.xml, folder)'); const iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; - matches.forEach( - value => { assert(iso8601Regex.test(value), `LastModified "${value}" should be in ISO 8601 format`); + matches.forEach(value => { + assert(iso8601Regex.test(value), `LastModified "${value}" should be in ISO 8601 format`); }); }); it('should handle 404 from UtilizationService on LIST and return 200', async () => { const error404 = new Error('Not Found'); error404.response = { status: 404 }; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const request = createRequest(); const response = createResponse(); @@ -565,8 +565,7 @@ describe('Veeam routes - LIST request handling', () => { assert(logWarnSpy.calledOnce, 'log.warn should have been called once'); const warnCall = logWarnSpy.getCall(0); - assert(warnCall.args[0].includes('UtilizationService returned 404'), - 'warning message should mention 404'); + assert(warnCall.args[0].includes('UtilizationService returned 404'), 'warning message should mention 404'); assert(response.writeHead.calledWith(200), 'should return 200 despite 404'); assert(response.end.called, 'response should be ended'); }); @@ -574,7 +573,7 @@ describe('Veeam routes - LIST request handling', () => { it('should handle non-404 error from UtilizationService on LIST and return 500', async () => { const error500 = new Error('Internal Server Error'); error500.response = { status: 500 }; - utilizationStub.callsArgWith(4, error500); + utilizationStub.callsArgWith(5, error500); const request = createRequest(); const response = createResponse(); @@ -585,7 +584,7 @@ describe('Veeam routes - LIST request handling', () => { }); it('should handle versions query parameter', async () => { - utilizationStub.callsArgWith(4, null, { bytesTotal: 0, date: new Date().toISOString() }); + utilizationStub.callsArgWith(5, null, { bytesTotal: 0, date: new Date().toISOString() }); const request = createRequest({ versions: '' }); const response = createResponse(); @@ -615,31 +614,28 @@ describe('Veeam routes - LIST request handling', () => { assert(response.end.called, 'response should be ended'); }); - it( - 'should list only available files when only SystemInfo is present, without calling UtilizationService', - async () => { - const bucketMdOnlySystem = { - ...bucketMd, - _capabilities: { - VeeamSOSApi: { - SystemInfo: { - ProtocolVersion: '1.0', - ModelName: 'ARTESCA', - LastModified: '2024-01-01T00:00:00.000Z', - }, + it('should list available files when only SystemInfo is present, without calling UtilizationService', async () => { + const bucketMdOnlySystem = { + ...bucketMd, + _capabilities: { + VeeamSOSApi: { + SystemInfo: { + ProtocolVersion: '1.0', + ModelName: 'ARTESCA', + LastModified: '2024-01-01T00:00:00.000Z', }, }, - }; - metadataStub.callsArgWith(2, null, bucketMdOnlySystem); + }, + }; + metadataStub.callsArgWith(2, null, bucketMdOnlySystem); - const request = createRequest(); - const response = createResponse(); + const request = createRequest(); + const response = createResponse(); - await listVeeamFiles(request, response, bucketMdOnlySystem, log); + await listVeeamFiles(request, response, bucketMdOnlySystem, log); - assert(!utilizationStub.called, 'should not call UtilizationService without CapacityInfo'); - assert(response.writeHead.calledWith(200), 'should return 200'); - assert(response.end.called, 'response should be ended'); - }, - ); + assert(!utilizationStub.called, 'should not call UtilizationService without CapacityInfo'); + assert(response.writeHead.calledWith(200), 'should return 200'); + assert(response.end.called, 'response should be ended'); + }); }); diff --git a/tests/unit/routes/veeam-utils.js b/tests/unit/routes/veeam-utils.js index a76ee65f7c..b896b0e90d 100644 --- a/tests/unit/routes/veeam-utils.js +++ b/tests/unit/routes/veeam-utils.js @@ -35,7 +35,7 @@ describe('fetchCapacityMetrics', () => { }); it('should call UtilizationService with the correct bucket key', async () => { - utilizationStub.callsArgWith(4, null, {}); + utilizationStub.callsArgWith(5, null, {}); await fetchCapacityMetrics(bucketMd, request, log); @@ -46,7 +46,7 @@ describe('fetchCapacityMetrics', () => { it('should resolve with metrics on success', async () => { const bucketMetrics = { bytesTotal: 42, date: '2026-03-26T19:00:08.996Z' }; - utilizationStub.callsArgWith(4, null, bucketMetrics); + utilizationStub.callsArgWith(5, null, bucketMetrics); const metrics = await fetchCapacityMetrics(bucketMd, request, log); @@ -58,7 +58,7 @@ describe('fetchCapacityMetrics', () => { it('should resolve with no error and a default date on 404', async () => { const error404 = new Error('Not Found'); error404.response = { status: 404 }; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const metrics = await fetchCapacityMetrics(bucketMd, request, log); @@ -72,7 +72,7 @@ describe('fetchCapacityMetrics', () => { it('should also handle 404 via statusCode property', async () => { const error404 = new Error('Not Found'); error404.statusCode = 404; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const metrics = await fetchCapacityMetrics(bucketMd, request, log); @@ -83,12 +83,9 @@ describe('fetchCapacityMetrics', () => { it('should reject with error on non-404 failures', async () => { const error500 = new Error('Internal Server Error'); error500.response = { status: 500 }; - utilizationStub.callsArgWith(4, error500); + utilizationStub.callsArgWith(5, error500); - await assert.rejects( - fetchCapacityMetrics(bucketMd, request, log), - err => err === error500, - ); + await assert.rejects(fetchCapacityMetrics(bucketMd, request, log), err => err === error500); assert(logErrorSpy.calledOnce); assert.strictEqual(logErrorSpy.getCall(0).args[1].bucket, 'test-bucket'); @@ -99,12 +96,9 @@ describe('fetchCapacityMetrics', () => { it('should reject with error on connection errors', async () => { const connError = new Error('Connection refused'); connError.code = 'ECONNREFUSED'; - utilizationStub.callsArgWith(4, connError); + utilizationStub.callsArgWith(5, connError); - await assert.rejects( - fetchCapacityMetrics(bucketMd, request, log), - err => err === connError, - ); + await assert.rejects(fetchCapacityMetrics(bucketMd, request, log), err => err === connError); assert(logErrorSpy.calledOnce); assert.strictEqual(logErrorSpy.getCall(0).args[1].statusCode, 'ECONNREFUSED'); @@ -190,7 +184,7 @@ describe('buildVeeamFileData', () => { metadataStub.callsArgWith(2, null, bucketMd); const error500 = new Error('Internal Server Error'); error500.response = { status: 500 }; - utilizationStub.callsArgWith(4, error500); + utilizationStub.callsArgWith(5, error500); await assert.rejects( buildVeeamFileData(createRequest(capacityObjectKey), bucketMd, log), @@ -201,7 +195,7 @@ describe('buildVeeamFileData', () => { it('should build capacity.xml with SUR metrics date and applied Used/Available', async () => { const metricsDate = new Date('2026-03-26T19:00:08.996Z'); metadataStub.callsArgWith(2, null, bucketMd); - utilizationStub.callsArgWith(4, null, { date: metricsDate, bytesTotal: 100 }); + utilizationStub.callsArgWith(5, null, { date: metricsDate, bytesTotal: 100 }); const result = await buildVeeamFileData(createRequest(capacityObjectKey), bucketMd, log); @@ -219,7 +213,7 @@ describe('buildVeeamFileData', () => { metadataStub.callsArgWith(2, null, bucketMd); const error404 = new Error('Not Found'); error404.response = { status: 404 }; - utilizationStub.callsArgWith(4, error404); + utilizationStub.callsArgWith(5, error404); const result = await buildVeeamFileData(createRequest(capacityObjectKey), bucketMd, log); @@ -257,7 +251,7 @@ describe('buildVeeamFileData', () => { }, }; metadataStub.callsArgWith(2, null, bucketMdWithUsed); - utilizationStub.callsArgWith(4, null, { date: new Date(), bytesTotal: 999 }); + utilizationStub.callsArgWith(5, null, { date: new Date(), bytesTotal: 999 }); const result = await buildVeeamFileData(createRequest(capacityObjectKey), bucketMdWithUsed, log);