diff --git a/handwritten/storage/package.json b/handwritten/storage/package.json index e38124dfa44f..c856420deaa1 100644 --- a/handwritten/storage/package.json +++ b/handwritten/storage/package.json @@ -54,6 +54,7 @@ "compile:cjs": "tsc -p ./tsconfig.cjs.json", "compile:esm": "tsc -p .", "compile": "npm run compile:cjs && npm run compile:esm", + "compile:sourcemaps": "npm run compile:cjs -- --sourceMap && npm run compile:esm -- --sourceMap", "conformance-test": "mocha --parallel build/cjs/conformance-test/ --require build/cjs/conformance-test/globalHooks.js", "docs": "jsdoc -c .jsdoc.json", "fix": "gts fix", @@ -61,13 +62,13 @@ "postcompile": "cp ./src/package-json-helper.cjs ./build/cjs/src && cp ./src/package-json-helper.cjs ./build/esm/src", "postcompile:cjs": "babel --plugins gapic-tools/build/src/replaceImportMetaUrl,gapic-tools/build/src/toggleESMFlagVariable build/cjs/src/util.js -o build/cjs/src/util.js && cp internal-tooling/helpers/package.cjs.json build/cjs/package.json", "precompile": "rm -rf build/", - "preconformance-test": "npm run compile:cjs -- --sourceMap", - "predocs": "npm run compile:cjs -- --sourceMap", + "preconformance-test": "npm run compile:sourcemaps", + "predocs": "npm run compile:sourcemaps", "prelint": "cd samples; npm link ../; npm install", "prepare": "npm run compile", "presystem-test:esm": "npm run compile:esm", - "presystem-test": "npm run compile -- --sourceMap", - "pretest": "npm run compile -- --sourceMap", + "presystem-test": "npm run compile:sourcemaps", + "pretest": "npm run compile:sourcemaps", "samples-test": "npm link && cd samples/ && npm link ../ && npm test && cd ../", "system-test:esm": "mocha build/esm/system-test --timeout 600000 --exit", "system-test": "mocha build/cjs/system-test --timeout 600000 --exit", diff --git a/handwritten/storage/src/nodejs-common/service.ts b/handwritten/storage/src/nodejs-common/service.ts index 9173a38f73d7..5a54ebf6ff86 100644 --- a/handwritten/storage/src/nodejs-common/service.ts +++ b/handwritten/storage/src/nodejs-common/service.ts @@ -269,13 +269,28 @@ export class Service { if (this.providedUserAgent) { userAgent = `${this.providedUserAgent} ${userAgent}`; } + const headers = reqOpts.headers || {}; + const userTokenKey = Object.keys(headers).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey ? headers[userTokenKey] : undefined; + const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== ''; + const idempotencyToken = hasValidUserToken + ? (userTokenValue as string) + : crypto.randomUUID(); reqOpts.headers = { - ...reqOpts.headers, + ...headers, 'User-Agent': userAgent, 'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${ pkg.version - }-${getModuleFormat()} gccl-invocation-id/${crypto.randomUUID()}`, + }-${getModuleFormat()} gccl-invocation-id/${idempotencyToken}`, }; + if (!hasValidUserToken) { + if (userTokenKey) { + delete reqOpts.headers[userTokenKey]; + } + reqOpts.headers['x-goog-gcs-idempotency-token'] = idempotencyToken; + } if (reqOpts[GCCL_GCS_CMD_KEY]) { reqOpts.headers['x-goog-api-client'] += diff --git a/handwritten/storage/src/nodejs-common/util.ts b/handwritten/storage/src/nodejs-common/util.ts index e6c4db98b095..f0c4c7f419b2 100644 --- a/handwritten/storage/src/nodejs-common/util.ts +++ b/handwritten/storage/src/nodejs-common/util.ts @@ -1042,11 +1042,13 @@ export class Util { } _getDefaultHeaders(gcclGcsCmd?: string) { + const idempotencyToken = crypto.randomUUID(); const headers = { 'User-Agent': getUserAgentString(), 'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${ packageJson.version - }-${getModuleFormat()} gccl-invocation-id/${crypto.randomUUID()}`, + }-${getModuleFormat()} gccl-invocation-id/${idempotencyToken}`, + 'x-goog-gcs-idempotency-token': idempotencyToken, }; if (gcclGcsCmd) { diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts index 9ebbb6f37a85..9515868c051a 100644 --- a/handwritten/storage/src/resumable-upload.ts +++ b/handwritten/storage/src/resumable-upload.ts @@ -338,7 +338,11 @@ export class Upload extends Writable { timeOfFirstRequest: number; isPartialUpload: boolean; - private currentInvocationId = { + private currentInvocationId: { + checkUploadStatus: string; + chunk: string; + uri: string; + } = { checkUploadStatus: crypto.randomUUID(), chunk: crypto.randomUUID(), uri: crypto.randomUUID(), @@ -806,6 +810,21 @@ export class Upload extends Writable { delete metadata.contentType; } + const userTokenKey = Object.keys(this.customRequestOptions?.headers || {}).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey ? this.customRequestOptions?.headers?.[userTokenKey] : undefined; + const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== ''; + if (hasValidUserToken) { + this.currentInvocationId.uri = userTokenValue as string; + } else if (userTokenKey && this.customRequestOptions?.headers) { + this.customRequestOptions = { + ...this.customRequestOptions, + headers: { ...this.customRequestOptions.headers }, + }; + delete this.customRequestOptions.headers![userTokenKey]; + } + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ packageJson.version }-${getModuleFormat()} gccl-invocation-id/${this.currentInvocationId.uri}`; @@ -833,6 +852,10 @@ export class Upload extends Writable { }, }; + if (!hasValidUserToken) { + reqOpts.headers!['x-goog-gcs-idempotency-token'] = this.currentInvocationId.uri; + } + if (metadata.contentLength) { reqOpts.headers!['X-Upload-Content-Length'] = metadata.contentLength.toString(); @@ -995,6 +1018,21 @@ export class Upload extends Writable { }, }); + const userTokenKey = Object.keys(this.customRequestOptions?.headers || {}).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey ? this.customRequestOptions?.headers?.[userTokenKey] : undefined; + const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== ''; + if (hasValidUserToken) { + this.currentInvocationId.chunk = userTokenValue as string; + } else if (userTokenKey && this.customRequestOptions?.headers) { + this.customRequestOptions = { + ...this.customRequestOptions, + headers: { ...this.customRequestOptions.headers }, + }; + delete this.customRequestOptions.headers![userTokenKey]; + } + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ packageJson.version }-${getModuleFormat()} gccl-invocation-id/${ @@ -1010,6 +1048,10 @@ export class Upload extends Writable { 'x-goog-api-client': googAPIClient, }; + if (!hasValidUserToken) { + headers['x-goog-gcs-idempotency-token'] = this.currentInvocationId.chunk; + } + // If using multiple chunk upload, set appropriate header if (multiChunkMode) { // We need to know how much data is available upstream to set the `Content-Range` header. @@ -1209,6 +1251,21 @@ export class Upload extends Writable { async checkUploadStatus( config: CheckUploadStatusConfig = {}, ): Promise> { + const userTokenKey = Object.keys(this.customRequestOptions?.headers || {}).find( + key => key.toLowerCase() === 'x-goog-gcs-idempotency-token' + ); + const userTokenValue = userTokenKey ? this.customRequestOptions?.headers?.[userTokenKey] : undefined; + const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== ''; + if (hasValidUserToken) { + this.currentInvocationId.checkUploadStatus = userTokenValue as string; + } else if (userTokenKey && this.customRequestOptions?.headers) { + this.customRequestOptions = { + ...this.customRequestOptions, + headers: { ...this.customRequestOptions.headers }, + }; + delete this.customRequestOptions.headers![userTokenKey]; + } + let googAPIClient = `${getRuntimeTrackingString()} gccl/${ packageJson.version }-${getModuleFormat()} gccl-invocation-id/${ @@ -1230,6 +1287,10 @@ export class Upload extends Writable { }, }; + if (!hasValidUserToken) { + opts.headers!['x-goog-gcs-idempotency-token'] = this.currentInvocationId.checkUploadStatus; + } + try { const resp = await this.makeRequest(opts); diff --git a/handwritten/storage/test/headers.ts b/handwritten/storage/test/headers.ts index 9ccc685814bb..a19e56556597 100644 --- a/handwritten/storage/test/headers.ts +++ b/handwritten/storage/test/headers.ts @@ -65,11 +65,15 @@ describe('headers', () => { } catch (err) { if (err !== error) throw err; } - assert.ok( - /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - requests[0].headers['x-goog-api-client'] - ) - ); + const apiClientHeader = requests[0].headers['x-goog-api-client']; + const match = + /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.exec( + apiClientHeader + ); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = requests[0].headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); }); it('populates x-goog-api-client header (deno)', async () => { @@ -87,10 +91,34 @@ describe('headers', () => { } catch (err) { if (err !== error) throw err; } - assert.ok( - /^gl-deno\/0.00.0 gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - requests[1].headers['x-goog-api-client'] - ) - ); + const apiClientHeader = requests[1].headers['x-goog-api-client']; + const match = + /^gl-deno\/0.00.0 gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.exec( + apiClientHeader + ); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = requests[1].headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + }); + + it('generates unique tokens for different requests', async () => { + const storage = new Storage(); + const bucket = storage.bucket('foo-bucket'); + try { + await bucket.create(); + } catch (err) { + if (err !== error) throw err; + } + try { + await bucket.create(); + } catch (err) { + if (err !== error) throw err; + } + const token1 = requests[requests.length - 2].headers['x-goog-gcs-idempotency-token']; + const token2 = requests[requests.length - 1].headers['x-goog-gcs-idempotency-token']; + assert.ok(token1); + assert.ok(token2); + assert.notStrictEqual(token1, token2); }); }); diff --git a/handwritten/storage/test/nodejs-common/service.ts b/handwritten/storage/test/nodejs-common/service.ts index 502c4e5419f9..66ebd1cc8f53 100644 --- a/handwritten/storage/test/nodejs-common/service.ts +++ b/handwritten/storage/test/nodejs-common/service.ts @@ -487,6 +487,82 @@ describe('Service', () => { service.request_(reqOpts, assert.ifError); }); + it('should add the x-goog-gcs-idempotency-token header matching the gccl-invocation-id', done => { + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?[^W]+)$` + ); + const match = r.exec(reqOpts.headers!['x-goog-api-client']); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = reqOpts.headers!['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + done(); + }; + + service.request_(reqOpts, assert.ifError); + }); + + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id', done => { + const customToken = 'my-custom-token-123'; + const customReqOpts = { + ...reqOpts, + headers: { + 'X-Goog-Gcs-Idempotency-Token': customToken, + }, + }; + + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${ + pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?[^W]+)$` + ); + const match = r.exec(reqOpts.headers!['x-goog-api-client']); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual(invocationId, customToken); + + // Verify there is no duplicate x-goog-gcs-idempotency-token header + assert.strictEqual(reqOpts.headers!['x-goog-gcs-idempotency-token'], undefined); + assert.strictEqual(reqOpts.headers!['X-Goog-Gcs-Idempotency-Token'], customToken); + done(); + }; + + service.request_(customReqOpts, assert.ifError); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID', done => { + const customReqOpts = { + ...reqOpts, + headers: { + 'X-Goog-Gcs-Idempotency-Token': undefined as unknown as string, + }, + }; + + service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { + const pkg = service.packageJson; + const r = new RegExp( + `^gl-node/${process.versions.node} gccl/${pkg.version + }-${getModuleFormat()} gccl-invocation-id/(?[^W]+)$` + ); + const match = r.exec(reqOpts.headers!['x-goog-api-client']); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + + // Verify a fallback token was generated and matches the invocation ID + const idempotencyToken = reqOpts.headers!['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + done(); + }; + + service.request_(customReqOpts, assert.ifError); + }); + it('should add the `gccl-gcs-cmd` to the api-client header when provided', done => { const expected = 'example.expected/value'; service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => { diff --git a/handwritten/storage/test/nodejs-common/util.ts b/handwritten/storage/test/nodejs-common/util.ts index 88e6905ffb52..cf3a9ef47d25 100644 --- a/handwritten/storage/test/nodejs-common/util.ts +++ b/handwritten/storage/test/nodejs-common/util.ts @@ -77,10 +77,15 @@ function fakeRequest() { } fakeRequest.defaults = (defaults: r.CoreOptions) => { - assert.ok( - /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.test( - defaults.headers!['x-goog-api-client'] - ) + const match = + /^gl-node\/(?[^W]+) gccl\/(?[^W]+) gccl-invocation-id\/(?[^W]+)$/.exec( + defaults.headers!['x-goog-api-client'] as string + ); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual( + defaults.headers!['x-goog-gcs-idempotency-token'], + invocationId ); return fakeRequest; }; diff --git a/handwritten/storage/test/resumable-upload.ts b/handwritten/storage/test/resumable-upload.ts index 381044d64d9d..ad8243e28cee 100644 --- a/handwritten/storage/test/resumable-upload.ts +++ b/handwritten/storage/test/resumable-upload.ts @@ -877,12 +877,96 @@ describe('resumable-upload', () => { delete metadataNoHeaders.contentLength; delete metadataNoHeaders.contentType; assert.deepStrictEqual(reqOpts.data, metadataNoHeaders); + assert(reqOpts.headers); + const apiClientHeader = reqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + const idempotencyToken = reqOpts.headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); done(); return {headers: {location: '/foo'}}; }; up.createURI(); }); + it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id in createURI', async () => { + const customToken = 'my-custom-resumable-token'; + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': customToken, + }, + }; + + up.authClient.request = async (combinedReqOpts: GaxiosOptions) => { + assert(combinedReqOpts.headers); + const apiClientHeader = combinedReqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + assert.strictEqual(invocationId, customToken); + + // Verify there is no duplicate x-goog-gcs-idempotency-token header + assert.strictEqual(combinedReqOpts.headers['x-goog-gcs-idempotency-token'], undefined); + assert.strictEqual(combinedReqOpts.headers['X-Goog-Gcs-Idempotency-Token'], customToken); + return {headers: {location: '/foo'}}; + }; + + await up.createURI(); + }); + + it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID in createURI', async () => { + up.customRequestOptions = { + headers: { + 'X-Goog-Gcs-Idempotency-Token': '', // invalid empty string + }, + }; + + up.authClient.request = async (combinedReqOpts: GaxiosOptions) => { + assert(combinedReqOpts.headers); + const apiClientHeader = combinedReqOpts.headers['x-goog-api-client']; + const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string); + assert.ok(match); + const invocationId = match.groups!.gcclInvocationId; + + // Verify a fallback token was generated and matches the invocation ID + const idempotencyToken = combinedReqOpts.headers['x-goog-gcs-idempotency-token']; + assert.strictEqual(idempotencyToken, invocationId); + return {headers: {location: '/foo'}}; + }; + + await up.createURI(); + }); + + it('should reuse the same x-goog-gcs-idempotency-token on retry of createURI', async () => { + let invocationCount = 0; + let token1 = ''; + let token2 = ''; + + up.makeRequest = async (reqOpts: GaxiosOptions) => { + invocationCount++; + assert(reqOpts.headers); + if (invocationCount === 1) { + token1 = reqOpts.headers['x-goog-gcs-idempotency-token'] as string; + const error = new GaxiosError( + 'Retriable error', + {} as GaxiosOptions, + { status: 500 } as GaxiosResponse + ); + throw error; + } else if (invocationCount === 2) { + token2 = reqOpts.headers['x-goog-gcs-idempotency-token'] as string; + return { headers: { location: '/foo' } }; + } + return { headers: { location: '/foo' } }; + }; + + await up.createURI(); + assert.strictEqual(invocationCount, 2); + assert.ok(token1); + assert.strictEqual(token1, token2); + }); + it('should pass through the KMS key name', done => { const kmsKeyName = 'kms-key-name'; const up = upload({