Skip to content

feat(storage): add x-goog-gcs-idempotency-token header #8837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
thiyaguk09 wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat(storage): add x-goog-gcs-idempotency-token header #8837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0629b7f
feat(storage): add x-goog-gcs-idempotency-token header linked to gccl…
thiyaguk09 Jul 3, 2026
86c1c74
feat: allow user-provided x-goog-gcs-idempotency-token and synchroniz…
thiyaguk09 Jul 3, 2026
d96e9b8
refactor: simplify sourcemap compilation scripts and add optional cha…
thiyaguk09 Jul 3, 2026
f42ee80
fix: ignore invalid empty or undefined idempotency tokens and fallbac…
thiyaguk09 Jul 3, 2026
f482198
fix: remove user-provided token header when injecting GCS idempotency…
thiyaguk09 Jul 3, 2026
b9aa424
fix: improve idempotency token validation and standardize token remov…
thiyaguk09 Jul 3, 2026
46f25f7
fix: perform immutable updates on customRequestOptions to avoid mutat…
thiyaguk09 Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions handwritten/storage/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,20 +54,21 @@
"compile:cjs": "tsc -p ./tsconfig.cjs.json",
"compile:esm": "tsc -p .",
"compile": "npm run compile:cjs && npm run compile:esm",
"compile:sourcemaps": "npm run compile:cjs -- --sourceMap && npm run compile:esm -- --sourceMap",
"conformance-test": "mocha --parallel build/cjs/conformance-test/ --require build/cjs/conformance-test/globalHooks.js",
"docs": "jsdoc -c .jsdoc.json",
"fix": "gts fix",
"lint": "gts check",
"postcompile": "cp ./src/package-json-helper.cjs ./build/cjs/src && cp ./src/package-json-helper.cjs ./build/esm/src",
"postcompile:cjs": "babel --plugins gapic-tools/build/src/replaceImportMetaUrl,gapic-tools/build/src/toggleESMFlagVariable build/cjs/src/util.js -o build/cjs/src/util.js && cp internal-tooling/helpers/package.cjs.json build/cjs/package.json",
"precompile": "rm -rf build/",
"preconformance-test": "npm run compile:cjs -- --sourceMap",
"predocs": "npm run compile:cjs -- --sourceMap",
"preconformance-test": "npm run compile:sourcemaps",
"predocs": "npm run compile:sourcemaps",
"prelint": "cd samples; npm link ../; npm install",
"prepare": "npm run compile",
"presystem-test:esm": "npm run compile:esm",
"presystem-test": "npm run compile -- --sourceMap",
"pretest": "npm run compile -- --sourceMap",
"presystem-test": "npm run compile:sourcemaps",
"pretest": "npm run compile:sourcemaps",
"samples-test": "npm link && cd samples/ && npm link ../ && npm test && cd ../",
"system-test:esm": "mocha build/esm/system-test --timeout 600000 --exit",
"system-test": "mocha build/cjs/system-test --timeout 600000 --exit",
Expand Down
19 changes: 17 additions & 2 deletions handwritten/storage/src/nodejs-common/service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,13 +269,28 @@ export class Service {
if (this.providedUserAgent) {
userAgent = `${this.providedUserAgent} ${userAgent}`;
}
const headers = reqOpts.headers || {};
const userTokenKey = Object.keys(headers).find(
key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
);
const userTokenValue = userTokenKey ? headers[userTokenKey] : undefined;
const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== '';
const idempotencyToken = hasValidUserToken
? (userTokenValue as string)
: crypto.randomUUID();
reqOpts.headers = {
...reqOpts.headers,
...headers,
'User-Agent': userAgent,
'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
pkg.version
}-${getModuleFormat()} gccl-invocation-id/${crypto.randomUUID()}`,
}-${getModuleFormat()} gccl-invocation-id/${idempotencyToken}`,
};
if (!hasValidUserToken) {
if (userTokenKey) {
delete reqOpts.headers[userTokenKey];
}
reqOpts.headers['x-goog-gcs-idempotency-token'] = idempotencyToken;
}

if (reqOpts[GCCL_GCS_CMD_KEY]) {
reqOpts.headers['x-goog-api-client'] +=
Expand Down
4 changes: 3 additions & 1 deletion handwritten/storage/src/nodejs-common/util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1042,11 +1042,13 @@ export class Util {
}

_getDefaultHeaders(gcclGcsCmd?: string) {
const idempotencyToken = crypto.randomUUID();
const headers = {
'User-Agent': getUserAgentString(),
'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
packageJson.version
}-${getModuleFormat()} gccl-invocation-id/${crypto.randomUUID()}`,
}-${getModuleFormat()} gccl-invocation-id/${idempotencyToken}`,
'x-goog-gcs-idempotency-token': idempotencyToken,
};

if (gcclGcsCmd) {
Expand Down
63 changes: 62 additions & 1 deletion handwritten/storage/src/resumable-upload.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,11 @@ export class Upload extends Writable {
timeOfFirstRequest: number;
isPartialUpload: boolean;

private currentInvocationId = {
private currentInvocationId: {
checkUploadStatus: string;
chunk: string;
uri: string;
} = {
checkUploadStatus: crypto.randomUUID(),
chunk: crypto.randomUUID(),
uri: crypto.randomUUID(),
Expand Down Expand Up @@ -806,6 +810,21 @@ export class Upload extends Writable {
delete metadata.contentType;
}

const userTokenKey = Object.keys(this.customRequestOptions?.headers || {}).find(
key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
);
const userTokenValue = userTokenKey ? this.customRequestOptions?.headers?.[userTokenKey] : undefined;
const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== '';
if (hasValidUserToken) {
this.currentInvocationId.uri = userTokenValue as string;
} else if (userTokenKey && this.customRequestOptions?.headers) {
this.customRequestOptions = {
...this.customRequestOptions,
headers: { ...this.customRequestOptions.headers },
};
delete this.customRequestOptions.headers![userTokenKey];
}
Comment thread
thiyaguk09 marked this conversation as resolved.

let googAPIClient = `${getRuntimeTrackingString()} gccl/${
packageJson.version
}-${getModuleFormat()} gccl-invocation-id/${this.currentInvocationId.uri}`;
Expand Down Expand Up @@ -833,6 +852,10 @@ export class Upload extends Writable {
},
};

if (!hasValidUserToken) {
reqOpts.headers!['x-goog-gcs-idempotency-token'] = this.currentInvocationId.uri;
}

if (metadata.contentLength) {
reqOpts.headers!['X-Upload-Content-Length'] =
metadata.contentLength.toString();
Expand Down Expand Up @@ -995,6 +1018,21 @@ export class Upload extends Writable {
},
});

const userTokenKey = Object.keys(this.customRequestOptions?.headers || {}).find(
key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
);
const userTokenValue = userTokenKey ? this.customRequestOptions?.headers?.[userTokenKey] : undefined;
const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== '';
if (hasValidUserToken) {
this.currentInvocationId.chunk = userTokenValue as string;
} else if (userTokenKey && this.customRequestOptions?.headers) {
this.customRequestOptions = {
...this.customRequestOptions,
headers: { ...this.customRequestOptions.headers },
};
delete this.customRequestOptions.headers![userTokenKey];
}
Comment thread
thiyaguk09 marked this conversation as resolved.

let googAPIClient = `${getRuntimeTrackingString()} gccl/${
packageJson.version
}-${getModuleFormat()} gccl-invocation-id/${
Expand All @@ -1010,6 +1048,10 @@ export class Upload extends Writable {
'x-goog-api-client': googAPIClient,
};

if (!hasValidUserToken) {
headers['x-goog-gcs-idempotency-token'] = this.currentInvocationId.chunk;
}

// If using multiple chunk upload, set appropriate header
if (multiChunkMode) {
// We need to know how much data is available upstream to set the `Content-Range` header.
Expand Down Expand Up @@ -1209,6 +1251,21 @@ export class Upload extends Writable {
async checkUploadStatus(
config: CheckUploadStatusConfig = {},
): Promise<GaxiosResponse<FileMetadata | void>> {
const userTokenKey = Object.keys(this.customRequestOptions?.headers || {}).find(
key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
);
const userTokenValue = userTokenKey ? this.customRequestOptions?.headers?.[userTokenKey] : undefined;
const hasValidUserToken = typeof userTokenValue === 'string' && userTokenValue !== '';
if (hasValidUserToken) {
this.currentInvocationId.checkUploadStatus = userTokenValue as string;
} else if (userTokenKey && this.customRequestOptions?.headers) {
this.customRequestOptions = {
...this.customRequestOptions,
headers: { ...this.customRequestOptions.headers },
};
delete this.customRequestOptions.headers![userTokenKey];
}
Comment thread
thiyaguk09 marked this conversation as resolved.

let googAPIClient = `${getRuntimeTrackingString()} gccl/${
packageJson.version
}-${getModuleFormat()} gccl-invocation-id/${
Expand All @@ -1230,6 +1287,10 @@ export class Upload extends Writable {
},
};

if (!hasValidUserToken) {
opts.headers!['x-goog-gcs-idempotency-token'] = this.currentInvocationId.checkUploadStatus;
}

try {
const resp = await this.makeRequest(opts);

Expand Down
48 changes: 38 additions & 10 deletions handwritten/storage/test/headers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,15 @@ describe('headers', () => {
} catch (err) {
if (err !== error) throw err;
}
assert.ok(
/^gl-node\/(?<nodeVersion>[^W]+) gccl\/(?<gccl>[^W]+) gccl-invocation-id\/(?<gcclInvocationId>[^W]+)$/.test(
requests[0].headers['x-goog-api-client']
)
);
const apiClientHeader = requests[0].headers['x-goog-api-client'];
const match =
/^gl-node\/(?<nodeVersion>[^W]+) gccl\/(?<gccl>[^W]+) gccl-invocation-id\/(?<gcclInvocationId>[^W]+)$/.exec(
apiClientHeader
);
assert.ok(match);
const invocationId = match.groups!.gcclInvocationId;
const idempotencyToken = requests[0].headers['x-goog-gcs-idempotency-token'];
assert.strictEqual(idempotencyToken, invocationId);
});

it('populates x-goog-api-client header (deno)', async () => {
Expand All @@ -87,10 +91,34 @@ describe('headers', () => {
} catch (err) {
if (err !== error) throw err;
}
assert.ok(
/^gl-deno\/0.00.0 gccl\/(?<gccl>[^W]+) gccl-invocation-id\/(?<gcclInvocationId>[^W]+)$/.test(
requests[1].headers['x-goog-api-client']
)
);
const apiClientHeader = requests[1].headers['x-goog-api-client'];
const match =
/^gl-deno\/0.00.0 gccl\/(?<gccl>[^W]+) gccl-invocation-id\/(?<gcclInvocationId>[^W]+)$/.exec(
apiClientHeader
);
assert.ok(match);
const invocationId = match.groups!.gcclInvocationId;
const idempotencyToken = requests[1].headers['x-goog-gcs-idempotency-token'];
assert.strictEqual(idempotencyToken, invocationId);
});

it('generates unique tokens for different requests', async () => {
const storage = new Storage();
const bucket = storage.bucket('foo-bucket');
try {
await bucket.create();
} catch (err) {
if (err !== error) throw err;
}
try {
await bucket.create();
} catch (err) {
if (err !== error) throw err;
}
const token1 = requests[requests.length - 2].headers['x-goog-gcs-idempotency-token'];
const token2 = requests[requests.length - 1].headers['x-goog-gcs-idempotency-token'];
assert.ok(token1);
assert.ok(token2);
assert.notStrictEqual(token1, token2);
});
});
76 changes: 76 additions & 0 deletions handwritten/storage/test/nodejs-common/service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,82 @@ describe('Service', () => {
service.request_(reqOpts, assert.ifError);
});

it('should add the x-goog-gcs-idempotency-token header matching the gccl-invocation-id', done => {
service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => {
const pkg = service.packageJson;
const r = new RegExp(
`^gl-node/${process.versions.node} gccl/${
pkg.version
}-${getModuleFormat()} gccl-invocation-id/(?<gcclInvocationId>[^W]+)$`
);
const match = r.exec(reqOpts.headers!['x-goog-api-client']);
assert.ok(match);
const invocationId = match.groups!.gcclInvocationId;
const idempotencyToken = reqOpts.headers!['x-goog-gcs-idempotency-token'];
assert.strictEqual(idempotencyToken, invocationId);
done();
};

service.request_(reqOpts, assert.ifError);
});

it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id', done => {
const customToken = 'my-custom-token-123';
const customReqOpts = {
...reqOpts,
headers: {
'X-Goog-Gcs-Idempotency-Token': customToken,
},
};

service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => {
const pkg = service.packageJson;
const r = new RegExp(
`^gl-node/${process.versions.node} gccl/${
pkg.version
}-${getModuleFormat()} gccl-invocation-id/(?<gcclInvocationId>[^W]+)$`
);
const match = r.exec(reqOpts.headers!['x-goog-api-client']);
assert.ok(match);
const invocationId = match.groups!.gcclInvocationId;
assert.strictEqual(invocationId, customToken);

// Verify there is no duplicate x-goog-gcs-idempotency-token header
assert.strictEqual(reqOpts.headers!['x-goog-gcs-idempotency-token'], undefined);
assert.strictEqual(reqOpts.headers!['X-Goog-Gcs-Idempotency-Token'], customToken);
done();
};

service.request_(customReqOpts, assert.ifError);
});

it('should ignore invalid user-provided idempotency tokens and fallback to generating a UUID', done => {
const customReqOpts = {
...reqOpts,
headers: {
'X-Goog-Gcs-Idempotency-Token': undefined as unknown as string,
},
};

service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => {
const pkg = service.packageJson;
const r = new RegExp(
`^gl-node/${process.versions.node} gccl/${pkg.version
}-${getModuleFormat()} gccl-invocation-id/(?<gcclInvocationId>[^W]+)$`
);
const match = r.exec(reqOpts.headers!['x-goog-api-client']);
assert.ok(match);
const invocationId = match.groups!.gcclInvocationId;

// Verify a fallback token was generated and matches the invocation ID
const idempotencyToken = reqOpts.headers!['x-goog-gcs-idempotency-token'];
assert.strictEqual(idempotencyToken, invocationId);
done();
};

service.request_(customReqOpts, assert.ifError);
});

it('should add the `gccl-gcs-cmd` to the api-client header when provided', done => {
const expected = 'example.expected/value';
service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => {
Expand Down
13 changes: 9 additions & 4 deletions handwritten/storage/test/nodejs-common/util.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,15 @@ function fakeRequest() {
}

fakeRequest.defaults = (defaults: r.CoreOptions) => {
assert.ok(
/^gl-node\/(?<nodeVersion>[^W]+) gccl\/(?<gccl>[^W]+) gccl-invocation-id\/(?<gcclInvocationId>[^W]+)$/.test(
defaults.headers!['x-goog-api-client']
)
const match =
/^gl-node\/(?<nodeVersion>[^W]+) gccl\/(?<gccl>[^W]+) gccl-invocation-id\/(?<gcclInvocationId>[^W]+)$/.exec(
defaults.headers!['x-goog-api-client'] as string
);
assert.ok(match);
const invocationId = match.groups!.gcclInvocationId;
assert.strictEqual(
defaults.headers!['x-goog-gcs-idempotency-token'],
invocationId
);
return fakeRequest;
};
Expand Down
Loading
Loading