diff --git a/handwritten/storage/package.json b/handwritten/storage/package.json index 3ea3b0fb2644..5afb090b3a11 100644 --- a/handwritten/storage/package.json +++ b/handwritten/storage/package.json @@ -81,13 +81,13 @@ "async-retry": "^1.3.3", "duplexify": "^4.1.3", "fast-xml-parser": "^5.3.4", - "gaxios": "^6.0.2", - "google-auth-library": "^9.6.3", + "gaxios": "^7.1.6", + "google-auth-library": "^10.9.0", "html-entities": "^2.5.2", "mime": "^3.0.0", "p-limit": "^3.0.1", "retry-request": "^7.0.0", - "teeny-request": "^9.0.0" + "teeny-request": "^10.1.0" }, "devDependencies": { "@babel/cli": "^7.22.10", diff --git a/handwritten/storage/src/resumable-upload.ts b/handwritten/storage/src/resumable-upload.ts index 9ebbb6f37a85..1ce79ed1f050 100644 --- a/handwritten/storage/src/resumable-upload.ts +++ b/handwritten/storage/src/resumable-upload.ts @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import AbortController from 'abort-controller'; import {createHash} from 'crypto'; import { GaxiosOptions, @@ -46,6 +45,13 @@ const NOT_FOUND_STATUS_CODE = 404; const RESUMABLE_INCOMPLETE_STATUS_CODE = 308; const packageJson = getPackageJSON(); +function getResponseHeader(headers: any, name: string): string | undefined { + if (headers && typeof headers.get === 'function') { + return headers.get(name) || undefined; + } + return headers?.[name] || headers?.[name.toLowerCase()] || undefined; +} + export const PROTOCOL_REGEX = /^(\w*):\/\//; export interface ErrorWithCode extends Error { @@ -615,7 +621,7 @@ export class Upload extends Writable { * * @param headers The headers object to modify. */ - #applyChecksumHeaders(headers: GaxiosOptions['headers']) { + #applyChecksumHeaders(headers: Record) { const checksums: string[] = []; if (this.#hashValidator?.crc32cEnabled) { @@ -631,7 +637,7 @@ export class Upload extends Writable { } if (checksums.length > 0) { - headers!['X-Goog-Hash'] = checksums.join(','); + headers['X-Goog-Hash'] = checksums.join(','); } } @@ -792,7 +798,7 @@ export class Upload extends Writable { protected async createURIAsync(): Promise { const metadata = {...this.metadata}; - const headers: gaxios.Headers = {}; + const headers: Record = {}; // Delete content length and content type from metadata if they exist. // These are headers and should not be sent as part of the metadata. @@ -802,7 +808,7 @@ export class Upload extends Writable { } if (metadata.contentType) { - headers!['X-Upload-Content-Type'] = metadata.contentType; + headers['X-Upload-Content-Type'] = metadata.contentType; delete metadata.contentType; } @@ -833,13 +839,16 @@ export class Upload extends Writable { }, }; - if (metadata.contentLength) { - reqOpts.headers!['X-Upload-Content-Length'] = - metadata.contentLength.toString(); - } + const reqHeaders = reqOpts.headers; + if (reqHeaders && !(reqHeaders instanceof Headers) && !Array.isArray(reqHeaders)) { + if (metadata.contentLength) { + reqHeaders['X-Upload-Content-Length'] = + metadata.contentLength.toString(); + } - if (metadata.contentType) { - reqOpts.headers!['X-Upload-Content-Type'] = metadata.contentType; + if (metadata.contentType) { + reqHeaders['X-Upload-Content-Type'] = metadata.contentType; + } } if (typeof this.generation !== 'undefined') { @@ -855,7 +864,10 @@ export class Upload extends Writable { } if (this.origin) { - reqOpts.headers!.Origin = this.origin; + const reqHeaders = reqOpts.headers; + if (reqHeaders && !(reqHeaders instanceof Headers) && !Array.isArray(reqHeaders)) { + reqHeaders.Origin = this.origin; + } } const uri = await AsyncRetry( async (bail: (err: Error) => void) => { @@ -863,7 +875,7 @@ export class Upload extends Writable { const res = await this.makeRequest(reqOpts); // We have successfully got a URI we can now create a new invocation id this.currentInvocationId.uri = crypto.randomUUID(); - return res.headers.location; + return getResponseHeader(res.headers, 'location') || ''; } catch (err) { const e = err as GaxiosError; const apiError = { @@ -882,7 +894,8 @@ export class Upload extends Writable { ) { throw e; } else { - return bail(e); + bail(e); + return ''; } } }, @@ -1005,7 +1018,7 @@ export class Upload extends Writable { googAPIClient += ` gccl-gcs-cmd/${this.#gcclGcsCmd}`; } - const headers: GaxiosOptions['headers'] = { + const headers: Record = { 'User-Agent': getUserAgentString(), 'x-goog-api-client': googAPIClient, }; @@ -1050,7 +1063,7 @@ export class Upload extends Writable { // `Content-Length` for multiple chunk uploads is the size of the chunk, // not the overall object - headers['Content-Length'] = bytesToUpload; + headers['Content-Length'] = bytesToUpload.toString(); headers['Content-Range'] = `bytes ${this.offset}-${endingByte}/${totalObjectSize}`; @@ -1111,7 +1124,7 @@ export class Upload extends Writable { const shouldContinueWithNextMultiChunkRequest = this.chunkSize && resp.status === RESUMABLE_INCOMPLETE_STATUS_CODE && - resp.headers.range && + getResponseHeader(resp.headers, 'range') && moreDataToUpload; /** @@ -1127,7 +1140,7 @@ export class Upload extends Writable { // Use the upper value in this header to determine where to start the next chunk. // We should not assume that the server received all bytes sent in the request. // https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload - const range: string = resp.headers.range; + const range: string = getResponseHeader(resp.headers, 'range')!; this.offset = Number(range.split('-')[1]) + 1; // We should not assume that the server received all bytes sent in the request. @@ -1223,7 +1236,7 @@ export class Upload extends Writable { method: 'PUT', url: this.uri, headers: { - 'Content-Length': 0, + 'Content-Length': '0', 'Content-Range': 'bytes */*', 'User-Agent': getUserAgentString(), 'x-goog-api-client': googAPIClient, @@ -1264,8 +1277,9 @@ export class Upload extends Writable { const resp = await this.checkUploadStatus({retry: false}); if (resp.status === RESUMABLE_INCOMPLETE_STATUS_CODE) { - if (typeof resp.headers.range === 'string') { - this.offset = Number(resp.headers.range.split('-')[1]) + 1; + const range = getResponseHeader(resp.headers, 'range'); + if (typeof range === 'string') { + this.offset = Number(range.split('-')[1]) + 1; return; } } @@ -1288,10 +1302,12 @@ export class Upload extends Writable { private async makeRequest(reqOpts: GaxiosOptions): GaxiosPromise { if (this.encryption) { reqOpts.headers = reqOpts.headers || {}; - reqOpts.headers['x-goog-encryption-algorithm'] = 'AES256'; - reqOpts.headers['x-goog-encryption-key'] = this.encryption.key.toString(); - reqOpts.headers['x-goog-encryption-key-sha256'] = - this.encryption.hash.toString(); + if (!(reqOpts.headers instanceof Headers) && !Array.isArray(reqOpts.headers)) { + reqOpts.headers['x-goog-encryption-algorithm'] = 'AES256'; + reqOpts.headers['x-goog-encryption-key'] = this.encryption.key.toString(); + reqOpts.headers['x-goog-encryption-key-sha256'] = + this.encryption.hash.toString(); + } } if (this.userProject) { diff --git a/handwritten/storage/src/transfer-manager.ts b/handwritten/storage/src/transfer-manager.ts index 3a17e08a3fe4..0c1cf2e79ec8 100644 --- a/handwritten/storage/src/transfer-manager.ts +++ b/handwritten/storage/src/transfer-manager.ts @@ -32,7 +32,7 @@ import {GoogleAuth} from 'google-auth-library'; import {XMLParser, XMLBuilder} from 'fast-xml-parser'; import AsyncRetry from 'async-retry'; import {ApiError} from './nodejs-common/index.js'; -import {GaxiosResponse, Headers} from 'gaxios'; +import {GaxiosResponse} from 'gaxios'; import {createHash} from 'crypto'; import {GCCL_GCS_CMD_KEY} from './nodejs-common/util.js'; import {getRuntimeTrackingString, getUserAgentString} from './util.js'; @@ -220,7 +220,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { }; } - #setGoogApiClientHeaders(headers: Headers = {}): Headers { + #setGoogApiClientHeaders(headers: Record = {}): Record { let headerFound = false; let userAgentFound = false; @@ -258,11 +258,11 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { * * @returns {Promise} */ - async initiateUpload(headers: Headers = {}): Promise { + async initiateUpload(headers: Record = {}): Promise { const url = `${this.baseUrl}?uploads`; return AsyncRetry(async bail => { try { - const res = await this.authClient.request({ + const res = await this.authClient.request({ headers: this.#setGoogApiClientHeaders(headers), method: 'POST', url, @@ -294,7 +294,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { validation?: 'md5' | 'crc32c' | false, ): Promise { const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`; - let headers: Headers = this.#setGoogApiClientHeaders(); + let headers: Record = this.#setGoogApiClientHeaders(); if (validation === 'md5') { const hash = createHash('md5').update(chunk).digest('base64'); @@ -309,7 +309,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { return AsyncRetry(async bail => { try { - const res = await this.authClient.request({ + const res = await this.authClient.request({ url, method: 'PUT', body: chunk, @@ -318,7 +318,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { if (res.data && res.data.error) { throw res.data.error; } - this.partsMap.set(partNumber, res.headers['etag']); + this.partsMap.set(partNumber, res.headers.get('etag') || ''); } catch (e) { this.#handleErrorResponse(e as Error, bail); } @@ -344,7 +344,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { )}`; return AsyncRetry(async bail => { try { - const res = await this.authClient.request({ + const res = await this.authClient.request({ headers: this.#setGoogApiClientHeaders(), url, method: 'POST', @@ -371,7 +371,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { const url = `${this.baseUrl}?uploadId=${this.uploadId}`; return AsyncRetry(async bail => { try { - const res = await this.authClient.request({ + const res = await this.authClient.request({ url, method: 'DELETE', }); diff --git a/handwritten/storage/system-test/kitchen.ts b/handwritten/storage/system-test/kitchen.ts index fbfe9bd2effd..10b857b6846e 100644 --- a/handwritten/storage/system-test/kitchen.ts +++ b/handwritten/storage/system-test/kitchen.ts @@ -207,7 +207,7 @@ describe('resumable-upload', () => { }); assert.ok(!resp.data); - assert.equal(resp.headers['content-length'], '0'); + assert.equal(resp.headers.get('content-length'), '0'); }); it('should return a non-resumable failed upload', done => { diff --git a/handwritten/storage/test/nodejs-common/service.ts b/handwritten/storage/test/nodejs-common/service.ts index 502c4e5419f9..87e77133219e 100644 --- a/handwritten/storage/test/nodejs-common/service.ts +++ b/handwritten/storage/test/nodejs-common/service.ts @@ -149,7 +149,7 @@ describe('Service', () => { } async getRequestHeaders() { - return {}; + return new Headers(); } request = OAuth2Client.prototype.request.bind(this); diff --git a/handwritten/storage/test/nodejs-common/util.ts b/handwritten/storage/test/nodejs-common/util.ts index 88e6905ffb52..78fbcfbdd3d8 100644 --- a/handwritten/storage/test/nodejs-common/util.ts +++ b/handwritten/storage/test/nodejs-common/util.ts @@ -131,7 +131,7 @@ describe('common/util', () => { } async getRequestHeaders() { - return {}; + return new Headers(); } request = OAuth2Client.prototype.request.bind(this); diff --git a/handwritten/storage/test/resumable-upload.ts b/handwritten/storage/test/resumable-upload.ts index 381044d64d9d..b2fb353ee34a 100644 --- a/handwritten/storage/test/resumable-upload.ts +++ b/handwritten/storage/test/resumable-upload.ts @@ -28,6 +28,7 @@ import { MAX_RETRY_DEFAULT, RETRYABLE_ERR_FN_DEFAULT, } from '../src/storage.js'; +import {getHeader} from './test-utils.js'; import { ApiError, @@ -68,8 +69,8 @@ function mockAuthorizeRequest( access_token: 'abc123', } ) { - return nock('https://www.googleapis.com') - .post('/oauth2/v4/token') + return nock('https://oauth2.googleapis.com') + .post('/token') .reply(code, data); } @@ -1083,7 +1084,9 @@ describe('resumable-upload', () => { ); up.makeRequestStream = async (reqOpts: GaxiosOptions) => { - reqOpts.body.on('data', () => {}); + if (reqOpts.body instanceof Readable) { + reqOpts.body.on('data', () => {}); + } }; up.startUploading(); @@ -1129,13 +1132,17 @@ describe('resumable-upload', () => { let payload = Buffer.alloc(0); await new Promise(resolve => { - reqOpts.body.on('data', (data: Buffer) => { - payload = Buffer.concat([payload, data]); - }); + if (reqOpts.body instanceof Readable) { + reqOpts.body.on('data', (data: Buffer) => { + payload = Buffer.concat([payload, data]); + }); - reqOpts.body.on('end', () => { + reqOpts.body.on('end', () => { + resolve(payload); + }); + } else { resolve(payload); - }); + } }); return payload; @@ -1167,13 +1174,13 @@ describe('resumable-upload', () => { assert(reqOpts.headers); assert.equal( - reqOpts.headers['Content-Range'], + getHeader(reqOpts.headers, 'Content-Range'), `bytes ${OFFSET}-*/${CONTENT_LENGTH}` ); assert.ok( - X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + X_GOOG_API_HEADER_REGEX.test(getHeader(reqOpts.headers, 'x-goog-api-client')) ); - assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(reqOpts.headers, 'User-Agent'))); const data = await getAllDataFromRequest(); @@ -1186,11 +1193,11 @@ describe('resumable-upload', () => { await up.startUploading(); assert(reqOpts.headers); - assert.equal(reqOpts.headers['Content-Range'], 'bytes 0-*/*'); + assert.equal(getHeader(reqOpts.headers, 'Content-Range'), 'bytes 0-*/*'); assert.ok( - X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + X_GOOG_API_HEADER_REGEX.test(getHeader(reqOpts.headers, 'x-goog-api-client')) ); - assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(reqOpts.headers, 'User-Agent'))); const data = await getAllDataFromRequest(); @@ -1216,15 +1223,15 @@ describe('resumable-upload', () => { const endByte = OFFSET + CHUNK_SIZE - 1; assert(reqOpts.headers); - assert.equal(reqOpts.headers['Content-Length'], CHUNK_SIZE); + assert.equal(getHeader(reqOpts.headers, 'Content-Length'), CHUNK_SIZE); assert.equal( - reqOpts.headers['Content-Range'], + getHeader(reqOpts.headers, 'Content-Range'), `bytes ${OFFSET}-${endByte}/${CONTENT_LENGTH}` ); assert.ok( - X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + X_GOOG_API_HEADER_REGEX.test(getHeader(reqOpts.headers, 'x-goog-api-client')) ); - assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(reqOpts.headers, 'User-Agent'))); const data = await getAllDataFromRequest(); @@ -1246,17 +1253,17 @@ describe('resumable-upload', () => { assert(reqOpts.headers); assert.equal( - reqOpts.headers['Content-Length'], + getHeader(reqOpts.headers, 'Content-Length'), EXPECTED_STREAM_AMOUNT ); assert.equal( - reqOpts.headers['Content-Range'], + getHeader(reqOpts.headers, 'Content-Range'), `bytes ${OFFSET}-${ENDING_BYTE}/*` ); assert.ok( - X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + X_GOOG_API_HEADER_REGEX.test(getHeader(reqOpts.headers, 'x-goog-api-client')) ); - assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(reqOpts.headers, 'User-Agent'))); const data = await getAllDataFromRequest(); @@ -1277,17 +1284,17 @@ describe('resumable-upload', () => { const endByte = CONTENT_LENGTH - NUM_BYTES_WRITTEN + OFFSET - 1; assert(reqOpts.headers); assert.equal( - reqOpts.headers['Content-Length'], + getHeader(reqOpts.headers, 'Content-Length'), CONTENT_LENGTH - NUM_BYTES_WRITTEN ); assert.equal( - reqOpts.headers['Content-Range'], + getHeader(reqOpts.headers, 'Content-Range'), `bytes ${OFFSET}-${endByte}/${CONTENT_LENGTH}` ); assert.ok( - X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + X_GOOG_API_HEADER_REGEX.test(getHeader(reqOpts.headers, 'x-goog-api-client')) ); - assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(reqOpts.headers, 'User-Agent'))); const data = await getAllDataFromRequest(); assert.equal(data.byteLength, CONTENT_LENGTH - NUM_BYTES_WRITTEN); @@ -1397,8 +1404,12 @@ describe('resumable-upload', () => { capturedReqOpts.push(requestOptions); await new Promise(resolve => { - requestOptions.body.on('data', () => {}); - requestOptions.body.on('end', resolve); + if (requestOptions.body instanceof Readable) { + requestOptions.body.on('data', () => {}); + requestOptions.body.on('end', resolve); + } else { + resolve(); + } }); const serverCrc32c = expectedCrc32c || CALCULATED_CRC32C; @@ -1454,7 +1465,7 @@ describe('resumable-upload', () => { const reqOpts = await performUpload(up, DUMMY_CONTENT, false); assert.strictEqual(reqOpts.length, 1); assert.equal( - reqOpts[0].headers!['X-Goog-Hash'], + getHeader(reqOpts[0].headers, 'X-Goog-Hash'), `crc32c=${CALCULATED_CRC32C}` ); }); @@ -1464,7 +1475,7 @@ describe('resumable-upload', () => { const reqOpts = await performUpload(up, DUMMY_CONTENT, false); assert.strictEqual(reqOpts.length, 1); assert.equal( - reqOpts[0].headers!['X-Goog-Hash'], + getHeader(reqOpts[0].headers, 'X-Goog-Hash'), `md5=${CALCULATED_MD5}` ); }); @@ -1473,7 +1484,7 @@ describe('resumable-upload', () => { setupHashUploadInstance({crc32c: true, md5: true}); const reqOpts = await performUpload(up, DUMMY_CONTENT, false); assert.strictEqual(reqOpts.length, 1); - const xGoogHash = reqOpts[0].headers!['X-Goog-Hash']; + const xGoogHash = getHeader(reqOpts[0].headers, 'X-Goog-Hash'); assert.ok(xGoogHash); const expectedHashes = [ `crc32c=${CALCULATED_CRC32C}`, @@ -1496,7 +1507,7 @@ describe('resumable-upload', () => { ); assert.strictEqual(reqOpts.length, 1); assert.strictEqual( - reqOpts[0].headers!['X-Goog-Hash'], + getHeader(reqOpts[0].headers, 'X-Goog-Hash'), `crc32c=${customCrc32c}` ); }); @@ -1513,7 +1524,7 @@ describe('resumable-upload', () => { ); assert.strictEqual(reqOpts.length, 1); assert.strictEqual( - reqOpts[0].headers!['X-Goog-Hash'], + getHeader(reqOpts[0].headers, 'X-Goog-Hash'), `md5=${customMd5}` ); }); @@ -1522,7 +1533,7 @@ describe('resumable-upload', () => { setupHashUploadInstance({}); const reqOpts = await performUpload(up, DUMMY_CONTENT, false); assert.strictEqual(reqOpts.length, 1); - assert.strictEqual(reqOpts[0].headers!['X-Goog-Hash'], undefined); + assert.strictEqual(getHeader(reqOpts[0].headers, 'X-Goog-Hash'), undefined); }); }); @@ -1539,8 +1550,8 @@ describe('resumable-upload', () => { const reqOpts = await performUpload(up, DUMMY_CONTENT, true); assert.strictEqual(reqOpts.length, 2); - assert.strictEqual(reqOpts[0].headers!['Content-Length'], CHUNK_SIZE); - assert.strictEqual(reqOpts[0].headers!['X-Goog-Hash'], undefined); + assert.strictEqual(Number(getHeader(reqOpts[0].headers, 'Content-Length')), CHUNK_SIZE); + assert.strictEqual(getHeader(reqOpts[0].headers, 'X-Goog-Hash'), undefined); }); it('should include X-Goog-Hash header ONLY on the final multi-chunk request', async () => { @@ -1548,8 +1559,8 @@ describe('resumable-upload', () => { const reqOpts = await performUpload(up, DUMMY_CONTENT, true); assert.strictEqual(reqOpts.length, 2); - assert.strictEqual(reqOpts[1].headers!['Content-Length'], CHUNK_SIZE); - assert.equal(reqOpts[1].headers!['X-Goog-Hash'], expectedHashHeader); + assert.strictEqual(Number(getHeader(reqOpts[1].headers, 'Content-Length')), CHUNK_SIZE); + assert.equal(getHeader(reqOpts[1].headers, 'X-Goog-Hash'), expectedHashHeader); }); }); }); @@ -1840,12 +1851,12 @@ describe('resumable-upload', () => { assert.strictEqual(reqOpts.method, 'PUT'); assert.strictEqual(reqOpts.url, URI); assert(reqOpts.headers); - assert.equal(reqOpts.headers['Content-Length'], 0); - assert.equal(reqOpts.headers['Content-Range'], 'bytes */*'); + assert.equal(getHeader(reqOpts.headers, 'Content-Length'), '0'); + assert.equal(getHeader(reqOpts.headers, 'Content-Range'), 'bytes */*'); assert.ok( - X_GOOG_API_HEADER_REGEX.test(reqOpts.headers['x-goog-api-client']) + X_GOOG_API_HEADER_REGEX.test(getHeader(reqOpts.headers, 'x-goog-api-client')) ); - assert.ok(USER_AGENT_REGEX.test(reqOpts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(reqOpts.headers, 'User-Agent'))); done(); return {}; }; @@ -1900,10 +1911,10 @@ describe('resumable-upload', () => { const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); const headers = res.config.headers; - assert.strictEqual(headers['x-goog-encryption-algorithm'], 'AES256'); - assert.strictEqual(headers['x-goog-encryption-key'], up.encryption.key); + assert.strictEqual(getHeader(headers, 'x-goog-encryption-algorithm'), 'AES256'); + assert.strictEqual(getHeader(headers, 'x-goog-encryption-key'), up.encryption.key); assert.strictEqual( - headers['x-goog-encryption-key-sha256'], + getHeader(headers, 'x-goog-encryption-key-sha256'), up.encryption.hash ); }); @@ -1914,7 +1925,7 @@ describe('resumable-upload', () => { nock(REQ_OPTS.url!).get(queryPath).reply(200, {}), ]; const res: GaxiosResponse = await up.makeRequest(REQ_OPTS); - assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + assert.strictEqual(res.config.url.toString(), REQ_OPTS.url + queryPath); scopes.forEach(x => x.done()); }); @@ -1946,8 +1957,8 @@ describe('resumable-upload', () => { ]; const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); - assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); - assert.deepStrictEqual(res.headers, {}); + assert.strictEqual(res.config.url.toString(), REQ_OPTS.url + queryPath); + assert.deepStrictEqual(Array.from(res.headers.entries()), []); }); it('should bypass authentication if emulator context detected', async () => { @@ -1970,8 +1981,8 @@ describe('resumable-upload', () => { ]; const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); - assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); - assert.deepStrictEqual(res.headers, {}); + assert.strictEqual(res.config.url.toString(), REQ_OPTS.url + queryPath); + assert.deepStrictEqual(Array.from(res.headers.entries()), []); }); it('should use authentication with custom endpoint when useAuthWithCustomEndpoint is true', async () => { @@ -2004,9 +2015,9 @@ describe('resumable-upload', () => { const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); - assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + assert.strictEqual(res.config.url.toString(), REQ_OPTS.url + queryPath); // Headers should include authorization - assert.ok(res.config.headers?.['Authorization']); + assert.ok(getHeader(res.config.headers, 'Authorization')); }); it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is false', async () => { @@ -2031,9 +2042,9 @@ describe('resumable-upload', () => { ]; const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); - assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + assert.strictEqual(res.config.url.toString(), REQ_OPTS.url + queryPath); // When auth is bypassed, no auth headers should be present - assert.deepStrictEqual(res.headers, {}); + assert.deepStrictEqual(Array.from(res.headers.entries()), []); }); it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is undefined (backward compatibility)', async () => { @@ -2058,9 +2069,9 @@ describe('resumable-upload', () => { ]; const res = await up.makeRequest(REQ_OPTS); scopes.forEach(x => x.done()); - assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1)); + assert.strictEqual(res.config.url.toString(), REQ_OPTS.url + queryPath); // When auth is bypassed (backward compatibility), no auth headers should be present - assert.deepStrictEqual(res.headers, {}); + assert.deepStrictEqual(Array.from(res.headers.entries()), []); }); it('should combine customRequestOptions', done => { @@ -2077,8 +2088,7 @@ describe('resumable-upload', () => { mockAuthorizeRequest(); up.authClient = { request: (reqOpts: GaxiosOptions) => { - const customHeader = - reqOpts.headers && reqOpts.headers['X-My-Header']; + const customHeader = getHeader(reqOpts.headers, 'X-My-Header'); assert.strictEqual(customHeader, 'My custom value'); setImmediate(done); return {}; @@ -2088,13 +2098,13 @@ describe('resumable-upload', () => { }); it('should execute the callback with a body error & response', async () => { - const error = new GaxiosError('Error message', {}, { - config: {}, + const error = new GaxiosError('Error message', { headers: new Headers(), url: new URL('http://fake.local') }, { + config: { headers: new Headers(), url: new URL('http://fake.local') }, data: {}, status: 500, statusText: 'sad trombone', - headers: {}, - } as GaxiosResponse); + headers: new Headers(), + } as unknown as GaxiosResponse); mockAuthorizeRequest(); const scope = nock(REQ_OPTS.url!).get(queryPath).reply(500, {error}); await assert.rejects(up.makeRequest(REQ_OPTS), (err: GaxiosError) => { @@ -2105,13 +2115,13 @@ describe('resumable-upload', () => { }); it('should execute the callback with a body error & response for non-2xx status codes', async () => { - const error = new GaxiosError('Error message', {}, { - config: {}, + const error = new GaxiosError('Error message', { headers: new Headers(), url: new URL('http://fake.local') }, { + config: { headers: new Headers(), url: new URL('http://fake.local') }, data: {}, status: 500, statusText: 'sad trombone', - headers: {}, - } as GaxiosResponse); + headers: new Headers(), + } as unknown as GaxiosResponse); mockAuthorizeRequest(); const scope = nock(REQ_OPTS.url!).get(queryPath).reply(500, {error}); await assert.rejects(up.makeRequest(REQ_OPTS), (err: GaxiosError) => { @@ -2142,7 +2152,7 @@ describe('resumable-upload', () => { it('should pass a signal from the abort controller', done => { up.authClient = { request: (reqOpts: GaxiosOptions) => { - assert(reqOpts.signal instanceof AbortController); + assert(reqOpts.signal instanceof AbortSignal); done(); }, }; @@ -2152,11 +2162,11 @@ describe('resumable-upload', () => { it('should abort on an error', done => { up.on('error', () => {}); - let abortController: AbortController; + let abortSignal: AbortSignal; up.authClient = { request: (reqOpts: GaxiosOptions) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - abortController = reqOpts.signal as any; + abortSignal = reqOpts.signal as any; }, }; @@ -2164,7 +2174,7 @@ describe('resumable-upload', () => { up.emit('error', new Error('Error.')); setImmediate(() => { - assert.strictEqual(abortController.aborted, true); + assert.strictEqual(abortSignal.aborted, true); done(); }); }); @@ -2220,8 +2230,7 @@ describe('resumable-upload', () => { mockAuthorizeRequest(); up.authClient = { request: (reqOpts: GaxiosOptions) => { - const customHeader = - reqOpts.headers && reqOpts.headers['X-My-Header']; + const customHeader = getHeader(reqOpts.headers, 'X-My-Header'); assert.strictEqual(customHeader, 'My custom value'); setImmediate(done); return {}; @@ -2668,22 +2677,24 @@ describe('resumable-upload', () => { let chunkWritesInRequest = 0; const res = await new Promise(resolve => { - opts.body.on('data', (data: Buffer) => { - dataReceived += data.byteLength; - overallDataReceived += data.byteLength; - chunkWritesInRequest++; - }); + if (opts.body instanceof Readable) { + opts.body.on('data', (data: Buffer) => { + dataReceived += data.byteLength; + overallDataReceived += data.byteLength; + chunkWritesInRequest++; + }); - opts.body.on('end', () => { - requests.push({dataReceived, opts, chunkWritesInRequest}); + opts.body.on('end', () => { + requests.push({dataReceived, opts, chunkWritesInRequest}); - resolve({ - status: 200, - data: {}, + resolve({ + status: 200, + data: {}, + }); }); - + } else { resolve(null); - }); + } }); return res; @@ -2713,15 +2724,15 @@ describe('resumable-upload', () => { assert.equal(request.dataReceived, CONTENT_LENGTH); assert(request.opts.headers); assert.equal( - request.opts.headers['Content-Range'], + getHeader(request.opts.headers, 'Content-Range'), `bytes 0-*/${CONTENT_LENGTH}` ); assert.ok( X_GOOG_API_HEADER_REGEX.test( - request.opts.headers['x-goog-api-client'] + getHeader(request.opts.headers, 'x-goog-api-client') ) ); - assert.ok(USER_AGENT_REGEX.test(request.opts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(request.opts.headers, 'User-Agent'))); done(); }); @@ -2817,34 +2828,38 @@ describe('resumable-upload', () => { let chunkWritesInRequest = 0; const res = await new Promise(resolve => { - opts.body.on('data', (data: Buffer) => { - dataReceived += data.byteLength; - overallDataReceived += data.byteLength; - chunkWritesInRequest++; - }); - - opts.body.on('end', () => { - requests.push({dataReceived, opts, chunkWritesInRequest}); - - if (overallDataReceived < CONTENT_LENGTH) { - const lastByteReceived = overallDataReceived - ? overallDataReceived - 1 - : 0; + if (opts.body instanceof Readable) { + opts.body.on('data', (data: Buffer) => { + dataReceived += data.byteLength; + overallDataReceived += data.byteLength; + chunkWritesInRequest++; + }); - resolve({ - status: RESUMABLE_INCOMPLETE_STATUS_CODE, - headers: { - range: `bytes=0-${lastByteReceived}`, - }, - data: {}, - }); - } else { - resolve({ - status: 200, - data: {}, - }); - } - }); + opts.body.on('end', () => { + requests.push({dataReceived, opts, chunkWritesInRequest}); + + if (overallDataReceived < CONTENT_LENGTH) { + const lastByteReceived = overallDataReceived + ? overallDataReceived - 1 + : 0; + + resolve({ + status: RESUMABLE_INCOMPLETE_STATUS_CODE, + headers: { + range: `bytes=0-${lastByteReceived}`, + }, + data: {}, + }); + } else { + resolve({ + status: 200, + data: {}, + }); + } + }); + } else { + resolve(null); + } }); return res; @@ -2881,20 +2896,20 @@ describe('resumable-upload', () => { assert.equal(request.dataReceived, LAST_REQUEST_SIZE); assert(request.opts.headers); assert.equal( - request.opts.headers['Content-Length'], + getHeader(request.opts.headers, 'Content-Length'), LAST_REQUEST_SIZE ); assert.equal( - request.opts.headers['Content-Range'], + getHeader(request.opts.headers, 'Content-Range'), `bytes ${offset}-${endByte}/${CONTENT_LENGTH}` ); assert.ok( X_GOOG_API_HEADER_REGEX.test( - request.opts.headers['x-goog-api-client'] + getHeader(request.opts.headers, 'x-goog-api-client') ) ); assert.ok( - USER_AGENT_REGEX.test(request.opts.headers['User-Agent']) + USER_AGENT_REGEX.test(getHeader(request.opts.headers, 'User-Agent')) ); } else { // The preceding chunks @@ -2902,18 +2917,18 @@ describe('resumable-upload', () => { assert.equal(request.dataReceived, CHUNK_SIZE); assert(request.opts.headers); - assert.equal(request.opts.headers['Content-Length'], CHUNK_SIZE); + assert.equal(getHeader(request.opts.headers, 'Content-Length'), CHUNK_SIZE); assert.equal( - request.opts.headers['Content-Range'], + getHeader(request.opts.headers, 'Content-Range'), `bytes ${offset}-${endByte}/${CONTENT_LENGTH}` ); assert.ok( X_GOOG_API_HEADER_REGEX.test( - request.opts.headers['x-goog-api-client'] + getHeader(request.opts.headers, 'x-goog-api-client') ) ); assert.ok( - USER_AGENT_REGEX.test(request.opts.headers['User-Agent']) + USER_AGENT_REGEX.test(getHeader(request.opts.headers, 'User-Agent')) ); } } @@ -2964,22 +2979,24 @@ describe('resumable-upload', () => { let chunkWritesInRequest = 0; const res = await new Promise(resolve => { - opts.body.on('data', (data: Buffer) => { - dataReceived += data.byteLength; - overallDataReceived += data.byteLength; - chunkWritesInRequest++; - }); + if (opts.body instanceof Readable) { + opts.body.on('data', (data: Buffer) => { + dataReceived += data.byteLength; + overallDataReceived += data.byteLength; + chunkWritesInRequest++; + }); - opts.body.on('end', () => { - requests.push({dataReceived, opts, chunkWritesInRequest}); + opts.body.on('end', () => { + requests.push({dataReceived, opts, chunkWritesInRequest}); - resolve({ - status: 200, - data: {}, + resolve({ + status: 200, + data: {}, + }); }); - + } else { resolve(null); - }); + } }); return res; @@ -3005,15 +3022,15 @@ describe('resumable-upload', () => { assert(request.opts.headers); assert.equal( - request.opts.headers['Content-Range'], + getHeader(request.opts.headers, 'Content-Range'), `bytes 0-*/${CONTENT_LENGTH}` ); assert.ok( X_GOOG_API_HEADER_REGEX.test( - request.opts.headers['x-goog-api-client'] + getHeader(request.opts.headers, 'x-goog-api-client') ) ); - assert.ok(USER_AGENT_REGEX.test(request.opts.headers['User-Agent'])); + assert.ok(USER_AGENT_REGEX.test(getHeader(request.opts.headers, 'User-Agent'))); done(); }); @@ -3073,8 +3090,12 @@ describe('resumable-upload', () => { it(`should ${scenario.desc}`, done => { up.makeRequestStream = async (opts: GaxiosOptions) => { await new Promise(resolve => { - opts.body.on('data', () => {}); - opts.body.on('end', resolve); + if (opts.body instanceof Readable) { + opts.body.on('data', () => {}); + opts.body.on('end', resolve); + } else { + resolve(); + } }); return { diff --git a/handwritten/storage/test/test-utils.ts b/handwritten/storage/test/test-utils.ts new file mode 100644 index 000000000000..1e945851f11f --- /dev/null +++ b/handwritten/storage/test/test-utils.ts @@ -0,0 +1,26 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export function getHeader(headers: any, name: string): any { + if (!headers) return undefined; + if (typeof headers.get === 'function') { + return headers.get(name) === null ? undefined : (headers.get(name) || ''); + } + if (Array.isArray(headers)) { + const entry = headers.find(h => h[0].toLowerCase() === name.toLowerCase()); + return entry ? entry[1] : undefined; + } + const key = Object.keys(headers).find(k => k.toLowerCase() === name.toLowerCase()); + return key ? headers[key] : undefined; +} diff --git a/handwritten/storage/test/transfer-manager.ts b/handwritten/storage/test/transfer-manager.ts index 364618cc6f84..bc9e495acc47 100644 --- a/handwritten/storage/test/transfer-manager.ts +++ b/handwritten/storage/test/transfer-manager.ts @@ -42,6 +42,7 @@ import fs from 'fs'; import {promises as fsp, Stats} from 'fs'; import * as sinon from 'sinon'; import {DownloadResponseWithStatus, SkipReason} from '../src/file.js'; +import {getHeader} from './test-utils.js'; describe('Transfer Manager', () => { const BUCKET_NAME = 'test-bucket'; @@ -886,16 +887,25 @@ describe('Transfer Manager', () => { } async getRequestHeaders() { - return {}; + return new Headers(); } async request(opts: GaxiosOptions) { + if (opts.url && opts.url.toString().includes('oauth2.googleapis.com')) { + return { + data: { + access_token: 'abc123', + }, + headers: {}, + } as GaxiosResponse; + } + called = true; assert(opts.headers); - assert('x-goog-api-client' in opts.headers); + assert(getHeader(opts.headers, 'x-goog-api-client')); assert.match( - opts.headers['x-goog-api-client'], + getHeader(opts.headers, 'x-goog-api-client'), /gccl-gcs-cmd\/tm.upload_sharded/ ); @@ -905,7 +915,7 @@ describe('Transfer Manager', () => { 1 ` ), - headers: {}, + headers: new Headers({ etag: 'etag-val' }), } as GaxiosResponse; } } @@ -927,15 +937,24 @@ describe('Transfer Manager', () => { } async getRequestHeaders() { - return {}; + return new Headers(); } async request(opts: GaxiosOptions) { + if (opts.url && opts.url.toString().includes('oauth2.googleapis.com')) { + return { + data: { + access_token: 'abc123', + }, + headers: {}, + } as GaxiosResponse; + } + called = true; assert(opts.headers); - assert('User-Agent' in opts.headers); - assert.match(opts.headers['User-Agent'], /gcloud-node/); + assert(getHeader(opts.headers, 'User-Agent')); + assert.match(getHeader(opts.headers, 'User-Agent'), /gcloud-node/); return { data: Buffer.from( @@ -943,7 +962,7 @@ describe('Transfer Manager', () => { 1 ` ), - headers: {}, + headers: new Headers({ etag: 'etag-val' }), } as GaxiosResponse; } }