From 564dccd3204e6cea410f847d0e557313e6ded248 Mon Sep 17 00:00:00 2001 From: ruby Date: Tue, 17 Mar 2026 04:15:20 +0400 Subject: [PATCH 01/15] feat: hooks --- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/workflows/test.yml | 3 + .oxfmtrc.json | 19 + .oxlintrc.json | 3 + .prettierignore | 22 - .prettierrc | 10 - LICENSE | 2 +- package-lock.json | 744 +++++++++++++++++++++++++++++- package.json | 17 +- src/client.ts | 83 ++++ src/errors.ts | 35 ++ src/fetch.ts | 246 ++++++++++ src/headers.ts | 139 ++++++ src/hooks.ts | 76 +++ src/native.ts | 91 ++++ src/node-wreq.ts | 224 +++------ src/response.ts | 39 ++ src/test/node-wreq.spec.ts | 151 +++++- src/types.ts | 199 +++++--- 19 files changed, 1799 insertions(+), 306 deletions(-) create mode 100644 .oxfmtrc.json create mode 100644 .oxlintrc.json delete mode 100644 .prettierignore delete mode 100644 .prettierrc create mode 100644 src/client.ts create mode 100644 src/errors.ts create mode 100644 src/fetch.ts create mode 100644 src/headers.ts create mode 100644 src/hooks.ts create mode 100644 src/native.ts create mode 100644 src/response.ts diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 49df015..3e8b260 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: 📚 Documentation - url: https://github.com/will-work-for-meal/node-wreq#readme + url: https://github.com/StopMakingThatBigFace/node-wreq#readme about: Read the documentation and guides diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da95a25..096079f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,5 +82,8 @@ jobs: - name: Check TypeScript types run: npx tsc --noEmit + - name: Run oxlint + run: npm run lint + - name: Check code formatting run: npm run format:check diff --git a/.oxfmtrc.json b/.oxfmtrc.json new file mode 100644 index 0000000..26d3627 --- /dev/null +++ b/.oxfmtrc.json @@ -0,0 +1,19 @@ +{ + "$schema": "./node_modules/oxfmt/configuration_schema.json", + "sortPackageJson": false, + "sortImports": { + "sortSideEffects": true, + "newlinesBetween": false + }, + "ignorePatterns": [ + "**/dist/**", + "**/node_modules/**", + ], + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "bracketSpacing": true, + "tabWidth": 2, + "printWidth": 100, + "endOfLine": "lf" +} \ No newline at end of file diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 0000000..44d3cf4 --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,3 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json" +} \ No newline at end of file diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index e578bea..0000000 --- a/.prettierignore +++ /dev/null @@ -1,22 +0,0 @@ -# Dependencies -node_modules -package-lock.json -yarn.lock -pnpm-lock.yaml - -# Build output -dist -rust/target -*.node - -# Cache -.cache -*.log - -# IDE -.vscode -.idea - -# Misc -coverage -.DS_Store diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index 32a2397..0000000 --- a/.prettierrc +++ /dev/null @@ -1,10 +0,0 @@ -{ - "semi": true, - "trailingComma": "es5", - "singleQuote": true, - "printWidth": 100, - "tabWidth": 2, - "useTabs": false, - "arrowParens": "always", - "endOfLine": "lf" -} diff --git a/LICENSE b/LICENSE index 4e8bd1e..b88a2d2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 will-work-for-meal +Copyright (c) 2025 StopMakingThatBigFace Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/package-lock.json b/package-lock.json index a08b9be..ba6b2c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "node-wreq", - "version": "0.1.0", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "node-wreq", - "version": "0.1.0", + "version": "0.1.2", "cpu": [ "x64", "arm64" @@ -20,7 +20,8 @@ "devDependencies": { "@napi-rs/cli": "^2.18.0", "@types/node": "^20.0.0", - "prettier": "^3.2.5", + "oxfmt": "^0.41.0", + "oxlint": "^1.56.0", "typescript": "^5.0.0" }, "engines": { @@ -42,6 +43,652 @@ "url": "https://github.com/sponsors/Brooooooklyn" } }, + "node_modules/@oxfmt/binding-android-arm-eabi": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.41.0.tgz", + "integrity": "sha512-REfrqeMKGkfMP+m/ScX4f5jJBSmVNYcpoDF8vP8f8eYPDuPGZmzp56NIUsYmx3h7f6NzC6cE3gqh8GDWrJHCKw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-android-arm64": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.41.0.tgz", + "integrity": "sha512-s0b1dxNgb2KomspFV2LfogC2XtSJB42POXF4bMCLJyvQmAGos4ZtjGPfQreToQEaY0FQFjz3030ggI36rF1q5g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-darwin-arm64": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.41.0.tgz", + "integrity": "sha512-EGXGualADbv/ZmamE7/2DbsrYmjoPlAmHEpTL4vapLF4EfVD6fr8/uQDFnPJkUBjiSWFJZtFNsGeN1B6V3owmA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-darwin-x64": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.41.0.tgz", + "integrity": "sha512-WxySJEvdQQYMmyvISH3qDpTvoS0ebnIP63IMxLLWowJyPp/AAH0hdWtlo+iGNK5y3eVfa5jZguwNaQkDKWpGSw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-freebsd-x64": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.41.0.tgz", + "integrity": "sha512-Y2kzMkv3U3oyuYaR4wTfGjOTYTXiFC/hXmG0yVASKkbh02BJkvD98Ij8bIevr45hNZ0DmZEgqiXF+9buD4yMYQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm-gnueabihf": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.41.0.tgz", + "integrity": "sha512-ptazDjdUyhket01IjPTT6ULS1KFuBfTUU97osTP96X5y/0oso+AgAaJzuH81oP0+XXyrWIHbRzozSAuQm4p48g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm-musleabihf": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.41.0.tgz", + "integrity": "sha512-UkoL2OKxFD+56bPEBcdGn+4juTW4HRv/T6w1dIDLnvKKWr6DbarB/mtHXlADKlFiJubJz8pRkttOR7qjYR6lTA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm64-gnu": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.41.0.tgz", + "integrity": "sha512-gofu0PuumSOHYczD8p62CPY4UF6ee+rSLZJdUXkpwxg6pILiwSDBIouPskjF/5nF3A7QZTz2O9KFNkNxxFN9tA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm64-musl": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.41.0.tgz", + "integrity": "sha512-VfVZxL0+6RU86T8F8vKiDBa+iHsr8PAjQmKGBzSCAX70b6x+UOMFl+2dNihmKmUwqkCazCPfYjt6SuAPOeQJ3g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-ppc64-gnu": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.41.0.tgz", + "integrity": "sha512-bwzokz2eGvdfJbc0i+zXMJ4BBjQPqg13jyWpEEZDOrBCQ91r8KeY2Mi2kUeuMTZNFXju+jcAbAbpyJxRGla0eg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-riscv64-gnu": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.41.0.tgz", + "integrity": "sha512-POLM//PCH9uqDeNDwWL3b3DkMmI3oI2cU6hwc2lnztD1o7dzrQs3R9nq555BZ6wI7t2lyhT9CS+CRaz5X0XqLA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-riscv64-musl": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.41.0.tgz", + "integrity": "sha512-NNK7PzhFqLUwx/G12Xtm6scGv7UITvyGdAR5Y+TlqsG+essnuRWR4jRNODWRjzLZod0T3SayRbnkSIWMBov33w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-s390x-gnu": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.41.0.tgz", + "integrity": "sha512-qVf/zDC5cN9eKe4qI/O/m445er1IRl6swsSl7jHkqmOSVfknwCe5JXitYjZca+V/cNJSU/xPlC5EFMabMMFDpw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-x64-gnu": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.41.0.tgz", + "integrity": "sha512-ojxYWu7vUb6ysYqVCPHuAPVZHAI40gfZ0PDtZAMwVmh2f0V8ExpPIKoAKr7/8sNbAXJBBpZhs2coypIo2jJX4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-x64-musl": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.41.0.tgz", + "integrity": "sha512-O2exZLBxoCMIv2vlvcbkdedazJPTdG0VSup+0QUCfYQtx751zCZNboX2ZUOiQ/gDTdhtXvSiot0h6GEGkOyalA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-openharmony-arm64": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.41.0.tgz", + "integrity": "sha512-N+31/VoL+z+NNBt8viy3I4NaIdPbiYeOnB884LKqvXldaE2dRztdPv3q5ipfZYv0RwFp7JfqS4I27K/DSHCakg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-win32-arm64-msvc": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.41.0.tgz", + "integrity": "sha512-Z7NAtu/RN8kjCQ1y5oDD0nTAeRswh3GJ93qwcW51srmidP7XPBmZbLlwERu1W5veCevQJtPS9xmkpcDTYsGIwQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-win32-ia32-msvc": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.41.0.tgz", + "integrity": "sha512-uNxxP3l4bJ6VyzIeRqCmBU2Q0SkCFgIhvx9/9dJ9V8t/v+jP1IBsuaLwCXGR8JPHtkj4tFp+RHtUmU2ZYAUpMA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-win32-x64-msvc": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.41.0.tgz", + "integrity": "sha512-49ZSpbZ1noozyPapE8SUOSm3IN0Ze4b5nkO+4+7fq6oEYQQJFhE0saj5k/Gg4oewVPdjn0L3ZFeWk2Vehjcw7A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.56.0.tgz", + "integrity": "sha512-IyfYPthZyiSKwAv/dLjeO18SaK8MxLI9Yss2JrRDyweQAkuL3LhEy7pwIwI7uA3KQc1Vdn20kdmj3q0oUIQL6A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.56.0.tgz", + "integrity": "sha512-Ga5zYrzH6vc/VFxhn6MmyUnYEfy9vRpwTIks99mY3j6Nz30yYpIkWryI0QKPCgvGUtDSXVLEaMum5nA+WrNOSg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.56.0.tgz", + "integrity": "sha512-ogmbdJysnw/D4bDcpf1sPLpFThZ48lYp4aKYm10Z/6Nh1SON6NtnNhTNOlhEY296tDFItsZUz+2tgcSYqh8Eyw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.56.0.tgz", + "integrity": "sha512-x8QE1h+RAtQ2g+3KPsP6Fk/tdz6zJQUv5c7fTrJxXV3GHOo+Ry5p/PsogU4U+iUZg0rj6hS+E4xi+mnwwlDCWQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.56.0.tgz", + "integrity": "sha512-6G+WMZvwJpMvY7my+/SHEjb7BTk/PFbePqLpmVmUJRIsJMy/UlyYqjpuh0RCgYYkPLcnXm1rUM04kbTk8yS1Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.56.0.tgz", + "integrity": "sha512-YYHBsk/sl7fYwQOok+6W5lBPeUEvisznV/HZD2IfZmF3Bns6cPC3Z0vCtSEOaAWTjYWN3jVsdu55jMxKlsdlhg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.56.0.tgz", + "integrity": "sha512-+AZK8rOUr78y8WT6XkDb04IbMRqauNV+vgT6f8ZLOH8wnpQ9i7Nol0XLxAu+Cq7Sb+J9wC0j6Km5hG8rj47/yQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.56.0.tgz", + "integrity": "sha512-urse2SnugwJRojUkGSSeH2LPMaje5Q50yQtvtL9HFckiyeqXzoFwOAZqD5TR29R2lq7UHidfFDM9EGcchcbb8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.56.0.tgz", + "integrity": "sha512-rkTZkBfJ4TYLjansjSzL6mgZOdN5IvUnSq3oNJSLwBcNvy3dlgQtpHPrRxrCEbbcp7oQ6If0tkNaqfOsphYZ9g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.56.0.tgz", + "integrity": "sha512-uqL1kMH3u69/e1CH2EJhP3CP28jw2ExLsku4o8RVAZ7fySo9zOyI2fy9pVlTAp4voBLVgzndXi3SgtdyCTa2aA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.56.0.tgz", + "integrity": "sha512-j0CcMBOgV6KsRaBdsebIeiy7hCjEvq2KdEsiULf2LZqAq0v1M1lWjelhCV57LxsqaIGChXFuFJ0RiFrSRHPhSg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.56.0.tgz", + "integrity": "sha512-7VDOiL8cDG3DQ/CY3yKjbV1c4YPvc4vH8qW09Vv+5ukq3l/Kcyr6XGCd5NvxUmxqDb2vjMpM+eW/4JrEEsUetA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.56.0.tgz", + "integrity": "sha512-JGRpX0M+ikD3WpwJ7vKcHKV6Kg0dT52BW2Eu2BupXotYeqGXBrbY+QPkAyKO6MNgKozyTNaRh3r7g+VWgyAQYQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.56.0.tgz", + "integrity": "sha512-dNaICPvtmuxFP/VbqdofrLqdS3bM/AKJN3LMJD52si44ea7Be1cBk6NpfIahaysG9Uo+L98QKddU9CD5L8UHnQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.56.0.tgz", + "integrity": "sha512-pF1vOtM+GuXmbklM1hV8WMsn6tCNPvkUzklj/Ej98JhlanbmA2RB1BILgOpwSuCTRTIYx2MXssmEyQQ90QF5aA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.56.0.tgz", + "integrity": "sha512-bp8NQ4RE6fDIFLa4bdBiOA+TAvkNkg+rslR+AvvjlLTYXLy9/uKAYLQudaQouWihLD/hgkrXIKKzXi5IXOewwg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.56.0.tgz", + "integrity": "sha512-PxT4OJDfMOQBzo3OlzFb9gkoSD+n8qSBxyVq2wQSZIHFQYGEqIRTo9M0ZStvZm5fdhMqaVYpOnJvH2hUMEDk/g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.56.0.tgz", + "integrity": "sha512-PTRy6sIEPqy2x8PTP1baBNReN/BNEFmde0L+mYeHmjXE1Vlcc9+I5nsqENsB2yAm5wLkzPoTNCMY/7AnabT4/A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.56.0.tgz", + "integrity": "sha512-ZHa0clocjLmIDr+1LwoWtxRcoYniAvERotvwKUYKhH41NVfl0Y4LNbyQkwMZzwDvKklKGvGZ5+DAG58/Ik47tQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, "node_modules/@types/node": { "version": "20.19.21", "dev": true, @@ -50,18 +697,99 @@ "undici-types": "~6.21.0" } }, - "node_modules/prettier": { - "version": "3.6.2", + "node_modules/oxfmt": { + "version": "0.41.0", + "resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.41.0.tgz", + "integrity": "sha512-sKLdJZdQ3bw6x9qKiT7+eID4MNEXlDHf5ZacfIircrq6Qwjk0L6t2/JQlZZrVHTXJawK3KaMuBoJnEJPcqCEdg==", "dev": true, "license": "MIT", + "dependencies": { + "tinypool": "2.1.0" + }, "bin": { - "prettier": "bin/prettier.cjs" + "oxfmt": "bin/oxfmt" }, "engines": { - "node": ">=14" + "node": "^20.19.0 || >=22.12.0" }, "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxfmt/binding-android-arm-eabi": "0.41.0", + "@oxfmt/binding-android-arm64": "0.41.0", + "@oxfmt/binding-darwin-arm64": "0.41.0", + "@oxfmt/binding-darwin-x64": "0.41.0", + "@oxfmt/binding-freebsd-x64": "0.41.0", + "@oxfmt/binding-linux-arm-gnueabihf": "0.41.0", + "@oxfmt/binding-linux-arm-musleabihf": "0.41.0", + "@oxfmt/binding-linux-arm64-gnu": "0.41.0", + "@oxfmt/binding-linux-arm64-musl": "0.41.0", + "@oxfmt/binding-linux-ppc64-gnu": "0.41.0", + "@oxfmt/binding-linux-riscv64-gnu": "0.41.0", + "@oxfmt/binding-linux-riscv64-musl": "0.41.0", + "@oxfmt/binding-linux-s390x-gnu": "0.41.0", + "@oxfmt/binding-linux-x64-gnu": "0.41.0", + "@oxfmt/binding-linux-x64-musl": "0.41.0", + "@oxfmt/binding-openharmony-arm64": "0.41.0", + "@oxfmt/binding-win32-arm64-msvc": "0.41.0", + "@oxfmt/binding-win32-ia32-msvc": "0.41.0", + "@oxfmt/binding-win32-x64-msvc": "0.41.0" + } + }, + "node_modules/oxlint": { + "version": "1.56.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.56.0.tgz", + "integrity": "sha512-Q+5Mj5PVaH/R6/fhMMFzw4dT+KPB+kQW4kaL8FOIq7tfhlnEVp6+3lcWqFruuTNlUo9srZUW3qH7Id4pskeR6g==", + "dev": true, + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.56.0", + "@oxlint/binding-android-arm64": "1.56.0", + "@oxlint/binding-darwin-arm64": "1.56.0", + "@oxlint/binding-darwin-x64": "1.56.0", + "@oxlint/binding-freebsd-x64": "1.56.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.56.0", + "@oxlint/binding-linux-arm-musleabihf": "1.56.0", + "@oxlint/binding-linux-arm64-gnu": "1.56.0", + "@oxlint/binding-linux-arm64-musl": "1.56.0", + "@oxlint/binding-linux-ppc64-gnu": "1.56.0", + "@oxlint/binding-linux-riscv64-gnu": "1.56.0", + "@oxlint/binding-linux-riscv64-musl": "1.56.0", + "@oxlint/binding-linux-s390x-gnu": "1.56.0", + "@oxlint/binding-linux-x64-gnu": "1.56.0", + "@oxlint/binding-linux-x64-musl": "1.56.0", + "@oxlint/binding-openharmony-arm64": "1.56.0", + "@oxlint/binding-win32-arm64-msvc": "1.56.0", + "@oxlint/binding-win32-ia32-msvc": "1.56.0", + "@oxlint/binding-win32-x64-msvc": "1.56.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=0.15.0" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + } + } + }, + "node_modules/tinypool": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-2.1.0.tgz", + "integrity": "sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.0.0 || >=22.0.0" } }, "node_modules/typescript": { diff --git a/package.json b/package.json index 733459b..a4fa47d 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,10 @@ "artifacts": "napi artifacts", "clean": "rm -rf dist rust/target rust/*.node", "test": "npm run build && node --test dist/test/node-wreq.spec.js", - "format": "prettier --write \"src/**/*.{ts,js,json,md}\"", - "format:check": "prettier --check \"src/**/*.{ts,js,json,md}\"" + "lint": "oxlint --deny-warnings src", + "lint:fix": "oxlint --fix src", + "format": "oxfmt --write \"src/**/*.ts\"", + "format:check": "oxfmt --check \"src/**/*.ts\"" }, "keywords": [ "browser", @@ -24,22 +26,23 @@ "impersonation", "web-scraping" ], - "author": "will-work-for-meal", + "author": "StopMakingThatBigFace", "license": "MIT", "repository": { "type": "git", - "url": "git+https://github.com/will-work-for-meal/node-wreq.git" + "url": "git+https://github.com/StopMakingThatBigFace/node-wreq.git" }, "bugs": { - "url": "https://github.com/will-work-for-meal/node-wreq/issues" + "url": "https://github.com/StopMakingThatBigFace/node-wreq/issues" }, - "homepage": "https://github.com/will-work-for-meal/node-wreq#readme", + "homepage": "https://github.com/StopMakingThatBigFace/node-wreq#readme", "dependencies": {}, "devDependencies": { "@types/node": "^20.0.0", "typescript": "^5.0.0", "@napi-rs/cli": "^2.18.0", - "prettier": "^3.2.5" + "oxfmt": "^0.41.0", + "oxlint": "^1.56.0" }, "engines": { "node": ">=18.0.0" diff --git a/src/client.ts b/src/client.ts new file mode 100644 index 0000000..cb28d17 --- /dev/null +++ b/src/client.ts @@ -0,0 +1,83 @@ +import { Headers } from './headers'; +import { fetch } from './fetch'; +import { mergeHooks } from './hooks'; +import type { + Client, + ClientDefaults, + HeaderTuple, + HeadersInit, + RequestInput, + WreqInit, +} from './types'; + +function mergeHeaders(...sources: Array): HeaderTuple[] | undefined { + const merged = new Headers(); + + for (const source of sources) { + if (!source) { + continue; + } + + const headers = source instanceof Headers ? source : new Headers(source); + for (const [name, value] of headers) { + merged.set(name, value); + } + } + + const tuples = merged.toTuples(); + return tuples.length > 0 ? tuples : undefined; +} + +function mergeDefaults(base: ClientDefaults, override?: ClientDefaults): ClientDefaults { + if (!override) { + return { ...base }; + } + + return { + ...base, + ...override, + headers: mergeHeaders(base.headers, override.headers), + hooks: mergeHooks(base.hooks, override.hooks), + }; +} + +class WreqClient implements Client { + readonly defaults: ClientDefaults; + + constructor(defaults: ClientDefaults = {}) { + this.defaults = { + ...defaults, + headers: defaults.headers ? mergeHeaders(defaults.headers) : undefined, + }; + } + + async fetch(input: RequestInput, init?: WreqInit) { + const merged: WreqInit = { + ...this.defaults, + ...init, + headers: mergeHeaders(this.defaults.headers, init?.headers), + hooks: mergeHooks(this.defaults.hooks, init?.hooks), + }; + return fetch(input, merged); + } + + async get(input: RequestInput, init?: Omit) { + return this.fetch(input, { ...init, method: 'GET' }); + } + + async post( + input: RequestInput, + body?: WreqInit['body'], + init?: Omit, + ) { + return this.fetch(input, { ...init, method: 'POST', body }); + } + + extend(defaults: ClientDefaults): Client { + return new WreqClient(mergeDefaults(this.defaults, defaults)); + } +} + +export function createClient(defaults: ClientDefaults = {}): Client { + return new WreqClient(defaults); +} diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..eb771a1 --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,35 @@ +export class RequestError extends Error { + code?: string; + cause?: unknown; + + constructor(message: string, options?: { code?: string; cause?: unknown }) { + super(message); + this.name = 'RequestError'; + this.code = options?.code; + this.cause = options?.cause; + } +} + +export class HTTPError extends RequestError { + status: number; + + constructor(message: string, status: number, options?: { cause?: unknown }) { + super(message, { code: 'ERR_HTTP_STATUS', cause: options?.cause }); + this.name = 'HTTPError'; + this.status = status; + } +} + +export class TimeoutError extends RequestError { + constructor(message = 'Request timed out', options?: { cause?: unknown }) { + super(message, { code: 'ERR_TIMEOUT', cause: options?.cause }); + this.name = 'TimeoutError'; + } +} + +export class AbortError extends RequestError { + constructor(message = 'The operation was aborted', options?: { cause?: unknown }) { + super(message, { code: 'ERR_ABORTED', cause: options?.cause }); + this.name = 'AbortError'; + } +} diff --git a/src/fetch.ts b/src/fetch.ts new file mode 100644 index 0000000..9dff33e --- /dev/null +++ b/src/fetch.ts @@ -0,0 +1,246 @@ +import { Buffer } from 'node:buffer'; +import { HTTPError, RequestError } from './errors'; +import { Headers } from './headers'; +import { + runAfterResponseHooks, + runBeforeErrorHooks, + runBeforeRequestHooks, + runInitHooks, +} from './hooks'; +import { nativeRequest, normalizeMethod, validateBrowserProfile } from './native'; +import { Response } from './response'; +import type { + HeadersInit, + NativeRequestOptions, + NormalizedRequest, + RequestInput, + RequestLike, + ResolvedOptions, + WreqInit, +} from './types'; + +function isRequestLike(value: unknown): value is RequestLike { + return typeof value === 'object' && value !== null && 'url' in value; +} + +function isResponseStatusAllowed( + status: number, + options: Pick, +): boolean { + if (options.validateStatus) { + return options.validateStatus(status); + } + + if (options.throwHttpErrors) { + return status >= 200 && status < 300; + } + + return true; +} + +function appendQuery(url: URL, query: WreqInit['query']): void { + if (!query) { + return; + } + + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) { + continue; + } + url.searchParams.set(key, String(value)); + } +} + +function resolveUrl(rawUrl: string | URL, baseURL?: string, query?: WreqInit['query']): string { + const url = baseURL ? new URL(String(rawUrl), baseURL) : new URL(String(rawUrl)); + appendQuery(url, query); + return url.toString(); +} + +async function normalizeBody(body: WreqInit['body']): Promise { + if (body === undefined || body === null) { + return undefined; + } + + if (typeof body === 'string') { + return body; + } + + if (body instanceof URLSearchParams) { + return body.toString(); + } + + if (Buffer.isBuffer(body)) { + return body.toString('utf8'); + } + + if (ArrayBuffer.isView(body)) { + return Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString('utf8'); + } + + if (body instanceof ArrayBuffer) { + return Buffer.from(body).toString('utf8'); + } + + throw new RequestError('Unsupported body type'); +} + +async function mergeInputAndInit( + input: RequestInput, + init?: WreqInit, +): Promise<{ + urlInput: string | URL; + init: WreqInit; +}> { + let urlInput: string | URL; + let mergedInit: WreqInit = { ...init }; + + if (typeof input === 'string' || input instanceof URL) { + urlInput = input; + } else if (isRequestLike(input)) { + urlInput = input.url; + const inputHeaders = input.headers as HeadersInit | undefined; + const inputBody = input.body as WreqInit['body']; + mergedInit = { + ...init, + method: init?.method ?? input.method, + headers: init?.headers ?? inputHeaders, + body: init?.body ?? inputBody, + signal: init?.signal ?? input.signal ?? undefined, + }; + + if (mergedInit.body === undefined && typeof input.arrayBuffer === 'function') { + if (input.bodyUsed) { + throw new TypeError('Request body is already used'); + } + const arrayBuffer = await input.arrayBuffer(); + mergedInit.body = arrayBuffer; + } + } else { + throw new RequestError('URL is required'); + } + + return { urlInput, init: mergedInit }; +} + +async function normalizeInput( + urlInput: string | URL, + init: WreqInit, +): Promise<{ + nativeRequest: NativeRequestOptions; + request: NormalizedRequest; + options: ResolvedOptions; +}> { + if (init.cookieJar) { + throw new RequestError('cookieJar support is not implemented yet'); + } + + const method = normalizeMethod(init.method); + validateBrowserProfile(init.browser); + + const headers = new Headers(init.headers); + const request: NormalizedRequest = { + url: resolveUrl(urlInput, init.baseURL, init.query), + method, + headers, + body: init.body, + }; + + const options: ResolvedOptions = { + ...init, + headers, + }; + + const nativeRequest: NativeRequestOptions = { + url: request.url, + method: request.method, + headers: request.headers.toObject(), + body: await normalizeBody(request.body), + browser: init.browser, + proxy: init.proxy, + timeout: init.timeout, + }; + + return { nativeRequest, request, options }; +} + +function isError(value: unknown): value is Error { + return value instanceof Error; +} + +function normalizeRequestError(error: unknown): Error { + if (isError(error)) { + return error; + } + + return new RequestError(String(error), { cause: error }); +} + +async function dispatchNativeRequest(options: NativeRequestOptions): Promise { + const nativeResponse = await nativeRequest(options).catch((error: unknown) => { + const message = String(error); + if (message.toLowerCase().includes('timed out')) { + throw new RequestError(message, { code: 'ERR_TIMEOUT', cause: error }); + } + throw new RequestError(message, { cause: error }); + }); + + return new Response(nativeResponse); +} + +export async function fetch(input: RequestInput, init?: WreqInit): Promise { + const state = (init?.context ? { ...init.context } : {}) as Record; + const merged = await mergeInputAndInit(input, init); + await runInitHooks(merged.init.hooks, { + input, + options: merged.init, + state, + }); + + const { nativeRequest, options, request } = await normalizeInput(merged.urlInput, merged.init); + + try { + const shortCircuit = await runBeforeRequestHooks(options.hooks, { + request, + options, + attempt: 1, + state, + }); + + let response = + shortCircuit ?? + (await dispatchNativeRequest({ + ...nativeRequest, + url: request.url, + method: request.method, + headers: request.headers.toObject(), + body: await normalizeBody(request.body), + proxy: options.proxy, + browser: options.browser, + timeout: options.timeout, + })); + + response = await runAfterResponseHooks(options.hooks, { + request, + options, + attempt: 1, + state, + response, + }); + + if (!isResponseStatusAllowed(response.status, options)) { + throw new HTTPError(`Request failed with status ${response.status}`, response.status); + } + + return response; + } catch (error: unknown) { + const normalizedError = await runBeforeErrorHooks(options.hooks, { + request, + options, + attempt: 1, + state, + error: normalizeRequestError(error), + }); + throw normalizedError; + } +} diff --git a/src/headers.ts b/src/headers.ts new file mode 100644 index 0000000..073c8f6 --- /dev/null +++ b/src/headers.ts @@ -0,0 +1,139 @@ +import type { HeaderTuple, HeadersInit } from './types'; + +type HeaderEntry = { + name: string; + values: string[]; +}; + +function isIterable(value: unknown): value is Iterable { + return Boolean(value) && typeof (value as Iterable)[Symbol.iterator] === 'function'; +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null) { + return false; + } + + const prototype = Object.getPrototypeOf(value); + + return prototype === Object.prototype || prototype === null; +} + +export class Headers implements Iterable { + private readonly store = new Map(); + + constructor(init?: HeadersInit) { + if (!init) { + return; + } + + if (init instanceof Headers) { + for (const [name, value] of init) { + this.append(name, value); + } + + return; + } + + if (Array.isArray(init) || isIterable(init)) { + for (const [name, value] of init as Iterable) { + this.append(name, value); + } + + return; + } + + if (isPlainObject(init)) { + for (const [name, value] of Object.entries(init)) { + if (value === undefined || value === null) { + continue; + } + + this.set(name, String(value)); + } + } + } + + private normalizeName(name: string): { key: string; display: string } { + if (typeof name !== 'string') { + throw new TypeError('Header name must be a string'); + } + + const trimmed = name.trim(); + + if (!trimmed) { + throw new TypeError('Header name must not be empty'); + } + + return { key: trimmed.toLowerCase(), display: trimmed }; + } + + append(name: string, value: unknown): void { + const normalized = this.normalizeName(name); + const entry = this.store.get(normalized.key); + const stringValue = String(value); + + if (entry) { + entry.values.push(stringValue); + + return; + } + + this.store.set(normalized.key, { + name: normalized.display, + values: [stringValue], + }); + } + + set(name: string, value: unknown): void { + const normalized = this.normalizeName(name); + + this.store.set(normalized.key, { + name: normalized.display, + values: [String(value)], + }); + } + + get(name: string): string | null { + const normalized = this.normalizeName(name); + const entry = this.store.get(normalized.key); + + return entry ? entry.values.join(', ') : null; + } + + has(name: string): boolean { + const normalized = this.normalizeName(name); + + return this.store.has(normalized.key); + } + + delete(name: string): void { + const normalized = this.normalizeName(name); + + this.store.delete(normalized.key); + } + + toObject(): Record { + const result: Record = {}; + + for (const [name, value] of this) { + result[name] = value; + } + + return result; + } + + toTuples(): HeaderTuple[] { + return [...this]; + } + + *entries(): IterableIterator { + for (const entry of this.store.values()) { + yield [entry.name, entry.values.join(', ')]; + } + } + + [Symbol.iterator](): IterableIterator { + return this.entries(); + } +} diff --git a/src/hooks.ts b/src/hooks.ts new file mode 100644 index 0000000..3f93569 --- /dev/null +++ b/src/hooks.ts @@ -0,0 +1,76 @@ +import type { + AfterResponseContext, + BeforeErrorContext, + BeforeRequestContext, + Hooks, + InitContext, +} from './types'; +import type { Response } from './response'; + +export function mergeHooks(base?: Hooks, override?: Hooks): Hooks | undefined { + if (!base && !override) { + return undefined; + } + + return { + init: [...(base?.init ?? []), ...(override?.init ?? [])], + beforeRequest: [...(base?.beforeRequest ?? []), ...(override?.beforeRequest ?? [])], + afterResponse: [...(base?.afterResponse ?? []), ...(override?.afterResponse ?? [])], + beforeRetry: [...(base?.beforeRetry ?? []), ...(override?.beforeRetry ?? [])], + beforeError: [...(base?.beforeError ?? []), ...(override?.beforeError ?? [])], + }; +} + +export async function runInitHooks(hooks: Hooks | undefined, context: InitContext): Promise { + for (const hook of hooks?.init ?? []) { + await hook(context); + } +} + +export async function runBeforeRequestHooks( + hooks: Hooks | undefined, + context: BeforeRequestContext, +): Promise { + for (const hook of hooks?.beforeRequest ?? []) { + const result = await hook(context); + + if (result) { + return result; + } + } + + return undefined; +} + +export async function runAfterResponseHooks( + hooks: Hooks | undefined, + context: AfterResponseContext, +): Promise { + let current = context.response; + + for (const hook of hooks?.afterResponse ?? []) { + const result = await hook({ ...context, response: current }); + + if (result) { + current = result; + } + } + + return current; +} + +export async function runBeforeErrorHooks( + hooks: Hooks | undefined, + context: BeforeErrorContext, +): Promise { + let current = context.error; + + for (const hook of hooks?.beforeError ?? []) { + const result = await hook({ ...context, error: current }); + if (result) { + current = result; + } + } + + return current; +} diff --git a/src/native.ts b/src/native.ts new file mode 100644 index 0000000..5009b86 --- /dev/null +++ b/src/native.ts @@ -0,0 +1,91 @@ +import type { BrowserProfile, HttpMethod, NativeResponse, NativeRequestOptions } from './types'; + +type NativeBinding = { + request: (options: NativeRequestOptions) => Promise; + getProfiles: () => string[]; +}; + +let nativeBinding: NativeBinding | undefined; + +function loadNativeBinding(): NativeBinding { + const platform = process.platform; + const arch = process.arch; + + const platformArchMap: Record> = { + darwin: { + x64: 'darwin-x64', + arm64: 'darwin-arm64', + }, + linux: { + x64: 'linux-x64-gnu', + }, + win32: { + x64: 'win32-x64-msvc', + }, + }; + + const platformArch = platformArchMap[platform]?.[arch]; + if (!platformArch) { + throw new Error( + `Unsupported platform: ${platform}-${arch}. ` + + `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64`, + ); + } + + const binaryName = `node-wreq.${platformArch}.node`; + + try { + return require(`../rust/${binaryName}`) as NativeBinding; + } catch { + try { + return require('../rust/node-wreq.node') as NativeBinding; + } catch { + throw new Error( + `Failed to load native module for ${platform}-${arch}. ` + + `Tried: ../rust/${binaryName} and ../rust/node-wreq.node. ` + + `Make sure the package is installed correctly and the native module is built for your platform.`, + ); + } + } +} + +function getBinding(): NativeBinding { + nativeBinding ??= loadNativeBinding(); + return nativeBinding; +} + +let cachedProfiles: BrowserProfile[] | undefined; + +export function getProfiles(): BrowserProfile[] { + cachedProfiles ??= getBinding().getProfiles() as BrowserProfile[]; + return cachedProfiles; +} + +export async function nativeRequest(options: NativeRequestOptions): Promise { + return getBinding().request(options); +} + +export function validateBrowserProfile(browser?: BrowserProfile): void { + if (!browser) { + return; + } + + if (!getProfiles().includes(browser)) { + throw new Error(`Invalid browser profile: ${browser}`); + } +} + +export function normalizeMethod(method?: string): HttpMethod { + const normalized = (method ?? 'GET').toUpperCase(); + switch (normalized) { + case 'GET': + case 'POST': + case 'PUT': + case 'DELETE': + case 'PATCH': + case 'HEAD': + return normalized; + default: + throw new Error(`Unsupported HTTP method: ${method}`); + } +} diff --git a/src/node-wreq.ts b/src/node-wreq.ts index 52acd13..1e2b6b4 100644 --- a/src/node-wreq.ts +++ b/src/node-wreq.ts @@ -1,178 +1,58 @@ -import type { RequestOptions, Response, BrowserProfile } from './types'; -import { RequestError } from './types'; - -let nativeBinding: { - request: (options: RequestOptions) => Promise; - getProfiles: () => string[]; +import { createClient } from './client'; +import { AbortError, HTTPError, RequestError, TimeoutError } from './errors'; +import { fetch } from './fetch'; +import { Headers } from './headers'; +import { getProfiles } from './native'; +import { Response } from './response'; +import type { + AfterResponseContext, + BeforeErrorContext, + BeforeRequestContext, + BrowserProfile, + ClientDefaults, + CookieJar, + Hooks, + HttpMethod, + InitContext, + RequestInput, + WreqInit, +} from './types'; + +export { + fetch, + createClient, + Headers, + Response, + RequestError, + HTTPError, + TimeoutError, + AbortError, }; -function loadNativeBinding() { - const platform = process.platform; - const arch = process.arch; - - // Map Node.js platform/arch to Rust target triple suffixes - // napi-rs creates files like: node-wreq.linux-x64-gnu.node - const platformArchMap: Record> = { - darwin: { - x64: 'darwin-x64', - arm64: 'darwin-arm64', - }, - linux: { - x64: 'linux-x64-gnu', - }, - win32: { - x64: 'win32-x64-msvc', - }, - }; - - const platformArch = platformArchMap[platform]?.[arch]; - if (!platformArch) { - throw new Error( - `Unsupported platform: ${platform}-${arch}. ` + - `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64` - ); - } - - // Try to load platform-specific binary - const binaryName = `node-wreq.${platformArch}.node`; - - try { - return require(`../rust/${binaryName}`); - } catch (e1) { - // Fallback to node-wreq.node (for local development) - try { - return require('../rust/node-wreq.node'); - } catch (e2) { - throw new Error( - `Failed to load native module for ${platform}-${arch}. ` + - `Tried: ../rust/${binaryName} and ../rust/node-wreq.node. ` + - `Make sure the package is installed correctly and the native module is built for your platform.` - ); - } - } -} - -try { - nativeBinding = loadNativeBinding(); -} catch (error) { - throw error; -} - -/** - * Make an HTTP request with browser impersonation - * - * @param options - Request options - * @returns Promise that resolves to the response - * - * @example - * ```typescript - * import { request } from 'node-wreq'; - * - * const response = await request({ - * url: 'https://example.com/api', - * browser: 'chrome_137', - * headers: { - * 'Custom-Header': 'value' - * } - * }); - * - * console.log(response.status); // 200 - * console.log(response.body); // Response body - * ``` - */ -export async function request(options: RequestOptions): Promise { - if (!options.url) { - throw new RequestError('URL is required'); - } - - if (options.browser) { - const profiles = getProfiles(); - if (!profiles.includes(options.browser)) { - throw new RequestError( - `Invalid browser profile: ${options.browser}. Available profiles: ${profiles.join(', ')}` - ); - } - } - - try { - return await nativeBinding.request(options); - } catch (error) { - throw new RequestError(String(error)); - } -} - -/** - * Get list of available browser profiles - * - * @returns Array of browser profile names - * - * @example - * ```typescript - * import { getProfiles } from 'node-wreq'; - * - * const profiles = getProfiles(); - * console.log(profiles); // ['chrome_120', 'chrome_131', 'firefox', ...] - * ``` - */ -export function getProfiles(): BrowserProfile[] { - return nativeBinding.getProfiles() as BrowserProfile[]; -} - -/** - * Convenience function for GET requests - * - * @param url - URL to request - * @param options - Additional request options - * @returns Promise that resolves to the response - * - * @example - * ```typescript - * import { get } from 'node-wreq'; - * - * const response = await get('https://example.com/api'); - * ``` - */ -export async function get( - url: string, - options?: Omit -): Promise { - return request({ ...options, url, method: 'GET' }); -} - -/** - * Convenience function for POST requests - * - * @param url - URL to request - * @param body - Request body - * @param options - Additional request options - * @returns Promise that resolves to the response - * - * @example - * ```typescript - * import { post } from 'node-wreq'; - * - * const response = await post( - * 'https://example.com/api', - * JSON.stringify({ foo: 'bar' }), - * { headers: { 'Content-Type': 'application/json' } } - * ); - * ``` - */ -export async function post( - url: string, - body?: string, - options?: Omit -): Promise { - return request({ ...options, url, method: 'POST', body }); -} - -export type { RequestOptions, Response, BrowserProfile, HttpMethod } from './types'; - -export type { RequestError }; +export { getProfiles }; + +export type { + AfterResponseContext, + BeforeErrorContext, + BeforeRequestContext, + BrowserProfile, + ClientDefaults, + CookieJar, + Hooks, + HttpMethod, + InitContext, + RequestInput, + WreqInit, +}; export default { - request, - get, - post, + fetch, + createClient, getProfiles, + Headers, + Response, + RequestError, + HTTPError, + TimeoutError, + AbortError, }; diff --git a/src/response.ts b/src/response.ts new file mode 100644 index 0000000..5a9328a --- /dev/null +++ b/src/response.ts @@ -0,0 +1,39 @@ +import { Buffer } from 'node:buffer'; +import { Headers } from './headers'; +import type { NativeResponse } from './types'; + +export class Response { + readonly status: number; + readonly ok: boolean; + readonly url: string; + readonly headers: Headers; + readonly cookies: Record; + private readonly payload: string; + + constructor(nativeResponse: NativeResponse) { + this.status = nativeResponse.status; + this.ok = this.status >= 200 && this.status < 300; + this.url = nativeResponse.url; + this.headers = new Headers(nativeResponse.headers); + this.cookies = { ...nativeResponse.cookies }; + this.payload = nativeResponse.body; + } + + get body(): string { + return this.payload; + } + + async text(): Promise { + return this.payload; + } + + async json(): Promise { + return JSON.parse(this.payload) as T; + } + + async arrayBuffer(): Promise { + const buffer = Buffer.from(this.payload, 'utf8'); + + return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); + } +} diff --git a/src/test/node-wreq.spec.ts b/src/test/node-wreq.spec.ts index da8381d..398233a 100644 --- a/src/test/node-wreq.spec.ts +++ b/src/test/node-wreq.spec.ts @@ -1,6 +1,6 @@ import { test, describe, before } from 'node:test'; import assert from 'node:assert'; -import { request, getProfiles } from '../node-wreq'; +import { Response as WreqResponse, createClient, fetch, getProfiles } from '../node-wreq'; describe('node-wreq', () => { before(() => { @@ -16,24 +16,23 @@ describe('node-wreq', () => { profiles.includes('chrome_137') || profiles.includes('firefox_139') || profiles.includes('safari_18'), - 'Should include standard browser profiles' + 'Should include standard browser profiles', ); console.log(' Available profiles:', profiles.join(', ')); }); test('should make a simple GET request', async () => { - const response = await request({ - url: 'https://httpbin.org/get', + const response = await fetch('https://httpbin.org/get', { browser: 'chrome_131', - timeout: 10000, + timeout: 15000, }); assert.ok(response.status >= 200 && response.status < 300, 'Should return successful status'); assert.ok(Object.keys(response.headers).length > 0, 'Should have response headers'); assert.ok(response.body.length > 0, 'Should have response body'); - const body = JSON.parse(response.body); + const body = await response.json<{ headers: Record }>(); assert.ok(body.headers['User-Agent'], 'Should have User-Agent header'); console.log(' Status:', response.status); @@ -45,15 +44,14 @@ describe('node-wreq', () => { const browsers = ['chrome_137', 'firefox_139', 'safari_18']; for (const browser of browsers) { - const response = await request({ - url: testUrl, + const response = await fetch(testUrl, { browser: browser as any, - timeout: 10000, + timeout: 30000, }); assert.ok(response.status === 200, `${browser} should return status 200`); - const data = JSON.parse(response.body); + const data = await response.json>(); assert.ok(data['user-agent'], `${browser} should have user-agent`); console.log(` ${browser}:`, data['user-agent'].substring(0, 50) + '...'); @@ -63,8 +61,7 @@ describe('node-wreq', () => { test('should handle timeout errors', async () => { await assert.rejects( async () => { - await request({ - url: 'https://httpbin.org/delay/10', + await fetch('https://httpbin.org/delay/10', { browser: 'chrome_137', timeout: 1000, // 1 second timeout for 10 second delay }); @@ -72,7 +69,135 @@ describe('node-wreq', () => { { name: 'RequestError', }, - 'Should throw an error on timeout' + 'Should throw an error on timeout', + ); + }); + + test('should support fetch-style requests', async () => { + const response = await fetch('https://httpbin.org/get', { + browser: 'chrome_137', + query: { source: 'fetch' }, + throwHttpErrors: true, + timeout: 15000, + }); + + assert.strictEqual(response.ok, true, 'Fetch response should expose ok'); + const body = await response.json<{ args: Record }>(); + assert.strictEqual(body.args.source, 'fetch', 'Should apply query params'); + }); + + test('should support createClient defaults', async () => { + const client = createClient({ + browser: 'chrome_137', + baseURL: 'https://httpbin.org', + timeout: 15000, + headers: { + 'X-Test-Client': 'node-wreq', + }, + }); + + const response = await client.get('/headers'); + const body = await response.json<{ headers: Record }>(); + + assert.strictEqual( + body.headers['X-Test-Client'], + 'node-wreq', + 'Client defaults should be merged into outgoing requests', + ); + }); + + test('should run init and beforeRequest hooks', async () => { + const response = await fetch('https://httpbin.org/anything', { + browser: 'chrome_137', + timeout: 15000, + hooks: { + init: [ + ({ options }) => { + options.query = { from: 'init-hook' }; + }, + ], + beforeRequest: [ + ({ request }) => { + request.headers.set('X-Hook-Header', 'active'); + }, + ], + }, + }); + + const body = await response.json<{ + args: Record; + headers: Record; + }>(); + + assert.strictEqual(body.args.from, 'init-hook', 'Init hook should mutate query'); + assert.strictEqual( + body.headers['X-Hook-Header'], + 'active', + 'beforeRequest should mutate headers', + ); + }); + + test('should allow afterResponse to replace the response', async () => { + const response = await fetch('https://httpbin.org/status/201', { + browser: 'chrome_137', + timeout: 15000, + hooks: { + afterResponse: [ + () => + new WreqResponse({ + status: 299, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ replaced: true }), + cookies: {}, + url: 'https://local/replaced', + }), + ], + }, + }); + + assert.strictEqual(response.status, 299, 'afterResponse should replace the response'); + const body = await response.json<{ replaced: boolean }>(); + assert.strictEqual(body.replaced, true, 'Replaced response body should be returned'); + }); + + test('should allow beforeError to rewrite thrown errors', async () => { + await assert.rejects( + async () => { + await fetch('https://httpbin.org/status/418', { + browser: 'chrome_137', + timeout: 15000, + throwHttpErrors: true, + hooks: { + beforeError: [ + ({ error }) => { + error.message = `hooked: ${error.message}`; + return error; + }, + ], + }, + }); + }, + (error: unknown) => + error instanceof Error && error.message.includes('hooked: Request failed with status 418'), + 'beforeError should be able to rewrite the final error', + ); + }); + + test('should reject cookieJar until it is implemented', async () => { + await assert.rejects( + async () => { + await fetch('https://httpbin.org/get', { + browser: 'chrome_137', + cookieJar: { + getCookies: () => [], + setCookie: () => undefined, + }, + }); + }, + (error: unknown) => + error instanceof Error && + error.message.includes('cookieJar support is not implemented yet'), + 'cookieJar should fail explicitly until the runtime support lands', ); }); }); diff --git a/src/types.ts b/src/types.ts index 55a5912..9fcb3ea 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,7 +2,6 @@ * Browser profile names supported */ export type BrowserProfile = - // Chrome | 'chrome_100' | 'chrome_101' | 'chrome_104' @@ -32,13 +31,11 @@ export type BrowserProfile = | 'chrome_135' | 'chrome_136' | 'chrome_137' - // Edge | 'edge_101' | 'edge_122' | 'edge_127' | 'edge_131' | 'edge_134' - // Safari | 'safari_ios_17_2' | 'safari_ios_17_4_1' | 'safari_ios_16_5' @@ -58,7 +55,6 @@ export type BrowserProfile = | 'safari_18_3' | 'safari_18_3_1' | 'safari_18_5' - // Firefox | 'firefox_109' | 'firefox_117' | 'firefox_128' @@ -69,12 +65,10 @@ export type BrowserProfile = | 'firefox_136' | 'firefox_private_136' | 'firefox_139' - // Opera | 'opera_116' | 'opera_117' | 'opera_118' | 'opera_119' - // OkHttp | 'okhttp_3_9' | 'okhttp_3_11' | 'okhttp_3_13' @@ -84,88 +78,149 @@ export type BrowserProfile = | 'okhttp_4_12' | 'okhttp_5'; -/** - * HTTP method types - */ export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD'; -/** - * Request options for making HTTP requests with browser impersonation - */ -export interface RequestOptions { - /** - * The URL to request - */ +export type HeaderTuple = [string, string]; + +export type HeadersInit = + | Record + | HeaderTuple[] + | Iterable; + +export type BodyInit = string | URLSearchParams | Buffer | ArrayBuffer | ArrayBufferView; + +export interface RequestLike { url: string; + method?: string; + headers?: HeadersInit; + body?: unknown; + signal?: AbortSignal | null; + bodyUsed?: boolean; + arrayBuffer?: () => Promise; +} - /** - * Browser profile to impersonate - * @default 'chrome_137' - */ - browser?: BrowserProfile; +export type RequestInput = string | URL | RequestLike; - /** - * HTTP method - * @default 'GET' - */ - method?: HttpMethod; - - /** - * Additional headers to send with the request - * Browser-specific headers will be automatically added - */ - headers?: Record; - - /** - * Request body (for POST, PUT, PATCH requests) - */ - body?: string; +export interface HookState { + [key: string]: unknown; +} - /** - * Proxy URL (e.g., 'http://proxy.example.com:8080') - */ - proxy?: string; +export interface CookieJarCookie { + name: string; + value: string; +} - /** - * Request timeout in milliseconds - * @default 30000 - */ +export interface CookieJar { + getCookies(url: string): Promise | CookieJarCookie[]; + setCookie(cookie: string, url: string): Promise | void; +} + +export interface WreqInit { + method?: string; + headers?: HeadersInit; + body?: BodyInit | null; + signal?: AbortSignal | null; + baseURL?: string; + query?: Record; + browser?: BrowserProfile; + proxy?: string; timeout?: number; + cookieJar?: CookieJar; + throwHttpErrors?: boolean; + validateStatus?: (status: number) => boolean; + disableDefaultHeaders?: boolean; + compress?: boolean; + context?: Record; + hooks?: Hooks; } -/** - * Response object returned from HTTP requests - */ -export interface Response { - /** - * HTTP status code - */ - status: number; +export interface NormalizedRequest { + url: string; + method: HttpMethod; + headers: import('./headers').Headers; + body?: BodyInit | null; +} + +export interface ResolvedOptions extends Omit { + headers: import('./headers').Headers; +} + +export interface InitContext { + input: RequestInput; + options: WreqInit; + state: HookState; +} + +export interface BaseHookContext { + request: NormalizedRequest; + options: ResolvedOptions; + attempt: number; + state: HookState; +} + +export interface BeforeRequestContext extends BaseHookContext {} + +export interface AfterResponseContext extends BaseHookContext { + response: import('./response').Response; +} + +export interface BeforeRetryContext extends BaseHookContext { + error: unknown; +} + +export interface BeforeErrorContext extends BaseHookContext { + error: Error; +} + +export interface Hooks { + init?: InitHook[]; + beforeRequest?: BeforeRequestHook[]; + afterResponse?: AfterResponseHook[]; + beforeRetry?: BeforeRetryHook[]; + beforeError?: BeforeErrorHook[]; +} + +export type InitHook = (ctx: InitContext) => void | Promise; +export type BeforeRequestHook = ( + ctx: BeforeRequestContext, +) => void | import('./response').Response | Promise; +export type AfterResponseHook = ( + ctx: AfterResponseContext, +) => void | import('./response').Response | Promise; +export type BeforeRetryHook = (ctx: BeforeRetryContext) => void | Promise; +export type BeforeErrorHook = (ctx: BeforeErrorContext) => Error | void | Promise; - /** - * Response headers - */ +export interface NativeRequestOptions { + url: string; + method: HttpMethod; headers: Record; + body?: string; + browser?: BrowserProfile; + proxy?: string; + timeout?: number; +} - /** - * Response body as string - */ +export interface NativeResponse { + status: number; + headers: Record; body: string; - - /** - * Cookies set by the server - */ cookies: Record; - - /** - * Final URL after redirects - */ url: string; } -export class RequestError extends Error { - constructor(message: string) { - super(message); - this.name = 'RequestError'; - } +export interface ClientDefaults extends Omit { + headers?: HeadersInit; + hooks?: Hooks; +} + +export interface Client { + readonly defaults: ClientDefaults; + fetch(input: RequestInput, init?: WreqInit): Promise; + get(input: RequestInput, init?: Omit): Promise; + post( + input: RequestInput, + body?: BodyInit | null, + init?: Omit, + ): Promise; + extend(defaults: ClientDefaults): Client; } From 5c78f2feb4859b9b78da601bd923a8e667a5d4f9 Mon Sep 17 00:00:00 2001 From: ruby Date: Tue, 17 Mar 2026 06:06:10 +0400 Subject: [PATCH 02/15] feat: whatwg websockets --- package-lock.json | 36 +- package.json | 4 +- rust/Cargo.lock | 179 +++++++++- rust/Cargo.toml | 2 +- rust/src/client.rs | 451 +++++++++++++++++++++++-- rust/src/lib.rs | 320 +++++++++++++++++- src/client.ts | 91 ++++- src/errors.ts | 38 ++- src/fetch.ts | 642 ++++++++++++++++++++++++++++++----- src/hooks.ts | 29 +- src/native.ts | 72 +++- src/node-wreq.ts | 31 +- src/response.ts | 184 +++++++++- src/test/node-wreq.spec.ts | 674 ++++++++++++++++++++++++++++++++++++- src/types.ts | 143 +++++++- src/websocket.ts | 502 +++++++++++++++++++++++++++ src/ws.d.ts | 28 ++ 17 files changed, 3253 insertions(+), 173 deletions(-) create mode 100644 src/websocket.ts create mode 100644 src/ws.d.ts diff --git a/package-lock.json b/package-lock.json index ba6b2c4..eda80e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,9 +20,11 @@ "devDependencies": { "@napi-rs/cli": "^2.18.0", "@types/node": "^20.0.0", + "@types/ws": "^8.18.1", "oxfmt": "^0.41.0", "oxlint": "^1.56.0", - "typescript": "^5.0.0" + "typescript": "^5.0.0", + "ws": "^8.18.3" }, "engines": { "node": ">=18.0.0" @@ -697,6 +699,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/oxfmt": { "version": "0.41.0", "resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.41.0.tgz", @@ -808,6 +820,28 @@ "version": "6.21.0", "dev": true, "license": "MIT" + }, + "node_modules/ws": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } } } } diff --git a/package.json b/package.json index a4fa47d..ace4182 100644 --- a/package.json +++ b/package.json @@ -39,10 +39,12 @@ "dependencies": {}, "devDependencies": { "@types/node": "^20.0.0", + "@types/ws": "^8.18.1", "typescript": "^5.0.0", "@napi-rs/cli": "^2.18.0", "oxfmt": "^0.41.0", - "oxlint": "^1.56.0" + "oxlint": "^1.56.0", + "ws": "^8.18.3" }, "engines": { "node": ">=18.0.0" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8bf0e39..41d2a12 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -123,6 +123,15 @@ version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "boring-sys2" version = "5.0.0-alpha.10" @@ -253,6 +262,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -262,6 +280,22 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + [[package]] name = "deranged" version = "0.5.4" @@ -271,6 +305,16 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.15.0" @@ -391,9 +435,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", + "futures-sink", "futures-task", "pin-project-lite", "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", ] [[package]] @@ -598,6 +654,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + [[package]] name = "memchr" version = "2.7.6" @@ -666,7 +728,7 @@ dependencies = [ "neon", "serde", "serde_json", - "thiserror", + "thiserror 1.0.69", "tokio", "wreq", "wreq-util", @@ -767,6 +829,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -791,6 +862,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -937,6 +1037,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1003,7 +1114,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", ] [[package]] @@ -1017,6 +1137,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.3.44" @@ -1098,10 +1229,22 @@ checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", - "thiserror", + "thiserror 1.0.69", "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.16" @@ -1168,6 +1311,23 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror 2.0.17", + "utf-8", +] + [[package]] name = "typed-builder" version = "0.22.0" @@ -1188,12 +1348,24 @@ dependencies = [ "syn", ] +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "version_check" version = "0.9.5" @@ -1382,6 +1554,7 @@ dependencies = [ "tokio", "tokio-boring2", "tokio-socks", + "tokio-tungstenite", "tower", "tower-http", "want", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dd2591e..718accc 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [dependencies] # HTTP client with browser impersonation -wreq = { version = "6.0.0-rc.20", default-features = false, features = ["gzip", "deflate", "socks", "cookies", "json", "webpki-roots"] } +wreq = { version = "6.0.0-rc.20", default-features = false, features = ["gzip", "deflate", "socks", "cookies", "json", "webpki-roots", "ws"] } wreq-util = "3.0.0-rc.1" # Neon for Node.js bindings diff --git a/rust/src/client.rs b/rust/src/client.rs index b31752e..4e0bbca 100644 --- a/rust/src/client.rs +++ b/rust/src/client.rs @@ -1,6 +1,13 @@ use anyhow::{Context, Result}; use std::collections::HashMap; +use std::sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicU64, Ordering}, +}; use std::time::Duration; +use wreq::redirect; +use wreq::ws::WebSocket; +use wreq::ws::message::{CloseCode, CloseFrame, Message}; use wreq_util::Emulation; #[derive(Debug, Clone)] @@ -18,28 +25,369 @@ pub struct RequestOptions { pub struct Response { pub status: u16, pub headers: HashMap, - pub body: String, + pub body_handle: u64, pub cookies: HashMap, + pub set_cookies: Vec, pub url: String, } +#[derive(Debug, Clone)] +pub struct WebSocketConnectOptions { + pub url: String, + pub emulation: Emulation, + pub headers: HashMap, + pub proxy: Option, + pub timeout: u64, + pub protocols: Vec, +} + +#[derive(Debug, Clone)] +pub struct WebSocketConnection { + pub handle: u64, + pub protocol: Option, + pub extensions: Option, + pub url: String, +} + +#[derive(Debug, Clone)] +pub enum WebSocketReadResult { + Text(String), + Binary(Vec), + Close { + code: u16, + reason: String, + was_clean: bool, + }, +} + +#[derive(Debug)] +enum WebSocketCommand { + Text(String), + Binary(Vec), + Close { + code: Option, + reason: Option, + }, +} + +#[derive(Debug)] +struct StoredBody { + response: wreq::Response, +} + +#[derive(Debug)] +struct StoredWebSocket { + commands: tokio::sync::mpsc::UnboundedSender, + events: tokio::sync::Mutex>, +} + +type SharedWebSocket = Arc; + +static NEXT_BODY_HANDLE: AtomicU64 = AtomicU64::new(1); +static NEXT_WEBSOCKET_HANDLE: AtomicU64 = AtomicU64::new(1); +static BODY_STORE: OnceLock>> = OnceLock::new(); +static WEBSOCKET_STORE: OnceLock>> = OnceLock::new(); +static TOKIO_RUNTIME: OnceLock = OnceLock::new(); + +fn body_store() -> &'static Mutex> { + BODY_STORE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn websocket_store() -> &'static Mutex> { + WEBSOCKET_STORE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn runtime() -> &'static tokio::runtime::Runtime { + TOKIO_RUNTIME.get_or_init(|| { + tokio::runtime::Runtime::new().expect("Failed to create Tokio runtime") + }) +} + +pub fn execute_request(options: RequestOptions) -> Result { + runtime().block_on(make_request(options)) +} + +pub fn connect_websocket(options: WebSocketConnectOptions) -> Result { + runtime().block_on(make_websocket(options)) +} + +fn store_body(response: wreq::Response) -> u64 { + let handle = NEXT_BODY_HANDLE.fetch_add(1, Ordering::Relaxed); + body_store() + .lock() + .expect("body store poisoned") + .insert(handle, StoredBody { response }); + handle +} + +fn store_websocket(websocket: WebSocket) -> u64 { + let handle = NEXT_WEBSOCKET_HANDLE.fetch_add(1, Ordering::Relaxed); + let (command_tx, command_rx) = tokio::sync::mpsc::unbounded_channel(); + let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel(); + + runtime().spawn(run_websocket_task(websocket, command_rx, event_tx)); + + websocket_store() + .lock() + .expect("websocket store poisoned") + .insert( + handle, + Arc::new(StoredWebSocket { + commands: command_tx, + events: tokio::sync::Mutex::new(event_rx), + }), + ); + + handle +} + +fn get_websocket(handle: u64) -> Result { + let store = websocket_store() + .lock() + .map_err(|_| anyhow::anyhow!("websocket store poisoned"))?; + + store + .get(&handle) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Unknown websocket handle: {}", handle)) +} + +pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { + let mut store = body_store() + .lock() + .map_err(|_| anyhow::anyhow!("body store poisoned"))?; + let Some(body) = store.get_mut(&handle) else { + return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); + }; + + let chunk = runtime() + .block_on(body.response.chunk()) + .context("Failed to read response body chunk")?; + + let Some(chunk) = chunk else { + store.remove(&handle); + return Ok((Vec::new(), true)); + }; + + Ok((chunk.to_vec(), false)) +} + +pub fn read_body_all(handle: u64) -> Result> { + let mut store = body_store() + .lock() + .map_err(|_| anyhow::anyhow!("body store poisoned"))?; + let Some(body) = store.remove(&handle) else { + return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); + }; + + let mut bytes = Vec::new(); + let mut response = body.response; + + runtime().block_on(async { + while let Some(chunk) = response + .chunk() + .await + .context("Failed to read response body chunk")? + { + bytes.extend_from_slice(&chunk); + } + + Ok::<(), anyhow::Error>(()) + })?; + + Ok(bytes) +} + +pub fn cancel_body(handle: u64) -> bool { + body_store() + .lock() + .expect("body store poisoned") + .remove(&handle) + .is_some() +} + +pub fn read_websocket_message(handle: u64) -> Result { + let websocket = get_websocket(handle)?; + + let result = runtime().block_on(async { + let mut events = websocket.events.lock().await; + events + .recv() + .await + .ok_or_else(|| anyhow::anyhow!("WebSocket event stream is closed")) + }); + + if matches!(result, Ok(WebSocketReadResult::Close { .. })) { + websocket_store() + .lock() + .expect("websocket store poisoned") + .remove(&handle); + } + + result +} + +pub fn send_websocket_text(handle: u64, text: String) -> Result<()> { + let websocket = get_websocket(handle)?; + websocket + .commands + .send(WebSocketCommand::Text(text)) + .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) +} + +pub fn send_websocket_binary(handle: u64, bytes: Vec) -> Result<()> { + let websocket = get_websocket(handle)?; + websocket + .commands + .send(WebSocketCommand::Binary(bytes)) + .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) +} + +pub fn close_websocket(handle: u64, code: Option, reason: Option) -> Result<()> { + let websocket = get_websocket(handle)?; + websocket + .commands + .send(WebSocketCommand::Close { code, reason }) + .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) +} + +fn parse_cookie_pair(set_cookie: &str) -> Option<(String, String)> { + let pair = set_cookie.split(';').next()?.trim(); + let (name, value) = pair.split_once('=')?; + + Some((name.to_string(), value.to_string())) +} + +async fn run_websocket_task( + mut websocket: WebSocket, + mut commands: tokio::sync::mpsc::UnboundedReceiver, + events: tokio::sync::mpsc::UnboundedSender, +) { + let mut close_requested = false; + let mut requested_close_code = 1000; + let mut requested_close_reason = String::new(); + + loop { + tokio::select! { + command = commands.recv() => { + match command { + Some(WebSocketCommand::Text(text)) => { + if websocket.send(Message::Text(text.into())).await.is_err() { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + } + Some(WebSocketCommand::Binary(bytes)) => { + if websocket.send(Message::Binary(bytes.into())).await.is_err() { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + } + Some(WebSocketCommand::Close { code, reason }) => { + close_requested = true; + requested_close_code = code.unwrap_or(1000); + requested_close_reason = reason.unwrap_or_default(); + + let frame = Message::Close(Some(CloseFrame { + code: CloseCode::from(requested_close_code), + reason: requested_close_reason.clone().into(), + })); + + if websocket.send(frame).await.is_err() { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + } + None => { + break; + } + } + } + message = websocket.recv() => { + match message { + Some(Ok(Message::Text(text))) => { + if events.send(WebSocketReadResult::Text(text.to_string())).is_err() { + break; + } + } + Some(Ok(Message::Binary(bytes))) => { + if events.send(WebSocketReadResult::Binary(bytes.to_vec())).is_err() { + break; + } + } + Some(Ok(Message::Close(frame))) => { + let (code, reason) = match frame { + Some(frame) => (u16::from(frame.code), frame.reason.to_string()), + None => { + if close_requested { + (requested_close_code, requested_close_reason.clone()) + } else { + (1005, String::new()) + } + } + }; + + let _ = events.send(WebSocketReadResult::Close { + code, + reason, + was_clean: true, + }); + break; + } + Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => {} + Some(Err(_)) => { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + None => { + let _ = events.send(WebSocketReadResult::Close { + code: if close_requested { + requested_close_code + } else { + 1006 + }, + reason: if close_requested { + requested_close_reason.clone() + } else { + String::new() + }, + was_clean: close_requested, + }); + break; + } + } + } + } + } +} + pub async fn make_request(options: RequestOptions) -> Result { - // Create client builder with emulation let mut client_builder = wreq::Client::builder() .emulation(options.emulation) .cookie_store(true); - // Apply proxy if present (must be set at client builder level) if let Some(proxy_url) = &options.proxy { - let proxy = wreq::Proxy::all(proxy_url) - .context("Failed to create proxy")?; + let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; client_builder = client_builder.proxy(proxy); } - // Build the client - let client = client_builder - .build() - .context("Failed to build HTTP client")?; + let client = client_builder.build().context("Failed to build HTTP client")?; let method = if options.method.is_empty() { "GET" @@ -47,7 +395,6 @@ pub async fn make_request(options: RequestOptions) -> Result { &options.method }; - // Build request let mut request = match method.to_uppercase().as_str() { "GET" => client.get(&options.url), "POST" => client.post(&options.url), @@ -58,30 +405,25 @@ pub async fn make_request(options: RequestOptions) -> Result { _ => return Err(anyhow::anyhow!("Unsupported HTTP method: {}", method)), }; - // Apply custom headers for (key, value) in &options.headers { request = request.header(key, value); } - // Apply body if present if let Some(body) = options.body { request = request.body(body); } - // Apply timeout request = request.timeout(Duration::from_millis(options.timeout)); + request = request.redirect(redirect::Policy::none()); - // Execute request let response = request .send() .await .with_context(|| format!("{} {}", method, options.url))?; - // Extract response data let status = response.status().as_u16(); let final_url = response.uri().to_string(); - // Extract headers let mut response_headers = HashMap::new(); for (key, value) in response.headers() { if let Ok(value_str) = value.to_str() { @@ -89,30 +431,81 @@ pub async fn make_request(options: RequestOptions) -> Result { } } - // Extract cookies let mut cookies = HashMap::new(); - if let Some(cookie_header) = response.headers().get("set-cookie") { + let mut set_cookies = Vec::new(); + for cookie_header in response.headers().get_all("set-cookie") { if let Ok(cookie_str) = cookie_header.to_str() { - // Simple cookie parsing (name=value) - for cookie_part in cookie_str.split(';') { - if let Some((key, value)) = cookie_part.trim().split_once('=') { - cookies.insert(key.to_string(), value.to_string()); - } + set_cookies.push(cookie_str.to_string()); + + if let Some((key, value)) = parse_cookie_pair(cookie_str) { + cookies.insert(key, value); } } } - // Get body - let body = response - .text() - .await - .context("Failed to read response body")?; + let body_handle = store_body(response); Ok(Response { status, headers: response_headers, - body, + body_handle, cookies, + set_cookies, url: final_url, }) } + +async fn make_websocket(options: WebSocketConnectOptions) -> Result { + let mut client_builder = wreq::Client::builder() + .emulation(options.emulation) + .cookie_store(true) + .timeout(Duration::from_millis(options.timeout)); + + if let Some(proxy_url) = &options.proxy { + let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; + client_builder = client_builder.proxy(proxy); + } + + let client = client_builder + .build() + .context("Failed to build WebSocket client")?; + + let mut request = client.websocket(&options.url); + for (key, value) in &options.headers { + request = request.header(key, value); + } + + if !options.protocols.is_empty() { + request = request.protocols(options.protocols.iter().cloned()); + } + + let response = request + .send() + .await + .with_context(|| format!("WS {}", options.url))?; + + let extensions = response + .headers() + .get("sec-websocket-extensions") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + + let websocket = response + .into_websocket() + .await + .with_context(|| format!("WS upgrade {}", options.url))?; + + let protocol = websocket + .protocol() + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + + let handle = store_websocket(websocket); + + Ok(WebSocketConnection { + handle, + protocol, + extensions, + url: options.url, + }) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f5e6611..71605bf 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,7 +1,13 @@ mod client; use neon::prelude::*; -use client::{make_request, RequestOptions, Response}; +use neon::types::buffer::TypedArray; +use neon::types::JsBuffer; +use client::{ + RequestOptions, Response, WebSocketConnectOptions, WebSocketConnection, WebSocketReadResult, + cancel_body, close_websocket, connect_websocket, execute_request, read_body_all, + read_body_chunk, read_websocket_message, send_websocket_binary, send_websocket_text, +}; use std::collections::HashMap; use wreq_util::Emulation; @@ -162,6 +168,66 @@ fn js_object_to_request_options(cx: &mut FunctionContext, obj: Handle) }) } +fn js_object_to_websocket_options( + cx: &mut FunctionContext, + obj: Handle, +) -> NeonResult { + let url: Handle = obj.get(cx, "url")?; + let url = url.value(cx); + + let browser_str = obj + .get_opt(cx, "browser")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or_else(|| "chrome_137".to_string()); + + let emulation = parse_emulation(&browser_str); + + let mut headers = HashMap::new(); + if let Ok(Some(headers_obj)) = obj.get_opt::(cx, "headers") { + let keys = headers_obj.get_own_property_names(cx)?; + let keys_vec = keys.to_vec(cx)?; + + for key_val in keys_vec { + if let Ok(key_str) = key_val.downcast::(cx) { + let key = key_str.value(cx); + if let Ok(value) = headers_obj.get::(cx, key.as_str()) { + headers.insert(key, value.value(cx)); + } + } + } + } + + let proxy = obj + .get_opt(cx, "proxy")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)); + + let timeout = obj + .get_opt(cx, "timeout")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx) as u64) + .unwrap_or(30000); + + let mut protocols = Vec::new(); + if let Some(values) = obj.get_opt::(cx, "protocols")? { + for value in values.to_vec(cx)? { + if let Ok(value) = value.downcast::(cx) { + protocols.push(value.value(cx)); + } + } + } + + Ok(WebSocketConnectOptions { + url, + emulation, + headers, + proxy, + timeout, + protocols, + }) +} + // Convert Response to JS object fn response_to_js_object<'a, C: Context<'a>>(cx: &mut C, response: Response) -> JsResult<'a, JsObject> { let obj = cx.empty_object(); @@ -190,9 +256,53 @@ fn response_to_js_object<'a, C: Context<'a>>(cx: &mut C, response: Response) -> } obj.set(cx, "cookies", cookies_obj)?; - // Body - let body = cx.string(&response.body); - obj.set(cx, "body", body)?; + // Raw Set-Cookie headers + let set_cookies = JsArray::new(cx, response.set_cookies.len()); + for (index, value) in response.set_cookies.into_iter().enumerate() { + let value_str = cx.string(&value); + set_cookies.set(cx, index as u32, value_str)?; + } + obj.set(cx, "setCookies", set_cookies)?; + + // Body handle + let body_handle = cx.number(response.body_handle as f64); + obj.set(cx, "bodyHandle", body_handle)?; + + Ok(obj) +} + +fn websocket_to_js_object<'a, C: Context<'a>>( + cx: &mut C, + websocket: WebSocketConnection, +) -> JsResult<'a, JsObject> { + let obj = cx.empty_object(); + let handle = cx.number(websocket.handle as f64); + let url = cx.string(&websocket.url); + + obj.set(cx, "handle", handle)?; + obj.set(cx, "url", url)?; + + match websocket.protocol { + Some(protocol) => { + let value = cx.string(protocol); + obj.set(cx, "protocol", value)?; + } + None => { + let value = cx.null(); + obj.set(cx, "protocol", value)?; + } + }; + + match websocket.extensions { + Some(extensions) => { + let value = cx.string(extensions); + obj.set(cx, "extensions", value)?; + } + None => { + let value = cx.null(); + obj.set(cx, "extensions", value)?; + } + }; Ok(obj) } @@ -211,10 +321,7 @@ fn request(mut cx: FunctionContext) -> JsResult { // Create a new Tokio runtime for this request std::thread::spawn(move || { - let rt = tokio::runtime::Runtime::new().expect("Failed to create Tokio runtime"); - - // Make the request - let result = rt.block_on(make_request(options)); + let result = execute_request(options); // Send result back to JS deferred.settle_with(&channel, move |mut cx| { @@ -232,6 +339,195 @@ fn request(mut cx: FunctionContext) -> JsResult { Ok(promise) } +fn read_body_chunk_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let size = cx.argument_opt(1) + .and_then(|value| value.downcast::(&mut cx).ok()) + .map(|value| value.value(&mut cx) as usize) + .unwrap_or(65_536); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = read_body_chunk(handle, size); + + deferred.settle_with(&channel, move |mut cx| { + match result { + Ok((chunk, done)) => { + let obj = cx.empty_object(); + let chunk_buffer = JsBuffer::from_slice(&mut cx, &chunk)?; + let done_value = cx.boolean(done); + obj.set(&mut cx, "chunk", chunk_buffer)?; + obj.set(&mut cx, "done", done_value)?; + Ok(obj) + } + Err(error) => cx.throw_error(format!("{:#}", error)), + } + }); + }); + + Ok(promise) +} + +fn read_body_all_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = read_body_all(handle); + + deferred.settle_with(&channel, move |mut cx| { + match result { + Ok(bytes) => JsBuffer::from_slice(&mut cx, &bytes), + Err(error) => cx.throw_error(format!("{:#}", error)), + } + }); + }); + + Ok(promise) +} + +fn cancel_body_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + Ok(cx.boolean(cancel_body(handle))) +} + +fn websocket_connect_js(mut cx: FunctionContext) -> JsResult { + let options_obj = cx.argument::(0)?; + let options = js_object_to_websocket_options(&mut cx, options_obj)?; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = connect_websocket(options); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(websocket) => websocket_to_js_object(&mut cx, websocket), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_read_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = read_websocket_message(handle); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(WebSocketReadResult::Text(text)) => { + let obj = cx.empty_object(); + let type_value = cx.string("text"); + let data_value = cx.string(text); + obj.set(&mut cx, "type", type_value)?; + obj.set(&mut cx, "data", data_value)?; + Ok(obj) + } + Ok(WebSocketReadResult::Binary(bytes)) => { + let obj = cx.empty_object(); + let type_value = cx.string("binary"); + let data_value = JsBuffer::from_slice(&mut cx, &bytes)?; + obj.set(&mut cx, "type", type_value)?; + obj.set(&mut cx, "data", data_value)?; + Ok(obj) + } + Ok(WebSocketReadResult::Close { + code, + reason, + was_clean, + }) => { + let obj = cx.empty_object(); + let type_value = cx.string("close"); + let code_value = cx.number(code as f64); + let reason_value = cx.string(reason); + let was_clean_value = cx.boolean(was_clean); + obj.set(&mut cx, "type", type_value)?; + obj.set(&mut cx, "code", code_value)?; + obj.set(&mut cx, "reason", reason_value)?; + obj.set(&mut cx, "wasClean", was_clean_value)?; + Ok(obj) + } + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_send_text_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let text = cx.argument::(1)?.value(&mut cx); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = send_websocket_text(handle, text); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(()) => Ok(cx.undefined()), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_send_binary_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let buffer = cx.argument::(1)?; + let bytes = buffer.as_slice(&cx).to_vec(); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = send_websocket_binary(handle, bytes); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(()) => Ok(cx.undefined()), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_close_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let code = cx + .argument_opt(1) + .and_then(|value| value.downcast::(&mut cx).ok()) + .map(|value| value.value(&mut cx) as u16); + let reason = cx + .argument_opt(2) + .and_then(|value| value.downcast::(&mut cx).ok()) + .map(|value| value.value(&mut cx)); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = close_websocket(handle, code, reason); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(()) => Ok(cx.undefined()), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + // Get list of available browser profiles fn get_profiles(mut cx: FunctionContext) -> JsResult { let profiles = vec![ @@ -274,6 +570,14 @@ fn get_profiles(mut cx: FunctionContext) -> JsResult { #[neon::main] fn main(mut cx: ModuleContext) -> NeonResult<()> { cx.export_function("request", request)?; + cx.export_function("readBodyChunk", read_body_chunk_js)?; + cx.export_function("readBodyAll", read_body_all_js)?; + cx.export_function("cancelBody", cancel_body_js)?; + cx.export_function("websocketConnect", websocket_connect_js)?; + cx.export_function("websocketRead", websocket_read_js)?; + cx.export_function("websocketSendText", websocket_send_text_js)?; + cx.export_function("websocketSendBinary", websocket_send_binary_js)?; + cx.export_function("websocketClose", websocket_close_js)?; cx.export_function("getProfiles", get_profiles)?; Ok(()) } diff --git a/src/client.ts b/src/client.ts index cb28d17..82f2664 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,14 +1,17 @@ -import { Headers } from './headers'; import { fetch } from './fetch'; +import { Headers } from './headers'; import { mergeHooks } from './hooks'; import type { Client, ClientDefaults, HeaderTuple, HeadersInit, + RetryOptions, RequestInput, + WebSocketInit, WreqInit, } from './types'; +import { websocket } from './websocket'; function mergeHeaders(...sources: Array): HeaderTuple[] | undefined { const merged = new Headers(); @@ -28,6 +31,70 @@ function mergeHeaders(...sources: Array): HeaderTuple[] return tuples.length > 0 ? tuples : undefined; } +function mergeQuery( + ...sources: Array +): WreqInit['query'] | undefined { + const merged: NonNullable = {}; + + for (const source of sources) { + if (!source) { + continue; + } + + Object.assign(merged, source); + } + + return Object.keys(merged).length > 0 ? merged : undefined; +} + +function mergeContext( + base?: Record, + override?: Record +): Record | undefined { + if (!base && !override) { + return undefined; + } + + return { + ...base, + ...override, + }; +} + +function normalizeRetryInput(retry?: number | RetryOptions): RetryOptions | undefined { + if (retry === undefined) { + return undefined; + } + + if (typeof retry === 'number') { + return { limit: retry }; + } + + return { ...retry }; +} + +function mergeRetry( + base?: number | RetryOptions, + override?: number | RetryOptions +): RetryOptions | undefined { + const normalizedBase = normalizeRetryInput(base); + const normalizedOverride = normalizeRetryInput(override); + + if (!normalizedBase && !normalizedOverride) { + return undefined; + } + + return { + ...normalizedBase, + ...normalizedOverride, + methods: normalizedOverride?.methods ?? normalizedBase?.methods, + statusCodes: normalizedOverride?.statusCodes ?? normalizedBase?.statusCodes, + errorCodes: normalizedOverride?.errorCodes ?? normalizedBase?.errorCodes, + backoff: normalizedOverride?.backoff ?? normalizedBase?.backoff, + shouldRetry: normalizedOverride?.shouldRetry ?? normalizedBase?.shouldRetry, + }; +} + function mergeDefaults(base: ClientDefaults, override?: ClientDefaults): ClientDefaults { if (!override) { return { ...base }; @@ -37,7 +104,10 @@ function mergeDefaults(base: ClientDefaults, override?: ClientDefaults): ClientD ...base, ...override, headers: mergeHeaders(base.headers, override.headers), + query: mergeQuery(base.query, override.query), + context: mergeContext(base.context, override.context), hooks: mergeHooks(base.hooks, override.hooks), + retry: mergeRetry(base.retry, override.retry), }; } @@ -48,6 +118,9 @@ class WreqClient implements Client { this.defaults = { ...defaults, headers: defaults.headers ? mergeHeaders(defaults.headers) : undefined, + query: mergeQuery(defaults.query), + context: mergeContext(defaults.context), + retry: mergeRetry(defaults.retry), }; } @@ -56,11 +129,25 @@ class WreqClient implements Client { ...this.defaults, ...init, headers: mergeHeaders(this.defaults.headers, init?.headers), + query: mergeQuery(this.defaults.query, init?.query), + context: mergeContext(this.defaults.context, init?.context), hooks: mergeHooks(this.defaults.hooks, init?.hooks), + retry: mergeRetry(this.defaults.retry, init?.retry), }; return fetch(input, merged); } + async websocket(input: string | URL, init?: WebSocketInit) { + const merged: WebSocketInit = { + ...this.defaults, + ...init, + headers: mergeHeaders(this.defaults.headers, init?.headers), + query: mergeQuery(this.defaults.query, init?.query), + }; + + return websocket(input, merged); + } + async get(input: RequestInput, init?: Omit) { return this.fetch(input, { ...init, method: 'GET' }); } @@ -68,7 +155,7 @@ class WreqClient implements Client { async post( input: RequestInput, body?: WreqInit['body'], - init?: Omit, + init?: Omit ) { return this.fetch(input, { ...init, method: 'POST', body }); } diff --git a/src/errors.ts b/src/errors.ts index eb771a1..597fe7b 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,35 +1,59 @@ +import type { Response } from './response'; +import type { NormalizedRequest } from './types'; + +type RequestErrorOptions = { + code?: string; + cause?: unknown; + request?: NormalizedRequest; + response?: Response; + attempt?: number; +}; + export class RequestError extends Error { code?: string; cause?: unknown; + request?: NormalizedRequest; + response?: Response; + attempt?: number; - constructor(message: string, options?: { code?: string; cause?: unknown }) { + constructor(message: string, options?: RequestErrorOptions) { super(message); this.name = 'RequestError'; this.code = options?.code; this.cause = options?.cause; + this.request = options?.request; + this.response = options?.response; + this.attempt = options?.attempt; } } export class HTTPError extends RequestError { status: number; - constructor(message: string, status: number, options?: { cause?: unknown }) { - super(message, { code: 'ERR_HTTP_STATUS', cause: options?.cause }); + constructor(message: string, status: number, options?: RequestErrorOptions) { + super(message, { ...options, code: 'ERR_HTTP_STATUS' }); this.name = 'HTTPError'; this.status = status; } } export class TimeoutError extends RequestError { - constructor(message = 'Request timed out', options?: { cause?: unknown }) { - super(message, { code: 'ERR_TIMEOUT', cause: options?.cause }); + constructor(message = 'Request timed out', options?: RequestErrorOptions) { + super(message, { ...options, code: 'ERR_TIMEOUT' }); this.name = 'TimeoutError'; } } export class AbortError extends RequestError { - constructor(message = 'The operation was aborted', options?: { cause?: unknown }) { - super(message, { code: 'ERR_ABORTED', cause: options?.cause }); + constructor(message = 'The operation was aborted', options?: RequestErrorOptions) { + super(message, { ...options, code: 'ERR_ABORTED' }); this.name = 'AbortError'; } } + +export class WebSocketError extends RequestError { + constructor(message = 'WebSocket operation failed', options?: RequestErrorOptions) { + super(message, { ...options, code: 'ERR_WEBSOCKET' }); + this.name = 'WebSocketError'; + } +} diff --git a/src/fetch.ts b/src/fetch.ts index 9dff33e..83fd7ee 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,31 +1,53 @@ import { Buffer } from 'node:buffer'; -import { HTTPError, RequestError } from './errors'; +import { AbortError, HTTPError, RequestError, TimeoutError } from './errors'; import { Headers } from './headers'; import { runAfterResponseHooks, runBeforeErrorHooks, + runBeforeRedirectHooks, runBeforeRequestHooks, + runBeforeRetryHooks, runInitHooks, } from './hooks'; import { nativeRequest, normalizeMethod, validateBrowserProfile } from './native'; import { Response } from './response'; import type { + BodyInit, + CookieJar, HeadersInit, + HttpMethod, NativeRequestOptions, NormalizedRequest, + RedirectEntry, RequestInput, RequestLike, + RequestStats, ResolvedOptions, + ResolvedRetryOptions, + RetryDecisionContext, WreqInit, } from './types'; +const DEFAULT_RETRY_METHODS = ['GET', 'HEAD'] as const; +const DEFAULT_RETRY_STATUS_CODES = [408, 425, 429, 500, 502, 503, 504]; +const DEFAULT_RETRY_ERROR_CODES = [ + 'ECONNABORTED', + 'ECONNRESET', + 'ECONNREFUSED', + 'EPIPE', + 'ETIMEDOUT', + 'ERR_ABORTED', + 'ERR_TIMEOUT', +]; +const REDIRECT_STATUS_CODES = new Set([300, 301, 302, 303, 307, 308]); + function isRequestLike(value: unknown): value is RequestLike { return typeof value === 'object' && value !== null && 'url' in value; } function isResponseStatusAllowed( status: number, - options: Pick, + options: Pick ): boolean { if (options.validateStatus) { return options.validateStatus(status); @@ -47,6 +69,7 @@ function appendQuery(url: URL, query: WreqInit['query']): void { if (value === undefined || value === null) { continue; } + url.searchParams.set(key, String(value)); } } @@ -57,7 +80,7 @@ function resolveUrl(rawUrl: string | URL, baseURL?: string, query?: WreqInit['qu return url.toString(); } -async function normalizeBody(body: WreqInit['body']): Promise { +async function normalizeBody(body: BodyInit | null | undefined): Promise { if (body === undefined || body === null) { return undefined; } @@ -87,7 +110,7 @@ async function normalizeBody(body: WreqInit['body']): Promise { - if (init.cookieJar) { - throw new RequestError('cookieJar support is not implemented yet'); - } - - const method = normalizeMethod(init.method); - validateBrowserProfile(init.browser); +function resolveRetryOptions(retry?: WreqInit['retry']): ResolvedRetryOptions { + const configured = typeof retry === 'number' ? { limit: retry } : retry; - const headers = new Headers(init.headers); - const request: NormalizedRequest = { - url: resolveUrl(urlInput, init.baseURL, init.query), - method, - headers, - body: init.body, + return { + limit: configured?.limit ?? 0, + methods: configured?.methods ?? [...DEFAULT_RETRY_METHODS], + statusCodes: configured?.statusCodes ?? [...DEFAULT_RETRY_STATUS_CODES], + errorCodes: configured?.errorCodes ?? [...DEFAULT_RETRY_ERROR_CODES], + backoff: configured?.backoff, + shouldRetry: configured?.shouldRetry, }; +} - const options: ResolvedOptions = { +function resolveOptions(init: WreqInit): ResolvedOptions { + return { ...init, - headers, + headers: new Headers(init.headers), + retry: resolveRetryOptions(init.retry), + redirect: init.redirect ?? 'follow', + maxRedirects: init.maxRedirects ?? 20, + throwHttpErrors: init.throwHttpErrors ?? false, + disableDefaultHeaders: init.disableDefaultHeaders ?? false, + compress: init.compress ?? true, }; +} + +function createRequest(urlInput: string | URL, options: ResolvedOptions): NormalizedRequest { + validateBrowserProfile(options.browser); + + return { + url: resolveUrl(urlInput, options.baseURL, options.query), + method: normalizeMethod(options.method), + headers: options.headers, + body: options.body, + }; +} - const nativeRequest: NativeRequestOptions = { +function buildNativeRequest( + request: NormalizedRequest, + options: ResolvedOptions, + body: string | undefined +): NativeRequestOptions { + return { url: request.url, method: request.method, headers: request.headers.toObject(), - body: await normalizeBody(request.body), - browser: init.browser, - proxy: init.proxy, - timeout: init.timeout, + body, + browser: options.browser, + proxy: options.proxy, + timeout: options.timeout, }; - - return { nativeRequest, request, options }; } function isError(value: unknown): value is Error { return value instanceof Error; } -function normalizeRequestError(error: unknown): Error { - if (isError(error)) { +function isRequestError(value: unknown): value is RequestError { + return value instanceof RequestError; +} + +function normalizeRequestError( + error: unknown, + request: NormalizedRequest, + attempt: number, + response?: Response +): RequestError { + if (error instanceof TimeoutError || error instanceof AbortError || error instanceof HTTPError) { + error.request ??= request; + error.response ??= response; + error.attempt ??= attempt; return error; } - return new RequestError(String(error), { cause: error }); + if (isRequestError(error)) { + error.request ??= request; + error.response ??= response; + error.attempt ??= attempt; + return error; + } + + if (isError(error)) { + return new RequestError(error.message, { + cause: error, + request, + response, + attempt, + }); + } + + return new RequestError(String(error), { + cause: error, + request, + response, + attempt, + }); +} + +function inferErrorCode(error: unknown): string | undefined { + if (error instanceof RequestError && error.code) { + return error.code; + } + + if (typeof error === 'object' && error !== null && 'code' in error) { + const code = (error as { code?: unknown }).code; + return typeof code === 'string' ? code : undefined; + } + + return undefined; } -async function dispatchNativeRequest(options: NativeRequestOptions): Promise { +function throwIfAborted(signal?: AbortSignal | null): void { + if (!signal?.aborted) { + return; + } + + throw new AbortError(undefined, { cause: signal.reason }); +} + +async function sleep(delayMs: number): Promise { + if (delayMs <= 0) { + return; + } + + await new Promise((resolve) => { + setTimeout(resolve, delayMs); + }); +} + +async function loadCookiesIntoRequest( + cookieJar: CookieJar | undefined, + request: NormalizedRequest +): Promise { + if (!cookieJar || request.headers.has('cookie')) { + return; + } + + const cookies = await cookieJar.getCookies(request.url); + if (cookies.length === 0) { + return; + } + + request.headers.set( + 'cookie', + cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ') + ); +} + +async function persistResponseCookies( + cookieJar: CookieJar | undefined, + requestUrl: string, + response: Response +): Promise { + if (!cookieJar) { + return; + } + + if (response.setCookies.length > 0) { + for (const cookie of response.setCookies) { + await cookieJar.setCookie(cookie, requestUrl); + } + + return; + } + + for (const [name, value] of Object.entries(response.cookies)) { + await cookieJar.setCookie(`${name}=${value}`, requestUrl); + } +} + +async function reportStats( + callback: WreqInit['onStats'] | undefined, + stats: RequestStats +): Promise { + if (!callback) { + return; + } + + await callback(stats); +} + +async function dispatchNativeRequest( + options: NativeRequestOptions, + startTime: number +): Promise { const nativeResponse = await nativeRequest(options).catch((error: unknown) => { const message = String(error); - if (message.toLowerCase().includes('timed out')) { - throw new RequestError(message, { code: 'ERR_TIMEOUT', cause: error }); + const lowered = message.toLowerCase(); + + if (lowered.includes('timed out') || lowered.includes('timeout')) { + throw new TimeoutError(message, { cause: error }); } + throw new RequestError(message, { cause: error }); }); - return new Response(nativeResponse); + const responseStart = Date.now(); + + return new Response({ + ...nativeResponse, + timings: { + startTime, + responseStart, + wait: responseStart - startTime, + }, + }); +} + +function isRedirectResponse(response: Response): boolean { + return REDIRECT_STATUS_CODES.has(response.status) && response.headers.has('location'); +} + +function resolveRedirectLocation(response: Response, requestUrl: string): string { + const location = response.headers.get('location'); + if (!location) { + throw new RequestError('Redirect response is missing Location header'); + } + + try { + return new URL(location, requestUrl).toString(); + } catch (error) { + throw new RequestError(`Invalid redirect URL: ${location}`, { cause: error }); + } +} + +function stripRedirectSensitiveHeaders( + headers: Headers, + previousUrl: string, + nextUrl: string, + bodyDropped: boolean +): void { + headers.delete('host'); + headers.delete('content-length'); + + if (bodyDropped) { + headers.delete('content-type'); + } + + if (new URL(previousUrl).origin !== new URL(nextUrl).origin) { + headers.delete('authorization'); + } +} + +function rewriteRedirectMethodAndBody( + method: HttpMethod, + status: number, + body: NormalizedRequest['body'] +): { + method: HttpMethod; + body: NormalizedRequest['body']; + bodyDropped: boolean; +} { + if (status === 303) { + return { + method: method === 'HEAD' ? 'HEAD' : 'GET', + body: undefined, + bodyDropped: true, + }; + } + + if ((status === 301 || status === 302) && method === 'POST') { + return { + method: 'GET', + body: undefined, + bodyDropped: true, + }; + } + + return { + method, + body, + bodyDropped: false, + }; +} + +async function shouldRetryRequest( + context: RetryDecisionContext, + retry: ResolvedRetryOptions +): Promise { + if (context.attempt > retry.limit + 1) { + return false; + } + + if (!retry.methods.includes(context.request.method)) { + return false; + } + + if (context.response) { + if (!retry.statusCodes.includes(context.response.status)) { + return false; + } + } else { + const code = inferErrorCode(context.error); + if (!code || !retry.errorCodes.includes(code)) { + return false; + } + } + + if (!retry.shouldRetry) { + return true; + } + + return retry.shouldRetry(context); +} + +async function runRetryDelay( + context: RetryDecisionContext, + retry: ResolvedRetryOptions +): Promise { + if (!retry.backoff) { + return; + } + + const delay = await retry.backoff(context); + await sleep(delay); +} + +function finalizeResponse(response: Response, redirectChain: RedirectEntry[]): Response { + return response.setRedirectMetadata(redirectChain); } export async function fetch(input: RequestInput, init?: WreqInit): Promise { - const state = (init?.context ? { ...init.context } : {}) as Record; const merged = await mergeInputAndInit(input, init); + const state = (merged.init.context ? { ...merged.init.context } : {}) as Record; + await runInitHooks(merged.init.hooks, { input, options: merged.init, state, }); - const { nativeRequest, options, request } = await normalizeInput(merged.urlInput, merged.init); + const options = resolveOptions(merged.init); + const request = createRequest(merged.urlInput, options); + const redirectChain: RedirectEntry[] = []; + const visitedRedirectTargets = new Set([request.url]); + + let attempt = 1; + + while (true) { + const requestBody = await normalizeBody(request.body); + const startTime = Date.now(); + + try { + throwIfAborted(options.signal); + await loadCookiesIntoRequest(options.cookieJar, request); + + const shortCircuit = await runBeforeRequestHooks(options.hooks, { + request, + options, + attempt, + state, + }); + + let response = + shortCircuit ?? + (await dispatchNativeRequest(buildNativeRequest(request, options, requestBody), startTime)); + + if (shortCircuit) { + response.setTimings({ + startTime, + responseStart: startTime, + wait: 0, + endTime: startTime, + total: 0, + }); + } - try { - const shortCircuit = await runBeforeRequestHooks(options.hooks, { - request, - options, - attempt: 1, - state, - }); + response = await runAfterResponseHooks(options.hooks, { + request, + options, + attempt, + state, + response, + }); + + await reportStats(options.onStats, { + request, + attempt, + timings: response.timings ?? { + startTime, + responseStart: startTime, + wait: 0, + }, + response, + }); + + await persistResponseCookies(options.cookieJar, request.url, response); + + if (isRedirectResponse(response)) { + if (options.redirect === 'manual') { + return finalizeResponse(response, redirectChain); + } + + if (options.redirect === 'error') { + throw new RequestError(`Redirect encountered for ${request.url}`, { + code: 'ERR_REDIRECT', + request, + response, + attempt, + }); + } + + if (redirectChain.length >= options.maxRedirects) { + throw new RequestError(`Maximum redirects exceeded: ${options.maxRedirects}`, { + code: 'ERR_TOO_MANY_REDIRECTS', + request, + response, + attempt, + }); + } + + const nextUrl = resolveRedirectLocation(response, request.url); + if (visitedRedirectTargets.has(nextUrl)) { + throw new RequestError(`Redirect loop detected for ${nextUrl}`, { + code: 'ERR_REDIRECT_LOOP', + request, + response, + attempt, + }); + } + + const rewritten = rewriteRedirectMethodAndBody( + request.method, + response.status, + request.body + ); + const nextRequest: NormalizedRequest = { + url: nextUrl, + method: rewritten.method, + headers: request.headers, + body: rewritten.body, + }; + + stripRedirectSensitiveHeaders( + nextRequest.headers, + request.url, + nextUrl, + rewritten.bodyDropped + ); + if (options.cookieJar) { + nextRequest.headers.delete('cookie'); + } + + const redirectEntry: RedirectEntry = { + fromUrl: request.url, + status: response.status, + location: response.headers.get('location') ?? nextUrl, + toUrl: nextUrl, + method: nextRequest.method, + }; + + await runBeforeRedirectHooks(options.hooks, { + request: nextRequest, + options, + attempt, + state, + response, + redirectCount: redirectChain.length + 1, + nextUrl, + nextMethod: nextRequest.method, + redirectChain: [...redirectChain, redirectEntry], + }); + + redirectChain.push({ + ...redirectEntry, + toUrl: nextRequest.url, + method: nextRequest.method, + }); + visitedRedirectTargets.add(nextRequest.url); + + request.url = nextRequest.url; + request.method = nextRequest.method; + request.body = nextRequest.body; + + continue; + } - let response = - shortCircuit ?? - (await dispatchNativeRequest({ - ...nativeRequest, - url: request.url, - method: request.method, - headers: request.headers.toObject(), - body: await normalizeBody(request.body), - proxy: options.proxy, - browser: options.browser, - timeout: options.timeout, - })); - - response = await runAfterResponseHooks(options.hooks, { - request, - options, - attempt: 1, - state, - response, - }); + const nextAttempt = attempt + 1; + const retryContext: RetryDecisionContext = { + request, + options, + attempt: nextAttempt, + state, + response, + }; + + if (await shouldRetryRequest(retryContext, options.retry)) { + const retryError = new HTTPError( + `Request failed with status ${response.status}`, + response.status, + { + request, + response, + attempt, + } + ); + + await runBeforeRetryHooks(options.hooks, { + request, + options, + attempt: nextAttempt, + state, + error: retryError, + response, + }); + + await runRetryDelay({ ...retryContext, error: retryError }, options.retry); + attempt = nextAttempt; + continue; + } - if (!isResponseStatusAllowed(response.status, options)) { - throw new HTTPError(`Request failed with status ${response.status}`, response.status); - } + if (!isResponseStatusAllowed(response.status, options)) { + throw new HTTPError(`Request failed with status ${response.status}`, response.status, { + request, + response, + attempt, + }); + } - return response; - } catch (error: unknown) { - const normalizedError = await runBeforeErrorHooks(options.hooks, { - request, - options, - attempt: 1, - state, - error: normalizeRequestError(error), - }); - throw normalizedError; + return finalizeResponse(response, redirectChain); + } catch (error: unknown) { + const normalizedError = normalizeRequestError(error, request, attempt); + const errorEndTime = Date.now(); + + await reportStats(options.onStats, { + request, + attempt, + timings: { + startTime, + responseStart: errorEndTime, + wait: errorEndTime - startTime, + endTime: errorEndTime, + total: errorEndTime - startTime, + }, + error: normalizedError, + response: normalizedError.response, + }); + + const nextAttempt = attempt + 1; + const retryContext: RetryDecisionContext = { + request, + options, + attempt: nextAttempt, + state, + error: normalizedError, + response: normalizedError.response, + }; + + if (await shouldRetryRequest(retryContext, options.retry)) { + await runBeforeRetryHooks(options.hooks, { + request, + options, + attempt: nextAttempt, + state, + error: normalizedError, + response: normalizedError.response, + }); + + await runRetryDelay(retryContext, options.retry); + attempt = nextAttempt; + continue; + } + + const finalError = await runBeforeErrorHooks(options.hooks, { + request, + options, + attempt, + state, + error: normalizedError, + }); + + throw finalError; + } } } diff --git a/src/hooks.ts b/src/hooks.ts index 3f93569..4d70fd8 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -1,11 +1,13 @@ +import type { Response } from './response'; import type { AfterResponseContext, BeforeErrorContext, + BeforeRedirectContext, + BeforeRetryContext, BeforeRequestContext, Hooks, InitContext, } from './types'; -import type { Response } from './response'; export function mergeHooks(base?: Hooks, override?: Hooks): Hooks | undefined { if (!base && !override) { @@ -18,6 +20,7 @@ export function mergeHooks(base?: Hooks, override?: Hooks): Hooks | undefined { afterResponse: [...(base?.afterResponse ?? []), ...(override?.afterResponse ?? [])], beforeRetry: [...(base?.beforeRetry ?? []), ...(override?.beforeRetry ?? [])], beforeError: [...(base?.beforeError ?? []), ...(override?.beforeError ?? [])], + beforeRedirect: [...(base?.beforeRedirect ?? []), ...(override?.beforeRedirect ?? [])], }; } @@ -29,7 +32,7 @@ export async function runInitHooks(hooks: Hooks | undefined, context: InitContex export async function runBeforeRequestHooks( hooks: Hooks | undefined, - context: BeforeRequestContext, + context: BeforeRequestContext ): Promise { for (const hook of hooks?.beforeRequest ?? []) { const result = await hook(context); @@ -44,7 +47,7 @@ export async function runBeforeRequestHooks( export async function runAfterResponseHooks( hooks: Hooks | undefined, - context: AfterResponseContext, + context: AfterResponseContext ): Promise { let current = context.response; @@ -61,7 +64,7 @@ export async function runAfterResponseHooks( export async function runBeforeErrorHooks( hooks: Hooks | undefined, - context: BeforeErrorContext, + context: BeforeErrorContext ): Promise { let current = context.error; @@ -74,3 +77,21 @@ export async function runBeforeErrorHooks( return current; } + +export async function runBeforeRetryHooks( + hooks: Hooks | undefined, + context: BeforeRetryContext +): Promise { + for (const hook of hooks?.beforeRetry ?? []) { + await hook(context); + } +} + +export async function runBeforeRedirectHooks( + hooks: Hooks | undefined, + context: BeforeRedirectContext +): Promise { + for (const hook of hooks?.beforeRedirect ?? []) { + await hook(context); + } +} diff --git a/src/native.ts b/src/native.ts index 5009b86..c374315 100644 --- a/src/native.ts +++ b/src/native.ts @@ -1,7 +1,29 @@ -import type { BrowserProfile, HttpMethod, NativeResponse, NativeRequestOptions } from './types'; +import type { + BrowserProfile, + HttpMethod, + NativeResponse, + NativeRequestOptions, + NativeWebSocketConnectOptions, + NativeWebSocketConnection, + NativeWebSocketReadResult, +} from './types'; type NativeBinding = { request: (options: NativeRequestOptions) => Promise; + websocketConnect: (options: NativeWebSocketConnectOptions) => Promise; + websocketRead: (handle: number) => Promise; + websocketSendText: (handle: number, text: string) => Promise; + websocketSendBinary: (handle: number, data: Buffer) => Promise; + websocketClose: (handle: number, code?: number, reason?: string) => Promise; + readBodyChunk: ( + handle: number, + size?: number + ) => Promise<{ + chunk: Buffer; + done: boolean; + }>; + readBodyAll: (handle: number) => Promise; + cancelBody: (handle: number) => boolean; getProfiles: () => string[]; }; @@ -28,7 +50,7 @@ function loadNativeBinding(): NativeBinding { if (!platformArch) { throw new Error( `Unsupported platform: ${platform}-${arch}. ` + - `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64`, + `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64` ); } @@ -43,7 +65,7 @@ function loadNativeBinding(): NativeBinding { throw new Error( `Failed to load native module for ${platform}-${arch}. ` + `Tried: ../rust/${binaryName} and ../rust/node-wreq.node. ` + - `Make sure the package is installed correctly and the native module is built for your platform.`, + `Make sure the package is installed correctly and the native module is built for your platform.` ); } } @@ -65,6 +87,50 @@ export async function nativeRequest(options: NativeRequestOptions): Promise { + return getBinding().readBodyChunk(handle, size); +} + +export async function nativeReadBodyAll(handle: number): Promise { + return getBinding().readBodyAll(handle); +} + +export function nativeCancelBody(handle: number): boolean { + return getBinding().cancelBody(handle); +} + +export async function nativeWebSocketConnect( + options: NativeWebSocketConnectOptions +): Promise { + return getBinding().websocketConnect(options); +} + +export async function nativeWebSocketRead(handle: number): Promise { + return getBinding().websocketRead(handle); +} + +export async function nativeWebSocketSendText(handle: number, text: string): Promise { + return getBinding().websocketSendText(handle, text); +} + +export async function nativeWebSocketSendBinary(handle: number, data: Uint8Array): Promise { + return getBinding().websocketSendBinary(handle, Buffer.from(data)); +} + +export async function nativeWebSocketClose( + handle: number, + code?: number, + reason?: string +): Promise { + return getBinding().websocketClose(handle, code, reason); +} + export function validateBrowserProfile(browser?: BrowserProfile): void { if (!browser) { return; diff --git a/src/node-wreq.ts b/src/node-wreq.ts index 1e2b6b4..74781bb 100644 --- a/src/node-wreq.ts +++ b/src/node-wreq.ts @@ -1,5 +1,5 @@ import { createClient } from './client'; -import { AbortError, HTTPError, RequestError, TimeoutError } from './errors'; +import { AbortError, HTTPError, RequestError, TimeoutError, WebSocketError } from './errors'; import { fetch } from './fetch'; import { Headers } from './headers'; import { getProfiles } from './native'; @@ -7,16 +7,27 @@ import { Response } from './response'; import type { AfterResponseContext, BeforeErrorContext, + BeforeRedirectContext, + BeforeRetryContext, BeforeRequestContext, BrowserProfile, ClientDefaults, CookieJar, + CookieJarCookie, Hooks, HttpMethod, InitContext, + RedirectEntry, + RedirectMode, + RequestStats, + RequestTimings, RequestInput, + RetryOptions, + WebSocketBinaryType, + WebSocketInit, WreqInit, } from './types'; +import { CloseEvent, WebSocket, websocket } from './websocket'; export { fetch, @@ -27,6 +38,10 @@ export { HTTPError, TimeoutError, AbortError, + WebSocket, + CloseEvent, + websocket, + WebSocketError, }; export { getProfiles }; @@ -34,14 +49,24 @@ export { getProfiles }; export type { AfterResponseContext, BeforeErrorContext, + BeforeRedirectContext, + BeforeRetryContext, BeforeRequestContext, BrowserProfile, ClientDefaults, CookieJar, + CookieJarCookie, Hooks, HttpMethod, InitContext, + RedirectEntry, + RedirectMode, + RequestStats, + RequestTimings, RequestInput, + RetryOptions, + WebSocketBinaryType, + WebSocketInit, WreqInit, }; @@ -55,4 +80,8 @@ export default { HTTPError, TimeoutError, AbortError, + WebSocket, + CloseEvent, + websocket, + WebSocketError, }; diff --git a/src/response.ts b/src/response.ts index 5a9328a..7836cc0 100644 --- a/src/response.ts +++ b/src/response.ts @@ -1,6 +1,8 @@ -import { Buffer } from 'node:buffer'; +import { Blob, Buffer } from 'node:buffer'; +import { ReadableStream } from 'node:stream/web'; import { Headers } from './headers'; -import type { NativeResponse } from './types'; +import { nativeCancelBody, nativeReadBodyAll, nativeReadBodyChunk } from './native'; +import type { NativeResponse, RedirectEntry, RequestTimings } from './types'; export class Response { readonly status: number; @@ -8,7 +10,15 @@ export class Response { readonly url: string; readonly headers: Headers; readonly cookies: Record; - private readonly payload: string; + readonly setCookies: string[]; + timings?: RequestTimings; + redirected: boolean; + redirectChain: RedirectEntry[]; + bodyUsed: boolean; + private payload: string | null; + private payloadBytes: Uint8Array | null; + private bodyHandle: number | null; + private stream: ReadableStream | null; constructor(nativeResponse: NativeResponse) { this.status = nativeResponse.status; @@ -16,24 +26,182 @@ export class Response { this.url = nativeResponse.url; this.headers = new Headers(nativeResponse.headers); this.cookies = { ...nativeResponse.cookies }; - this.payload = nativeResponse.body; + this.setCookies = [...(nativeResponse.setCookies ?? [])]; + this.timings = nativeResponse.timings ? { ...nativeResponse.timings } : undefined; + this.redirected = false; + this.redirectChain = []; + this.payload = nativeResponse.body ?? null; + this.payloadBytes = nativeResponse.body ? Buffer.from(nativeResponse.body, 'utf8') : null; + this.bodyHandle = nativeResponse.bodyHandle ?? null; + this.bodyUsed = false; + this.stream = null; } - get body(): string { + setRedirectMetadata(chain: RedirectEntry[]): this { + this.redirected = chain.length > 0; + this.redirectChain = [...chain]; + return this; + } + + setTimings(timings: RequestTimings): this { + this.timings = { ...timings }; + return this; + } + + private markBodyComplete(): void { + if (!this.timings || this.timings.endTime !== undefined) { + return; + } + + const endTime = Date.now(); + this.timings = { + ...this.timings, + endTime, + total: endTime - this.timings.startTime, + }; + } + + get body(): ReadableStream | null { + if (this.bodyUsed) { + return null; + } + + this.bodyUsed = true; + + if (this.payloadBytes) { + this.stream ??= new ReadableStream({ + start: (controller) => { + controller.enqueue(new Uint8Array(this.payloadBytes!)); + controller.close(); + }, + }); + + return this.stream; + } + + if (this.bodyHandle === null) { + return null; + } + + const handle = this.bodyHandle; + this.stream = new ReadableStream({ + pull: async (controller) => { + const result = await nativeReadBodyChunk(handle); + if (result.chunk.length > 0) { + controller.enqueue(new Uint8Array(result.chunk)); + } + + if (result.done) { + this.bodyHandle = null; + this.markBodyComplete(); + controller.close(); + } + }, + cancel: async () => { + this.bodyHandle = null; + nativeCancelBody(handle); + this.markBodyComplete(); + }, + }); + + return this.stream; + } + + get bodyText(): string { + if (this.payload === null) { + throw new TypeError('Response body text is not available synchronously'); + } + return this.payload; } + private async consumeBytes(): Promise { + if (this.bodyUsed) { + throw new TypeError('Response body is already used'); + } + + this.bodyUsed = true; + + if (this.payloadBytes) { + this.markBodyComplete(); + return new Uint8Array(this.payloadBytes); + } + + if (this.bodyHandle === null) { + this.markBodyComplete(); + return new Uint8Array(); + } + + const bytes = await nativeReadBodyAll(this.bodyHandle); + this.bodyHandle = null; + this.payloadBytes = new Uint8Array(bytes); + this.payload = Buffer.from(this.payloadBytes).toString('utf8'); + this.markBodyComplete(); + return new Uint8Array(this.payloadBytes); + } + + private getContentType(): string { + return this.headers.get('content-type')?.toLowerCase() ?? ''; + } + async text(): Promise { - return this.payload; + return Buffer.from(await this.consumeBytes()).toString('utf8'); } async json(): Promise { - return JSON.parse(this.payload) as T; + return JSON.parse(await this.text()) as T; } async arrayBuffer(): Promise { - const buffer = Buffer.from(this.payload, 'utf8'); + const buffer = Buffer.from(await this.consumeBytes()); return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); } + + async blob(): Promise { + return new Blob([await this.consumeBytes()]); + } + + async formData(): Promise { + const contentType = this.getContentType(); + + if (contentType.includes('application/x-www-form-urlencoded')) { + const formData = new FormData(); + const searchParams = new URLSearchParams(await this.text()); + + for (const [name, value] of searchParams) { + formData.append(name, value); + } + + return formData; + } + + if (contentType.includes('multipart/form-data')) { + throw new TypeError('multipart/form-data parsing is not implemented yet'); + } + + throw new TypeError(`Response content-type is not form data: ${contentType || 'unknown'}`); + } + + clone(): Response { + if (this.bodyUsed) { + throw new TypeError('Response body is already used'); + } + + if (!this.payloadBytes || this.payload === null) { + throw new TypeError('Cannot clone a native streaming response before it is buffered'); + } + + const cloned = new Response({ + status: this.status, + headers: this.headers.toObject(), + body: this.payload, + cookies: this.cookies, + setCookies: this.setCookies, + timings: this.timings, + url: this.url, + }); + + return cloned.setRedirectMetadata(this.redirectChain); + } } diff --git a/src/test/node-wreq.spec.ts b/src/test/node-wreq.spec.ts index 398233a..8445ee5 100644 --- a/src/test/node-wreq.spec.ts +++ b/src/test/node-wreq.spec.ts @@ -1,12 +1,212 @@ -import { test, describe, before } from 'node:test'; import assert from 'node:assert'; -import { Response as WreqResponse, createClient, fetch, getProfiles } from '../node-wreq'; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; +import { test, describe, before, after } from 'node:test'; +import { TextDecoder } from 'node:util'; +import { WebSocketServer, type WebSocket as WsPeer } from 'ws'; +import { + CloseEvent as WreqCloseEvent, + Response as WreqResponse, + WebSocket as WreqWebSocket, + createClient, + fetch, + getProfiles, + websocket, +} from '../node-wreq'; describe('node-wreq', () => { + let localBaseUrl = ''; + let localServer: Server | undefined; + let wsServer: WebSocketServer | undefined; + const retryAttempts = new Map(); + + function onceEvent(target: EventTarget, type: string): Promise { + return new Promise((resolve) => { + const listener = (event: Event) => { + resolve(event as T); + }; + + target.addEventListener(type, listener, { once: true }); + }); + } + + function readCookieHeader(request: IncomingMessage): string { + const cookie = request.headers.cookie; + if (Array.isArray(cookie)) { + return cookie.join('; '); + } + + return cookie ?? ''; + } + + function sendJson( + response: ServerResponse, + status: number, + body: unknown, + headers?: Record + ) { + response.writeHead(status, { + 'content-type': 'application/json', + ...headers, + }); + response.end(JSON.stringify(body)); + } + before(() => { console.log('🚀 node-wreq - Test Suite\n'); }); + before(async () => { + wsServer = new WebSocketServer({ + noServer: true, + perMessageDeflate: true, + handleProtocols(protocols: Set) { + if (protocols.has('chat')) { + return 'chat'; + } + + return false; + }, + }); + + wsServer.on('connection', (socket: WsPeer, request: IncomingMessage) => { + const cookie = readCookieHeader(request); + socket.send(JSON.stringify({ kind: 'connected', cookie, protocol: socket.protocol })); + + socket.on('message', (data: Buffer, isBinary: boolean) => { + if (!isBinary && data.toString() === 'close-me') { + socket.close(1000, 'done'); + return; + } + + socket.send(data, { binary: isBinary }); + }); + }); + + localServer = createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + + if (url.pathname === '/retry') { + const key = url.searchParams.get('key') ?? 'default'; + const failCount = Number(url.searchParams.get('failCount') ?? '0'); + const count = (retryAttempts.get(key) ?? 0) + 1; + retryAttempts.set(key, count); + + if (count <= failCount) { + sendJson(response, 503, { attempt: count, retried: false }); + return; + } + + sendJson(response, 200, { attempt: count, retried: count > 1 }); + return; + } + + if (url.pathname === '/timings/delay') { + setTimeout(() => { + sendJson(response, 200, { delayed: true }); + }, 50); + return; + } + + if (url.pathname === '/cookies/set') { + sendJson( + response, + 200, + { stored: true }, + { + 'set-cookie': 'session=abc123', + } + ); + return; + } + + if (url.pathname === '/cookies/set-multiple') { + sendJson( + response, + 200, + { stored: true }, + { + 'set-cookie': ['session=abc123; Path=/', 'csrf=token123; Path=/'], + } + ); + return; + } + + if (url.pathname === '/cookies/echo') { + sendJson(response, 200, { cookie: readCookieHeader(request) }); + return; + } + + if (url.pathname === '/redirect/start') { + response.writeHead(302, { + location: '/redirect/final', + 'set-cookie': 'redirect_session=1; Path=/', + }); + response.end(); + return; + } + + if (url.pathname === '/redirect/post-start') { + response.writeHead(302, { + location: '/redirect/final', + }); + response.end(); + return; + } + + if (url.pathname === '/redirect/final') { + sendJson(response, 200, { + method: request.method, + cookie: readCookieHeader(request), + hookHeader: request.headers['x-redirect-hook'] ?? '', + }); + return; + } + + sendJson(response, 404, { path: url.pathname }); + }); + + localServer.on('upgrade', (request, socket, head) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + if (url.pathname !== '/ws') { + socket.destroy(); + return; + } + + wsServer?.handleUpgrade(request, socket, head, (websocketSocket: WsPeer) => { + wsServer?.emit('connection', websocketSocket, request); + }); + }); + + await new Promise((resolve) => { + localServer?.listen(0, '127.0.0.1', () => { + const address = localServer?.address(); + if (!address || typeof address === 'string') { + throw new Error('Failed to bind local test server'); + } + + localBaseUrl = `http://127.0.0.1:${address.port}`; + resolve(); + }); + }); + }); + + after(async () => { + await new Promise((resolve) => { + wsServer?.close(() => resolve()); + }); + + await new Promise((resolve, reject) => { + localServer?.close((error) => { + if (error) { + reject(error); + return; + } + + resolve(); + }); + }); + }); + test('should return available browser profiles', () => { const profiles = getProfiles(); @@ -16,7 +216,7 @@ describe('node-wreq', () => { profiles.includes('chrome_137') || profiles.includes('firefox_139') || profiles.includes('safari_18'), - 'Should include standard browser profiles', + 'Should include standard browser profiles' ); console.log(' Available profiles:', profiles.join(', ')); @@ -30,7 +230,6 @@ describe('node-wreq', () => { assert.ok(response.status >= 200 && response.status < 300, 'Should return successful status'); assert.ok(Object.keys(response.headers).length > 0, 'Should have response headers'); - assert.ok(response.body.length > 0, 'Should have response body'); const body = await response.json<{ headers: Record }>(); assert.ok(body.headers['User-Agent'], 'Should have User-Agent header'); @@ -67,9 +266,9 @@ describe('node-wreq', () => { }); }, { - name: 'RequestError', + name: 'TimeoutError', }, - 'Should throw an error on timeout', + 'Should throw an error on timeout' ); }); @@ -102,7 +301,7 @@ describe('node-wreq', () => { assert.strictEqual( body.headers['X-Test-Client'], 'node-wreq', - 'Client defaults should be merged into outgoing requests', + 'Client defaults should be merged into outgoing requests' ); }); @@ -133,7 +332,7 @@ describe('node-wreq', () => { assert.strictEqual( body.headers['X-Hook-Header'], 'active', - 'beforeRequest should mutate headers', + 'beforeRequest should mutate headers' ); }); @@ -160,6 +359,127 @@ describe('node-wreq', () => { assert.strictEqual(body.replaced, true, 'Replaced response body should be returned'); }); + test('should expose fetch-style response body lifecycle', async () => { + const response = new WreqResponse({ + status: 200, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ streamed: true }), + cookies: {}, + setCookies: [], + url: 'https://local/body', + }); + + assert.strictEqual(response.bodyUsed, false, 'body should start unused'); + + const bodyStream = response.body; + assert.ok(bodyStream, 'body should expose a stream'); + assert.strictEqual(response.bodyUsed, true, 'accessing body should mark it used'); + + const reader = bodyStream?.getReader(); + const chunks: Uint8Array[] = []; + + while (reader) { + const result = await reader.read(); + if (result.done) { + break; + } + + chunks.push(result.value); + } + + const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + assert.strictEqual( + new TextDecoder().decode(merged), + JSON.stringify({ streamed: true }), + 'body stream should contain the response payload' + ); + + await assert.rejects( + async () => { + await response.text(); + }, + (error: unknown) => error instanceof TypeError && error.message.includes('already used'), + 'consumers should reject after the body is used' + ); + }); + + test('should support cloning unused responses', async () => { + const response = new WreqResponse({ + status: 200, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cloned: true }), + cookies: {}, + setCookies: [], + url: 'https://local/clone', + }); + + const cloned = response.clone(); + assert.notStrictEqual(cloned, response, 'clone should return a new response instance'); + assert.strictEqual(await cloned.text(), JSON.stringify({ cloned: true })); + assert.strictEqual(await response.text(), JSON.stringify({ cloned: true })); + }); + + test('should expose native-backed response streams for fetched responses', async () => { + const response = await fetch(`${localBaseUrl}/cookies/echo`); + const stream = response.body; + + assert.ok(stream, 'fetched response should expose a body stream'); + assert.strictEqual(response.bodyUsed, true, 'reading body stream should mark it used'); + + const reader = stream?.getReader(); + const chunks: Uint8Array[] = []; + while (reader) { + const result = await reader.read(); + if (result.done) { + break; + } + + chunks.push(result.value); + } + + const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + assert.ok( + new TextDecoder().decode(merged).includes('"cookie":""'), + 'streamed native response should contain the expected payload' + ); + }); + + test('should support formData for urlencoded responses', async () => { + const response = new WreqResponse({ + status: 200, + headers: { 'content-type': 'application/x-www-form-urlencoded; charset=utf-8' }, + body: 'alpha=1&beta=two&beta=three', + cookies: {}, + setCookies: [], + url: 'https://local/form', + }); + + const formData = await response.formData(); + assert.strictEqual(formData.get('alpha'), '1'); + assert.deepStrictEqual(formData.getAll('beta'), ['two', 'three']); + assert.strictEqual(response.bodyUsed, true, 'formData should consume the body'); + }); + + test('should reject unsupported multipart formData parsing for now', async () => { + const response = new WreqResponse({ + status: 200, + headers: { 'content-type': 'multipart/form-data; boundary=test' }, + body: '--test\r\ncontent-disposition: form-data; name="alpha"\r\n\r\n1\r\n--test--\r\n', + cookies: {}, + setCookies: [], + url: 'https://local/form', + }); + + await assert.rejects( + async () => { + await response.formData(); + }, + (error: unknown) => + error instanceof TypeError && error.message.includes('multipart/form-data parsing'), + 'multipart form parsing should fail explicitly until it is implemented' + ); + }); + test('should allow beforeError to rewrite thrown errors', async () => { await assert.rejects( async () => { @@ -179,25 +499,343 @@ describe('node-wreq', () => { }, (error: unknown) => error instanceof Error && error.message.includes('hooked: Request failed with status 418'), - 'beforeError should be able to rewrite the final error', + 'beforeError should be able to rewrite the final error' ); }); - test('should reject cookieJar until it is implemented', async () => { + test('should run beforeRetry hooks and retry retriable responses', async () => { + retryAttempts.set('status-retry', 0); + const hookAttempts: number[] = []; + + const response = await fetch(`${localBaseUrl}/retry?key=status-retry&failCount=2`, { + retry: { + limit: 2, + statusCodes: [503], + backoff: () => 0, + }, + hooks: { + beforeRetry: [ + ({ attempt, response: retryResponse }) => { + hookAttempts.push(attempt); + assert.strictEqual(retryResponse?.status, 503); + }, + ], + }, + }); + + assert.strictEqual(response.status, 200, 'request should eventually succeed'); + assert.deepStrictEqual( + hookAttempts, + [2, 3], + 'beforeRetry should run before each retry attempt' + ); + + const body = await response.json<{ attempt: number; retried: boolean }>(); + assert.strictEqual(body.attempt, 3, 'third attempt should be the successful response'); + assert.strictEqual(body.retried, true, 'server should observe retries'); + }); + + test('should integrate cookieJar across requests', async () => { + const jarStore = new Map(); + const cookieJar = { + getCookies: () => + [...jarStore.entries()].map(([name, value]) => ({ + name, + value, + })), + setCookie: (cookie: string) => { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + jarStore.set(name, value); + }, + }; + + const setResponse = await fetch(`${localBaseUrl}/cookies/set`, { + cookieJar, + }); + assert.strictEqual(setResponse.status, 200, 'cookie source request should succeed'); + assert.strictEqual( + jarStore.get('session'), + 'abc123', + 'cookieJar should persist response cookies' + ); + + const echoResponse = await fetch(`${localBaseUrl}/cookies/echo`, { + cookieJar, + }); + const body = await echoResponse.json<{ cookie: string }>(); + + assert.ok( + body.cookie.includes('session=abc123'), + 'stored cookies should be sent on the next request' + ); + }); + + test('should persist multiple raw set-cookie headers through cookieJar', async () => { + const rawCookies: string[] = []; + const cookieJar = { + getCookies: () => + rawCookies.map((cookie) => { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + return { name, value }; + }), + setCookie: (cookie: string) => { + rawCookies.push(cookie); + }, + }; + + const response = await fetch(`${localBaseUrl}/cookies/set-multiple`, { + cookieJar, + }); + + assert.strictEqual(response.status, 200, 'multiple set-cookie response should succeed'); + assert.deepStrictEqual( + rawCookies, + ['session=abc123; Path=/', 'csrf=token123; Path=/'], + 'cookieJar should receive each raw Set-Cookie value separately' + ); + + const echoResponse = await fetch(`${localBaseUrl}/cookies/echo`, { + cookieJar, + }); + const body = await echoResponse.json<{ cookie: string }>(); + + assert.ok(body.cookie.includes('session=abc123'), 'first cookie should be replayed'); + assert.ok(body.cookie.includes('csrf=token123'), 'second cookie should be replayed'); + }); + + test('should follow redirects manually and run beforeRedirect hooks', async () => { + const cookieJar = { + store: new Map(), + getCookies(this: { store: Map }) { + return [...this.store.entries()].map(([name, value]) => ({ name, value })); + }, + setCookie(this: { store: Map }, cookie: string) { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + this.store.set(name, value); + }, + }; + + const response = await fetch(`${localBaseUrl}/redirect/start`, { + cookieJar, + hooks: { + beforeRedirect: [ + ({ request, redirectCount, nextUrl }) => { + assert.strictEqual(redirectCount, 1); + assert.strictEqual(nextUrl, `${localBaseUrl}/redirect/final`); + request.headers.set('X-Redirect-Hook', 'active'); + }, + ], + }, + }); + + assert.strictEqual(response.status, 200, 'redirect chain should resolve to final response'); + assert.strictEqual(response.redirected, true, 'final response should be marked redirected'); + assert.strictEqual(response.redirectChain.length, 1, 'redirect chain should be recorded'); + + const body = await response.json<{ method: string; cookie: string; hookHeader: string }>(); + assert.strictEqual(body.method, 'GET', 'GET redirect should preserve method'); + assert.ok( + body.cookie.includes('redirect_session=1'), + 'intermediate set-cookie should affect the next redirect hop' + ); + assert.strictEqual( + body.hookHeader, + 'active', + 'beforeRedirect should be able to mutate next request' + ); + }); + + test('should rewrite POST to GET on 302 redirects', async () => { + const response = await fetch(`${localBaseUrl}/redirect/post-start`, { + method: 'POST', + body: 'hello', + }); + + const body = await response.json<{ method: string }>(); + assert.strictEqual(body.method, 'GET', '302 redirect from POST should be rewritten to GET'); + }); + + test('should support manual redirect mode', async () => { + const response = await fetch(`${localBaseUrl}/redirect/start`, { + redirect: 'manual', + }); + + assert.strictEqual( + response.status, + 302, + 'manual redirect mode should return the redirect response' + ); + assert.strictEqual( + response.headers.get('location'), + '/redirect/final', + 'manual redirect mode should expose Location' + ); + assert.strictEqual( + response.redirected, + false, + 'manual redirect response should not be marked redirected' + ); + }); + + test('should support redirect error mode', async () => { await assert.rejects( async () => { - await fetch('https://httpbin.org/get', { - browser: 'chrome_137', - cookieJar: { - getCookies: () => [], - setCookie: () => undefined, - }, + await fetch(`${localBaseUrl}/redirect/start`, { + redirect: 'error', }); }, (error: unknown) => error instanceof Error && - error.message.includes('cookieJar support is not implemented yet'), - 'cookieJar should fail explicitly until the runtime support lands', + error.message.includes('Redirect encountered') && + 'code' in (error as object), + 'redirect error mode should throw on first redirect response' + ); + }); + + test('should expose a WHATWG-like websocket helper and lifecycle', async () => { + const socket = await websocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { + protocols: 'chat', + }); + + assert.strictEqual(socket.readyState, WreqWebSocket.OPEN); + assert.strictEqual(socket.protocol, 'chat'); + + const connectedEvent = await onceEvent(socket, 'message'); + const connected = JSON.parse(String(connectedEvent.data)) as { + kind: string; + cookie: string; + protocol: string; + }; + assert.strictEqual(connected.kind, 'connected'); + assert.strictEqual(connected.protocol, 'chat'); + + const replyPromise = onceEvent(socket, 'message'); + socket.send('hello'); + const replyEvent = await replyPromise; + assert.strictEqual(replyEvent.data, 'hello'); + + const closePromise = onceEvent(socket, 'close'); + socket.close(1000, 'done'); + const closeEvent = await closePromise; + + assert.strictEqual(closeEvent.code, 1000); + assert.strictEqual(closeEvent.reason, 'done'); + assert.strictEqual(closeEvent.wasClean, true); + assert.strictEqual(socket.readyState, WreqWebSocket.CLOSED); + }); + + test('should support binary messages and arraybuffer binaryType', async () => { + const socket = new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { + binaryType: 'arraybuffer', + }); + + await onceEvent(socket, 'open'); + await onceEvent(socket, 'message'); + + const replyPromise = onceEvent(socket, 'message'); + socket.send(new Uint8Array([1, 2, 3])); + const replyEvent = await replyPromise; + + assert.ok(replyEvent.data instanceof ArrayBuffer); + assert.deepStrictEqual([...new Uint8Array(replyEvent.data)], [1, 2, 3]); + + const closePromise = onceEvent(socket, 'close'); + socket.close(1000, 'done'); + await closePromise; + }); + + test('should send cookieJar cookies during websocket handshake', async () => { + const cookieJar = { + getCookies: () => [{ name: 'session', value: 'ws123' }], + setCookie: () => {}, + }; + + const socket = await websocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { + cookieJar, + }); + + const connectedEvent = await onceEvent(socket, 'message'); + const connected = JSON.parse(String(connectedEvent.data)) as { cookie: string }; + assert.ok( + connected.cookie.includes('session=ws123'), + 'cookieJar cookies should be sent during the websocket handshake' ); + + const closePromise = onceEvent(socket, 'close'); + socket.close(1000, 'done'); + await closePromise; + }); + + test('should reject websocket URLs with fragments', () => { + assert.throws( + () => { + new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws#fragment'); + }, + (error: unknown) => error instanceof DOMException && error.name === 'SyntaxError', + 'fragment websocket URLs should be rejected' + ); + }); + + test('should reject forbidden websocket headers and duplicate protocols', () => { + assert.throws( + () => { + new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { + headers: { + Upgrade: 'websocket', + }, + }); + }, + (error: unknown) => error instanceof DOMException && error.name === 'SyntaxError', + 'forbidden managed websocket headers should be rejected' + ); + + assert.throws( + () => { + new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { + protocols: ['chat', 'chat'], + }); + }, + (error: unknown) => + error instanceof SyntaxError && error.message.includes('Duplicate WebSocket subprotocol'), + 'duplicate websocket subprotocols should be rejected' + ); + }); + + test('should expose negotiated websocket extensions as a string', async () => { + const socket = await websocket(localBaseUrl.replace('http://', 'ws://') + '/ws'); + assert.strictEqual(typeof socket.extensions, 'string'); + + const closePromise = onceEvent(socket, 'close'); + socket.close(1000, 'done'); + await closePromise; + }); + + test('should expose response timings and onStats callback data', async () => { + let capturedStats: + | { + attempt: number; + wait: number; + status?: number; + } + | undefined; + + const response = await fetch(`${localBaseUrl}/timings/delay`, { + onStats: ({ attempt, timings, response: statsResponse }) => { + capturedStats = { + attempt, + wait: timings.wait, + status: statsResponse?.status, + }; + }, + }); + + assert.ok(response.timings, 'response should expose timings'); + assert.ok((response.timings?.wait ?? 0) >= 20, 'timings should record server wait time'); + assert.strictEqual(capturedStats?.attempt, 1); + assert.strictEqual(capturedStats?.status, 200); + assert.ok((capturedStats?.wait ?? 0) >= 20, 'onStats should receive request timings'); }); }); diff --git a/src/types.ts b/src/types.ts index 9fcb3ea..7c5e1b8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -115,6 +115,64 @@ export interface CookieJar { setCookie(cookie: string, url: string): Promise | void; } +export interface RetryDecisionContext { + request: NormalizedRequest; + options: ResolvedOptions; + attempt: number; + state: HookState; + error?: unknown; + response?: import('./response').Response; +} + +export interface RetryOptions { + limit?: number; + methods?: HttpMethod[]; + statusCodes?: number[]; + errorCodes?: string[]; + backoff?: (ctx: RetryDecisionContext) => number | Promise; + shouldRetry?: (ctx: RetryDecisionContext) => boolean | Promise; +} + +export type RedirectMode = 'follow' | 'manual' | 'error'; + +export interface RedirectEntry { + fromUrl: string; + status: number; + location: string; + toUrl: string; + method: HttpMethod; +} + +export type WebSocketBinaryType = 'blob' | 'arraybuffer'; + +export interface RequestTimings { + startTime: number; + responseStart: number; + wait: number; + endTime?: number; + total?: number; +} + +export interface RequestStats { + request: NormalizedRequest; + attempt: number; + timings: RequestTimings; + response?: import('./response').Response; + error?: Error; +} + +export interface WebSocketInit { + headers?: HeadersInit; + baseURL?: string; + query?: Record; + browser?: BrowserProfile; + proxy?: string; + timeout?: number; + cookieJar?: CookieJar; + protocols?: string | string[]; + binaryType?: WebSocketBinaryType; +} + export interface WreqInit { method?: string; headers?: HeadersInit; @@ -125,11 +183,15 @@ export interface WreqInit { browser?: BrowserProfile; proxy?: string; timeout?: number; + retry?: number | RetryOptions; + redirect?: RedirectMode; + maxRedirects?: number; cookieJar?: CookieJar; throwHttpErrors?: boolean; validateStatus?: (status: number) => boolean; disableDefaultHeaders?: boolean; compress?: boolean; + onStats?: (stats: RequestStats) => void | Promise; context?: Record; hooks?: Hooks; } @@ -141,8 +203,32 @@ export interface NormalizedRequest { body?: BodyInit | null; } -export interface ResolvedOptions extends Omit { +export interface ResolvedRetryOptions { + limit: number; + methods: HttpMethod[]; + statusCodes: number[]; + errorCodes: string[]; + backoff?: RetryOptions['backoff']; + shouldRetry?: RetryOptions['shouldRetry']; +} + +export interface ResolvedOptions extends Omit< + WreqInit, + | 'headers' + | 'retry' + | 'throwHttpErrors' + | 'disableDefaultHeaders' + | 'compress' + | 'redirect' + | 'maxRedirects' +> { headers: import('./headers').Headers; + retry: ResolvedRetryOptions; + throwHttpErrors: boolean; + disableDefaultHeaders: boolean; + compress: boolean; + redirect: RedirectMode; + maxRedirects: number; } export interface InitContext { @@ -166,29 +252,40 @@ export interface AfterResponseContext extends BaseHookContext { export interface BeforeRetryContext extends BaseHookContext { error: unknown; + response?: import('./response').Response; } export interface BeforeErrorContext extends BaseHookContext { error: Error; } +export interface BeforeRedirectContext extends BaseHookContext { + response: import('./response').Response; + redirectCount: number; + nextUrl: string; + nextMethod: HttpMethod; + redirectChain: RedirectEntry[]; +} + export interface Hooks { init?: InitHook[]; beforeRequest?: BeforeRequestHook[]; afterResponse?: AfterResponseHook[]; beforeRetry?: BeforeRetryHook[]; beforeError?: BeforeErrorHook[]; + beforeRedirect?: BeforeRedirectHook[]; } export type InitHook = (ctx: InitContext) => void | Promise; export type BeforeRequestHook = ( - ctx: BeforeRequestContext, + ctx: BeforeRequestContext ) => void | import('./response').Response | Promise; export type AfterResponseHook = ( - ctx: AfterResponseContext, + ctx: AfterResponseContext ) => void | import('./response').Response | Promise; export type BeforeRetryHook = (ctx: BeforeRetryContext) => void | Promise; export type BeforeErrorHook = (ctx: BeforeErrorContext) => Error | void | Promise; +export type BeforeRedirectHook = (ctx: BeforeRedirectContext) => void | Promise; export interface NativeRequestOptions { url: string; @@ -203,11 +300,46 @@ export interface NativeRequestOptions { export interface NativeResponse { status: number; headers: Record; - body: string; + body?: string; + bodyHandle?: number; cookies: Record; + setCookies?: string[]; + timings?: RequestTimings; url: string; } +export interface NativeWebSocketConnectOptions { + url: string; + headers: Record; + browser?: BrowserProfile; + proxy?: string; + timeout?: number; + protocols: string[]; +} + +export interface NativeWebSocketConnection { + handle: number; + url: string; + protocol: string | null; + extensions: string | null; +} + +export type NativeWebSocketReadResult = + | { + type: 'text'; + data: string; + } + | { + type: 'binary'; + data: Uint8Array; + } + | { + type: 'close'; + code: number; + reason: string; + wasClean: boolean; + }; + export interface ClientDefaults extends Omit { headers?: HeadersInit; hooks?: Hooks; @@ -216,11 +348,12 @@ export interface ClientDefaults extends Omit; + websocket(input: string | URL, init?: WebSocketInit): Promise; get(input: RequestInput, init?: Omit): Promise; post( input: RequestInput, body?: BodyInit | null, - init?: Omit, + init?: Omit ): Promise; extend(defaults: ClientDefaults): Client; } diff --git a/src/websocket.ts b/src/websocket.ts new file mode 100644 index 0000000..efaddaf --- /dev/null +++ b/src/websocket.ts @@ -0,0 +1,502 @@ +import { Buffer } from 'node:buffer'; +import { WebSocketError } from './errors'; +import { Headers } from './headers'; +import { + nativeWebSocketClose, + nativeWebSocketConnect, + nativeWebSocketRead, + nativeWebSocketSendBinary, + nativeWebSocketSendText, + validateBrowserProfile, +} from './native'; +import type { + CookieJar, + HeadersInit, + NativeWebSocketReadResult, + WebSocketBinaryType, + WebSocketInit, +} from './types'; + +const SUBPROTOCOL_PATTERN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; +const DEFAULT_TIMEOUT = 30_000; +const FORBIDDEN_WEBSOCKET_HEADERS = new Set([ + 'connection', + 'sec-websocket-extensions', + 'sec-websocket-key', + 'sec-websocket-protocol', + 'sec-websocket-version', + 'upgrade', +]); + +function appendQuery(url: URL, query: WebSocketInit['query']): void { + if (!query) { + return; + } + + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) { + continue; + } + + url.searchParams.set(key, String(value)); + } +} + +function resolveWebSocketUrl(rawUrl: string | URL, init?: WebSocketInit): string { + const url = init?.baseURL ? new URL(String(rawUrl), init.baseURL) : new URL(String(rawUrl)); + appendQuery(url, init?.query); + + if (url.protocol !== 'ws:' && url.protocol !== 'wss:') { + throw new WebSocketError(`Invalid WebSocket URL protocol: ${url.protocol}`); + } + + if (url.hash) { + throw new DOMException('WebSocket URL must not include a fragment', 'SyntaxError'); + } + + return url.toString(); +} + +function normalizeHeaders(headers?: HeadersInit): Headers { + const normalized = headers instanceof Headers ? new Headers(headers) : new Headers(headers); + + for (const [name] of normalized) { + if (FORBIDDEN_WEBSOCKET_HEADERS.has(name.toLowerCase())) { + throw new DOMException(`Forbidden WebSocket header: ${name}`, 'SyntaxError'); + } + } + + return normalized; +} + +function normalizeProtocols(protocols?: string | string[]): string[] { + if (!protocols) { + return []; + } + + const values = Array.isArray(protocols) ? protocols : [protocols]; + const seen = new Set(); + + for (const value of values) { + if (!SUBPROTOCOL_PATTERN.test(value)) { + throw new SyntaxError(`Invalid WebSocket subprotocol: ${value}`); + } + + if (seen.has(value)) { + throw new SyntaxError(`Duplicate WebSocket subprotocol: ${value}`); + } + + seen.add(value); + } + + return values; +} + +async function loadCookiesIntoHeaders( + cookieJar: CookieJar | undefined, + url: string, + headers: Headers +) { + if (!cookieJar || headers.has('cookie')) { + return; + } + + const cookies = await cookieJar.getCookies(url); + if (cookies.length === 0) { + return; + } + + headers.set('cookie', cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ')); +} + +function validateCloseCode(code: number): void { + if (code === 1000) { + return; + } + + if (code >= 3000 && code <= 4999) { + return; + } + + throw new DOMException(`Invalid WebSocket close code: ${code}`, 'InvalidAccessError'); +} + +function validateCloseReason(reason: string): void { + if (Buffer.byteLength(reason) > 123) { + throw new DOMException('WebSocket close reason must be 123 bytes or fewer', 'SyntaxError'); + } +} + +async function normalizeSendData(data: string | Blob | ArrayBuffer | ArrayBufferView): Promise< + | { + type: 'text'; + data: string; + } + | { + type: 'binary'; + data: Uint8Array; + } +> { + if (typeof data === 'string') { + return { + type: 'text', + data, + }; + } + + if (data instanceof Blob) { + return { + type: 'binary', + data: new Uint8Array(await data.arrayBuffer()), + }; + } + + if (ArrayBuffer.isView(data)) { + return { + type: 'binary', + data: new Uint8Array(data.buffer, data.byteOffset, data.byteLength), + }; + } + + if (data instanceof ArrayBuffer) { + return { + type: 'binary', + data: new Uint8Array(data), + }; + } + + throw new TypeError('Unsupported WebSocket message type'); +} + +function toMessageEventData( + result: NativeWebSocketReadResult, + binaryType: WebSocketBinaryType +): unknown { + switch (result.type) { + case 'text': { + return result.data; + } + case 'binary': { + if (binaryType === 'arraybuffer') { + const bytes = result.data; + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + } + + return new Blob([result.data]); + } + case 'close': { + throw new TypeError('Close frames cannot be converted to message events'); + } + } +} + +type OpenHandler = ((event: Event) => void) | null; +type MessageHandler = ((event: MessageEvent) => void) | null; +type CloseHandler = ((event: CloseEvent) => void) | null; +type ErrorHandler = ((event: Event) => void) | null; + +export class CloseEvent extends Event { + readonly code: number; + readonly reason: string; + readonly wasClean: boolean; + + constructor(type: string, init?: { code?: number; reason?: string; wasClean?: boolean }) { + super(type); + this.code = init?.code ?? 1005; + this.reason = init?.reason ?? ''; + this.wasClean = init?.wasClean ?? false; + } +} + +export class WebSocket extends EventTarget { + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSING = 2; + static readonly CLOSED = 3; + + readonly CONNECTING = WebSocket.CONNECTING; + readonly OPEN = WebSocket.OPEN; + readonly CLOSING = WebSocket.CLOSING; + readonly CLOSED = WebSocket.CLOSED; + + readonly url: string; + readonly extensions: string; + readonly opened: Promise; + #resolveOpened!: () => void; + #rejectOpened!: (reason?: unknown) => void; + #readyState = WebSocket.CONNECTING; + #handle?: number; + #protocol = ''; + #binaryType: WebSocketBinaryType; + #sendQueue = Promise.resolve(); + #settled = false; + #onopen: OpenHandler = null; + #onmessage: MessageHandler = null; + #onclose: CloseHandler = null; + #onerror: ErrorHandler = null; + + constructor(url: string | URL, init: WebSocketInit = {}) { + super(); + + this.url = resolveWebSocketUrl(url, init); + validateBrowserProfile(init.browser); + const headers = normalizeHeaders(init.headers); + const protocols = normalizeProtocols(init.protocols); + if (protocols.length > 0 && headers.has('sec-websocket-protocol')) { + throw new DOMException( + 'Do not provide sec-websocket-protocol header when protocols option is used', + 'SyntaxError' + ); + } + this.extensions = ''; + this.#binaryType = init.binaryType ?? 'blob'; + this.opened = new Promise((resolve, reject) => { + this.#resolveOpened = resolve; + this.#rejectOpened = reject; + }); + + void this.#connect(init, headers, protocols); + } + + get readyState(): number { + return this.#readyState; + } + + get protocol(): string { + return this.#protocol; + } + + get binaryType(): WebSocketBinaryType { + return this.#binaryType; + } + + set binaryType(value: WebSocketBinaryType) { + if (value !== 'blob' && value !== 'arraybuffer') { + throw new TypeError(`Invalid WebSocket binaryType: ${value}`); + } + + this.#binaryType = value; + } + + get bufferedAmount(): number { + return 0; + } + + get onopen(): OpenHandler { + return this.#onopen; + } + + set onopen(handler: OpenHandler) { + this.#setEventHandler('open', this.#onopen, handler); + this.#onopen = handler; + } + + get onmessage(): MessageHandler { + return this.#onmessage; + } + + set onmessage(handler: MessageHandler) { + this.#setEventHandler('message', this.#onmessage, handler); + this.#onmessage = handler; + } + + get onclose(): CloseHandler { + return this.#onclose; + } + + set onclose(handler: CloseHandler) { + this.#setEventHandler('close', this.#onclose, handler); + this.#onclose = handler; + } + + get onerror(): ErrorHandler { + return this.#onerror; + } + + set onerror(handler: ErrorHandler) { + this.#setEventHandler('error', this.#onerror, handler); + this.#onerror = handler; + } + + send(data: string | Blob | ArrayBuffer | ArrayBufferView): void { + if (this.#readyState !== WebSocket.OPEN || this.#handle === undefined) { + throw new DOMException('WebSocket is not open', 'InvalidStateError'); + } + + this.#sendQueue = this.#sendQueue + .then(async () => { + const normalized = await normalizeSendData(data); + if (this.#readyState !== WebSocket.OPEN || this.#handle === undefined) { + throw new DOMException('WebSocket is not open', 'InvalidStateError'); + } + + if (normalized.type === 'text') { + await nativeWebSocketSendText(this.#handle, normalized.data); + return; + } + + await nativeWebSocketSendBinary(this.#handle, normalized.data); + }) + .catch((error: unknown) => { + this.#handleError(error); + }); + } + + close(code?: number, reason = ''): void { + if (code !== undefined) { + validateCloseCode(code); + } + + validateCloseReason(reason); + + if (this.#readyState === WebSocket.CLOSING || this.#readyState === WebSocket.CLOSED) { + return; + } + + if (this.#readyState === WebSocket.CONNECTING) { + this.#readyState = WebSocket.CLOSING; + } else { + this.#readyState = WebSocket.CLOSING; + } + + if (this.#handle === undefined) { + return; + } + + const handle = this.#handle; + this.#handle = undefined; + + void nativeWebSocketClose(handle, code, reason) + .then(() => { + this.#finalizeClose({ + code: code ?? 1000, + reason, + wasClean: true, + }); + }) + .catch((error: unknown) => { + this.#handleError(error); + this.#finalizeClose({ + code: code ?? 1006, + reason, + wasClean: false, + }); + }); + } + + async #connect(init: WebSocketInit, headers: Headers, protocols: string[]): Promise { + await loadCookiesIntoHeaders(init.cookieJar, this.url, headers); + + try { + const connection = await nativeWebSocketConnect({ + url: this.url, + headers: headers.toObject(), + browser: init.browser, + proxy: init.proxy, + timeout: init.timeout ?? DEFAULT_TIMEOUT, + protocols, + }); + + this.#handle = connection.handle; + this.#protocol = connection.protocol ?? ''; + if (connection.protocol && protocols.length > 0 && !protocols.includes(connection.protocol)) { + throw new WebSocketError(`Server selected unexpected subprotocol: ${connection.protocol}`); + } + + (this as { extensions: string }).extensions = connection.extensions ?? ''; + this.#readyState = WebSocket.OPEN; + this.#resolveOpened(); + this.dispatchEvent(new Event('open')); + void this.#pumpMessages(); + } catch (error) { + this.#handleError(error); + this.#finalizeClose({ + code: 1006, + reason: '', + wasClean: false, + }); + } + } + + async #pumpMessages(): Promise { + while (this.#readyState === WebSocket.OPEN && this.#handle !== undefined) { + try { + const result = await nativeWebSocketRead(this.#handle); + if (result.type === 'close') { + this.#handle = undefined; + this.#finalizeClose(result); + return; + } + + this.dispatchEvent( + new MessageEvent('message', { + data: toMessageEventData(result, this.#binaryType), + }) + ); + } catch (error) { + this.#handleError(error); + this.#handle = undefined; + this.#finalizeClose({ + code: 1006, + reason: '', + wasClean: false, + }); + return; + } + } + } + + #setEventHandler( + type: string, + current: ((event: any) => void) | null, + next: ((event: any) => void) | null + ): void { + if (current) { + this.removeEventListener(type, current); + } + + if (next) { + this.addEventListener(type, next); + } + } + + #handleError(error: unknown): void { + const event = new Event('error'); + Object.defineProperty(event, 'error', { + configurable: true, + enumerable: false, + value: error instanceof Error ? error : new WebSocketError(String(error)), + writable: false, + }); + + if (this.#readyState === WebSocket.CONNECTING) { + this.#rejectOpened(error); + } + + this.dispatchEvent(event); + } + + #finalizeClose(init: { code: number; reason: string; wasClean: boolean }): void { + if (this.#settled) { + return; + } + + this.#settled = true; + this.#readyState = WebSocket.CLOSED; + if (this.#handle !== undefined) { + this.#handle = undefined; + } + + if (init.wasClean === false) { + this.#rejectOpened(new WebSocketError('WebSocket connection closed before opening')); + } + + this.dispatchEvent(new CloseEvent('close', init)); + } +} + +export async function websocket(url: string | URL, init?: WebSocketInit): Promise { + const socket = new WebSocket(url, init); + await socket.opened; + return socket; +} diff --git a/src/ws.d.ts b/src/ws.d.ts new file mode 100644 index 0000000..9ebbfbe --- /dev/null +++ b/src/ws.d.ts @@ -0,0 +1,28 @@ +declare module 'ws' { + import { EventEmitter } from 'node:events'; + import type { IncomingMessage } from 'node:http'; + import type { Duplex } from 'node:stream'; + + export class WebSocket extends EventEmitter { + protocol: string; + send(data: string | Uint8Array | Buffer, options?: { binary?: boolean }): void; + close(code?: number, reason?: string): void; + on(event: 'message', listener: (data: Buffer, isBinary: boolean) => void): this; + } + + export class WebSocketServer extends EventEmitter { + constructor(options?: { + noServer?: boolean; + perMessageDeflate?: boolean; + handleProtocols?: (protocols: Set) => string | false; + }); + handleUpgrade( + request: IncomingMessage, + socket: Duplex, + head: Buffer, + callback: (socket: WebSocket) => void + ): void; + close(callback?: () => void): void; + on(event: 'connection', listener: (socket: WebSocket, request: IncomingMessage) => void): this; + } +} From a3b5f1be5fc57e2e4f2b08347147a3369f49d101 Mon Sep 17 00:00:00 2001 From: ruby Date: Sun, 12 Apr 2026 07:46:06 +0400 Subject: [PATCH 03/15] feat: brand new client api; hooks; refactor --- .github/ISSUE_TEMPLATE/bug_report.yml | 5 +- .github/workflows/build.yml | 72 +- .gitignore | 3 +- .oxlintrc.json | 42 +- README.md | 804 ++++++++++++++---- docs/ARCHITECTURE.md | 20 +- docs/PUBLISHING.md | 87 +- package-lock.json | 992 +++++++++++++++++++++++ package.json | 26 +- rust/Cargo.lock | 530 +++++++----- rust/Cargo.toml | 5 +- rust/src/client.rs | 511 ------------ rust/src/emulation/builders.rs | 346 ++++++++ rust/src/emulation/mod.rs | 6 + rust/src/emulation/parse.rs | 69 ++ rust/src/emulation/payload.rs | 163 ++++ rust/src/emulation/resolve.rs | 39 + rust/src/lib.rs | 582 +------------ rust/src/napi/body.rs | 63 ++ rust/src/napi/convert.rs | 275 +++++++ rust/src/napi/mod.rs | 15 + rust/src/napi/profiles.rs | 56 ++ rust/src/napi/request.rs | 27 + rust/src/napi/websocket.rs | 150 ++++ rust/src/store/body_store.rs | 82 ++ rust/src/store/mod.rs | 3 + rust/src/store/runtime.rs | 8 + rust/src/store/websocket_store.rs | 112 +++ rust/src/transport/cookies.rs | 6 + rust/src/transport/headers.rs | 9 + rust/src/transport/mod.rs | 8 + rust/src/transport/request.rs | 102 +++ rust/src/transport/types.rs | 57 ++ rust/src/transport/websocket.rs | 196 +++++ scripts/generate-browser-profiles.mjs | 75 ++ scripts/platform-targets.mjs | 57 ++ scripts/postbuild.mjs | 47 ++ scripts/prepare-main-package.mjs | 55 ++ scripts/prepare-platform-package.mjs | 98 +++ src/{client.ts => client/index.ts} | 15 +- src/config/emulation.ts | 36 + src/config/generated/browser-profiles.ts | 115 +++ src/{errors.ts => errors/index.ts} | 8 +- src/fetch.ts | 724 ----------------- src/generated/browser-profiles.ts | 90 ++ src/{headers.ts => headers/index.ts} | 38 +- src/{hooks.ts => hooks/index.ts} | 5 +- src/http/body/bytes.ts | 37 + src/http/body/form-data.ts | 79 ++ src/http/fetch.ts | 276 +++++++ src/http/pipeline/cookies.ts | 64 ++ src/http/pipeline/dispatch.ts | 42 + src/http/pipeline/errors.ts | 88 ++ src/http/pipeline/input.ts | 61 ++ src/http/pipeline/options.ts | 101 +++ src/http/pipeline/redirects.ts | 96 +++ src/http/pipeline/retries.ts | 57 ++ src/http/request.ts | 162 ++++ src/http/response-meta.ts | 41 + src/http/response.ts | 339 ++++++++ src/index.ts | 141 ++++ src/native.ts | 157 ---- src/native/binding.ts | 142 ++++ src/native/index.ts | 10 + src/native/methods.ts | 17 + src/native/profiles.ts | 20 + src/native/request.ts | 24 + src/native/websocket.ts | 33 + src/node-wreq.ts | 89 +- src/{ => platform}/ws.d.ts | 0 src/response.ts | 207 ----- src/test/cookies-redirects.spec.ts | 183 +++++ src/test/helpers/local-server.ts | 234 ++++++ src/test/hooks-retries.spec.ts | 143 ++++ src/test/http-client.spec.ts | 166 ++++ src/test/node-wreq.spec.ts | 846 +------------------ src/test/response.spec.ts | 298 +++++++ src/test/websocket.spec.ts | 177 ++++ src/types.ts | 360 +------- src/types/client.ts | 27 + src/types/hooks.ts | 64 ++ src/types/http.ts | 119 +++ src/types/index.ts | 8 + src/types/native.ts | 27 + src/types/shared.ts | 126 +++ src/types/websocket.ts | 63 ++ src/websocket/close-event.ts | 12 + src/{websocket.ts => websocket/index.ts} | 247 +----- src/websocket/send-data.ts | 88 ++ src/websocket/validation.ts | 97 +++ 90 files changed, 8311 insertions(+), 4091 deletions(-) delete mode 100644 rust/src/client.rs create mode 100644 rust/src/emulation/builders.rs create mode 100644 rust/src/emulation/mod.rs create mode 100644 rust/src/emulation/parse.rs create mode 100644 rust/src/emulation/payload.rs create mode 100644 rust/src/emulation/resolve.rs create mode 100644 rust/src/napi/body.rs create mode 100644 rust/src/napi/convert.rs create mode 100644 rust/src/napi/mod.rs create mode 100644 rust/src/napi/profiles.rs create mode 100644 rust/src/napi/request.rs create mode 100644 rust/src/napi/websocket.rs create mode 100644 rust/src/store/body_store.rs create mode 100644 rust/src/store/mod.rs create mode 100644 rust/src/store/runtime.rs create mode 100644 rust/src/store/websocket_store.rs create mode 100644 rust/src/transport/cookies.rs create mode 100644 rust/src/transport/headers.rs create mode 100644 rust/src/transport/mod.rs create mode 100644 rust/src/transport/request.rs create mode 100644 rust/src/transport/types.rs create mode 100644 rust/src/transport/websocket.rs create mode 100644 scripts/generate-browser-profiles.mjs create mode 100644 scripts/platform-targets.mjs create mode 100644 scripts/postbuild.mjs create mode 100644 scripts/prepare-main-package.mjs create mode 100644 scripts/prepare-platform-package.mjs rename src/{client.ts => client/index.ts} (94%) create mode 100644 src/config/emulation.ts create mode 100644 src/config/generated/browser-profiles.ts rename src/{errors.ts => errors/index.ts} (90%) delete mode 100644 src/fetch.ts create mode 100644 src/generated/browser-profiles.ts rename src/{headers.ts => headers/index.ts} (74%) rename src/{hooks.ts => hooks/index.ts} (97%) create mode 100644 src/http/body/bytes.ts create mode 100644 src/http/body/form-data.ts create mode 100644 src/http/fetch.ts create mode 100644 src/http/pipeline/cookies.ts create mode 100644 src/http/pipeline/dispatch.ts create mode 100644 src/http/pipeline/errors.ts create mode 100644 src/http/pipeline/input.ts create mode 100644 src/http/pipeline/options.ts create mode 100644 src/http/pipeline/redirects.ts create mode 100644 src/http/pipeline/retries.ts create mode 100644 src/http/request.ts create mode 100644 src/http/response-meta.ts create mode 100644 src/http/response.ts create mode 100644 src/index.ts delete mode 100644 src/native.ts create mode 100644 src/native/binding.ts create mode 100644 src/native/index.ts create mode 100644 src/native/methods.ts create mode 100644 src/native/profiles.ts create mode 100644 src/native/request.ts create mode 100644 src/native/websocket.ts rename src/{ => platform}/ws.d.ts (100%) delete mode 100644 src/response.ts create mode 100644 src/test/cookies-redirects.spec.ts create mode 100644 src/test/helpers/local-server.ts create mode 100644 src/test/hooks-retries.spec.ts create mode 100644 src/test/http-client.spec.ts create mode 100644 src/test/response.spec.ts create mode 100644 src/test/websocket.spec.ts create mode 100644 src/types/client.ts create mode 100644 src/types/hooks.ts create mode 100644 src/types/http.ts create mode 100644 src/types/index.ts create mode 100644 src/types/native.ts create mode 100644 src/types/shared.ts create mode 100644 src/types/websocket.ts create mode 100644 src/websocket/close-event.ts rename src/{websocket.ts => websocket/index.ts} (61%) create mode 100644 src/websocket/send-data.ts create mode 100644 src/websocket/validation.ts diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index b15cc60..95cc355 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -31,10 +31,9 @@ body: description: Minimal code to reproduce the issue render: typescript placeholder: | - import { request } from 'node-wreq'; + import { fetch } from 'node-wreq'; - const response = await request({ - url: 'https://example.com', + const response = await fetch('https://example.com', { browser: 'chrome_131' }); validations: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f389e3..f1802c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,14 @@ jobs: - host: ubuntu-latest target: x86_64-unknown-linux-gnu build: npm run build:rust -- --target x86_64-unknown-linux-gnu + - host: ubuntu-latest + target: aarch64-unknown-linux-gnu + setup: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + build: CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ npm run build:rust -- --target aarch64-unknown-linux-gnu + - host: ubuntu-latest + target: x86_64-unknown-linux-musl + setup: sudo apt-get update && sudo apt-get install -y musl-tools + build: CC_x86_64_unknown_linux_musl=musl-gcc npm run build:rust -- --target x86_64-unknown-linux-musl - host: windows-latest target: x86_64-pc-windows-msvc build: npm run build:rust -- --target x86_64-pc-windows-msvc @@ -40,6 +48,10 @@ jobs: with: targets: ${{ matrix.settings.target }} + - name: Install target toolchain dependencies + if: matrix.settings.setup != '' + run: ${{ matrix.settings.setup }} + - name: Install dependencies run: npm install @@ -54,13 +66,23 @@ jobs: if-no-files-found: error publish: - name: Publish to npm + name: Publish platform package - ${{ matrix.settings.target }} runs-on: ubuntu-latest needs: build if: github.event_name == 'release' permissions: contents: read id-token: write + strategy: + fail-fast: false + matrix: + settings: + - target: x86_64-apple-darwin + - target: aarch64-apple-darwin + - target: x86_64-unknown-linux-gnu + - target: aarch64-unknown-linux-gnu + - target: x86_64-unknown-linux-musl + - target: x86_64-pc-windows-msvc steps: - uses: actions/checkout@v4 @@ -74,24 +96,54 @@ jobs: - name: Install dependencies run: npm ci - - name: Download all artifacts + - name: Download artifact uses: actions/download-artifact@v4 with: + name: bindings-${{ matrix.settings.target }} path: artifacts - - name: Move artifacts to rust directory + - name: Prepare scoped platform package shell: bash run: | - mkdir -p rust - # Copy all .node files from artifacts to rust/ - find artifacts -name "*.node" -exec cp {} rust/ \; - echo "Collected binaries:" - ls -la rust/*.node + BINARY_PATH="$(find artifacts -name '*.node' | head -n 1)" + node ./scripts/prepare-platform-package.mjs \ + --target "${{ matrix.settings.target }}" \ + --binary "$BINARY_PATH" \ + --outDir ".release/${{ matrix.settings.target }}" + + - name: Publish scoped platform package + run: npm publish ".release/${{ matrix.settings.target }}" --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + publish-main: + name: Publish main package + runs-on: ubuntu-latest + needs: publish + if: github.event_name == 'release' + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: 'https://registry.npmjs.org' + + - name: Install dependencies + run: npm ci - name: Build TypeScript run: npm run build:ts - - name: Publish to npm - run: npm publish --provenance --access public + - name: Prepare main npm package + run: npm run prepare:publish:main -- .release/main-package + + - name: Publish main package + run: npm publish .release/main-package --provenance --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore index 57a5d16..f66f99f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules/ # Build outputs dist/ +.release/ rust/target/ rust/node-wreq.node *.node @@ -28,4 +29,4 @@ yarn-debug.log* yarn-error.log* # Agents -.claude/ \ No newline at end of file +.claude/ diff --git a/.oxlintrc.json b/.oxlintrc.json index 44d3cf4..68d641b 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -1,3 +1,41 @@ { - "$schema": "./node_modules/oxlint/configuration_schema.json" -} \ No newline at end of file + "$schema": "./node_modules/oxlint/configuration_schema.json", + "jsPlugins": [ + { + "name": "stylistic", + "specifier": "@stylistic/eslint-plugin" + } + ], + "rules": { + "stylistic/padding-line-between-statements": [ + "error", + { + "blankLine": "always", + "prev": [ + "const", + "let", + "var" + ], + "next": "*" + }, + { + "blankLine": "always", + "prev": "*", + "next": "return" + }, + { + "blankLine": "any", + "prev": [ + "const", + "let", + "var" + ], + "next": [ + "const", + "let", + "var" + ] + } + ] + } +} diff --git a/README.md b/README.md index 73f0fb2..3354d6f 100644 --- a/README.md +++ b/README.md @@ -1,225 +1,745 @@ # node-wreq -🚀 High-performance browser fingerprint bypass library using Rust for native TLS/HTTP2 impersonation. +`node-wreq` is a Node HTTP client with a Rust transport underneath backed by [`wreq`](https://github.com/0x676e67/wreq). -> **Node.js wrapper for [wreq](https://github.com/0x676e67/wreq)** — A powerful Rust HTTP client with browser impersonation capabilities. +This package helps if you need low-level control over the network layer — TLS configuration, transport fingerprinting, browser impersonation, or fine-grained HTTP/WebSocket behavior that standard Node.js clients don't expose. -## ✨ Features +## Install -- ⚡ **Native Performance** — 50-100x faster than curl-impersonate (no process spawning) -- 🔒 **TLS Fingerprinting** — Perfect JA3/JA4 signatures matching real browsers -- 🌐 **HTTP/2 Fingerprinting** — Authentic SETTINGS frames, PRIORITY, and header ordering -- 🎭 **Multiple Browser Profiles** — Chrome, Firefox, Safari, Edge -- 📦 **Zero Dependencies** — Pure Rust with BoringSSL under the hood -- 💎 **TypeScript Support** — Full type definitions included -- 🛡️ **Protection Bypass** — Cloudflare, Akamai, and other anti-bot systems +```bash +npm install node-wreq +``` -## 🔧 How It Works +## Contents -The library is a Node.js wrapper for **[wreq](https://github.com/0x676e67/wreq)** (Rust HTTP client) with **BoringSSL** (Google's TLS library — the same one used in Chrome) to create requests that are indistinguishable from real browsers at the network level. +⚡ **[Quick Start](#quick-start)**\ +🌐 **[Fetch](#fetch)**\ +🧩 **[Client](#client)** — shared defaults, reusable config.\ +🎭 **[Browser Profiles](#browser-profiles)**\ +🪝 **[Hooks](#hooks)** — request lifecycle, dynamic auth, retries, etc.\ +🍪 **[Cookies and Sessions](#cookie-jar)**\ +🔁 **[Redirects and Retries](#redirects-and-retries)**\ +📊 **[Observability](#observability)**\ +🚨 **[Error Handling](#errors)**\ +🔌 **[WebSockets](#websockets)**\ +🧪 **[Networking / Transport Knobs](#networking--transport-knobs)** — TLS, HTTP/1, HTTP/2 options; header ordering. -### Why It Works +## Quick Start -Traditional HTTP clients (axios, fetch, curl) have differences in: -- **TLS handshake signatures** — Different cipher suites and extensions -- **HTTP/2 frame ordering** — Different SETTINGS and PRIORITY patterns -- **Header ordering** — Different sequence and values +```ts +import { fetch } from 'node-wreq'; -This library precisely replicates the network behavior of real browsers. +const response = await fetch('https://httpbin.org/get', { + browser: 'chrome_137', +}); -## 📦 Installation +console.log(response.status); +console.log(await response.json()); +``` -```bash -# npm -npm install node-wreq +If you keep repeating config, move to a client: + +```ts +import { createClient } from 'node-wreq'; + +const client = createClient({ + baseURL: 'https://httpbin.org', + browser: 'chrome_137', + headers: { + 'x-client': 'node-wreq', + }, + retry: 2, +}); -# yarn -yarn add node-wreq +const response = await client.fetch('/anything', { + query: { from: 'client' }, +}); -# pnpm -pnpm add node-wreq +console.log(response.status); +console.log(await response.json()); ``` -That's it! 🎉 Pre-built native modules are included for all major platforms: -- 🍎 macOS (Intel & Apple Silicon) -- 🐧 Linux (x64 & ARM64) -- 🪟 Windows (x64) +## Fetch + +### Simple GET + +```ts +import { fetch } from 'node-wreq'; + +const response = await fetch('https://httpbin.org/get', { + browser: 'firefox_139', + query: { + source: 'node-wreq', + debug: true, + }, + timeout: 15_000, +}); + +const body = await response.json(); -## 💻 Usage +console.log(response.ok); +console.log(body.args); +``` -### Basic Request +### JSON POST -```typescript -import { request } from 'node-wreq'; +```ts +import { fetch } from 'node-wreq'; -const response = await request({ - url: 'https://example.com/api', +const response = await fetch('https://api.example.com/items', { + method: 'POST', browser: 'chrome_137', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify({ + name: 'example', + enabled: true, + }), + throwHttpErrors: true, }); -console.log(response.status); // 200 -console.log(response.body); // Response body -console.log(response.headers); // Response headers -console.log(response.cookies); // Cookies +console.log(await response.json()); ``` -### With Custom Headers +### Build a `Request` first -```typescript -import { request } from 'node-wreq'; +```ts +import { Request, fetch } from 'node-wreq'; -const response = await request({ - url: 'https://api.example.com/data', - browser: 'firefox_139', +const request = new Request('https://httpbin.org/post', { + method: 'POST', headers: { - 'Authorization': 'Bearer token123', - 'Custom-Header': 'value', + 'content-type': 'application/json', }, + body: JSON.stringify({ via: 'Request' }), }); + +const response = await fetch(request, { + browser: 'chrome_137', +}); + +console.log(await response.json()); +``` + +### Read extra metadata + +`fetch()` returns a fetch-style `Response`, plus extra metadata under `response.wreq`. + +```ts +const response = await fetch('https://example.com', { + browser: 'chrome_137', +}); + +console.log(response.status); +console.log(response.headers.get('content-type')); + +console.log(response.wreq.cookies); +console.log(response.wreq.setCookies); +console.log(response.wreq.timings); +console.log(response.wreq.redirectChain); +``` + +If you need a Node stream instead of a WHATWG stream: + +```ts +const readable = response.wreq.readable(); + +readable.pipe(process.stdout); ``` -### POST Request +## Client -```typescript -import { post } from 'node-wreq'; +Use `createClient(...)` when requests share defaults: -const response = await post( - 'https://api.example.com/submit', - JSON.stringify({ foo: 'bar' }), +- `baseURL` +- browser profile +- headers +- proxy +- timeout +- hooks +- retry policy +- cookie jar + +### Shared defaults + +```ts +import { createClient } from 'node-wreq'; + +const client = createClient({ + baseURL: 'https://api.example.com', + browser: 'chrome_137', + timeout: 10_000, + headers: { + authorization: `Bearer ${process.env.API_TOKEN}`, + }, + retry: { + limit: 2, + statusCodes: [429, 503], + }, +}); + +const users = await client.get('/users'); + +console.log(await users.json()); + +const created = await client.post( + '/users', + JSON.stringify({ email: 'user@example.com' }), { - browser: 'chrome_137', headers: { - 'Content-Type': 'application/json', + 'content-type': 'application/json', }, } ); + +console.log(created.status); ``` -### Convenience Methods +### Extend a client -```typescript -import { get, post } from 'node-wreq'; +```ts +const base = createClient({ + baseURL: 'https://api.example.com', + browser: 'chrome_137', +}); -// GET request -const data = await get('https://api.example.com/users'); +const admin = base.extend({ + headers: { + authorization: `Bearer ${process.env.ADMIN_TOKEN}`, + }, +}); -// POST request -const result = await post( - 'https://api.example.com/users', - JSON.stringify({ name: 'John' }) -); +await base.get('/health'); +await admin.get('/admin/stats'); +``` + +## Browser Profiles + +Inspect the available profiles at runtime: + +```ts +import { getProfiles } from 'node-wreq'; + +console.log(getProfiles()); +``` + +There is also `BROWSER_PROFILES` if you want the generated list directly. + +Typical profiles include browser families like: + +- Chrome +- Edge +- Firefox +- Safari +- Opera +- OkHttp + +## Hooks + +Hooks are the request pipeline. + +Available phases: + +- `init` +- `beforeRequest` +- `afterResponse` +- `beforeRetry` +- `beforeError` +- `beforeRedirect` + +### Common pattern: auth, tracing, proxy rotation + +```ts +import { createClient } from 'node-wreq'; + +const client = createClient({ + baseURL: 'https://example.com', + retry: { + limit: 2, + statusCodes: [429, 503], + backoff: ({ attempt }) => attempt * 250, + }, + hooks: { + init: [ + ({ options, state }) => { + options.query = { ...options.query, source: 'hook-init' }; + + state.startedAt = Date.now(); + }, + ], + beforeRequest: [ + ({ request, options, state }) => { + request.headers.set('x-trace-id', crypto.randomUUID()); + request.headers.set('authorization', `Bearer ${getAccessToken()}`); + + options.proxy = pickProxy(); + + state.lastProxy = options.proxy; + }, + ], + beforeRetry: [ + ({ options, attempt, error, state }) => { + options.proxy = pickProxy(attempt); + + console.log('retrying', { + attempt, + proxy: options.proxy, + previousProxy: state.lastProxy, + error, + }); + }, + ], + beforeError: [ + ({ error, state }) => { + error.message = `[trace=${String(state.startedAt)}] ${error.message}`; + + return error; + }, + ], + }, +}); +``` + +### Replace a response in `afterResponse` + +```ts +import { Response, fetch } from 'node-wreq'; + +const response = await fetch('https://example.com/account', { + hooks: { + afterResponse: [ + async ({ response }) => { + if (response.status === 401) { + return new Response(JSON.stringify({ guest: true }), { + status: 200, + headers: { + 'content-type': 'application/json', + }, + url: response.url, + }); + } + }, + ], + }, +}); + +console.log(await response.json()); +``` + +### Mutate redirect hops + +```ts +await fetch('https://example.com/login', { + hooks: { + beforeRedirect: [ + ({ request, nextUrl, redirectCount }) => { + request.headers.set('x-redirect-hop', String(redirectCount)); + request.headers.set('x-next-url', nextUrl); + }, + ], + }, +}); +``` + +Rule of thumb: + +- use hooks for dynamic behavior +- use client defaults for static behavior + +## Cookie Jar + +`node-wreq` does not force a built-in cookie store. + +You provide a `cookieJar` with two methods: + +- `getCookies(url)` +- `setCookie(cookie, url)` + +That jar can be: + +- in-memory +- `tough-cookie` +- Redis-backed +- DB-backed +- anything else that matches the interface + +### Tiny in-memory jar + +```ts +import { fetch, websocket } from 'node-wreq'; + +const jarStore = new Map(); + +const cookieJar = { + getCookies() { + return [...jarStore.entries()].map(([name, value]) => ({ + name, + value, + })); + }, + setCookie(cookie: string) { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + + jarStore.set(name, value); + }, +}; + +await fetch('https://example.com/login', { cookieJar }); +await fetch('https://example.com/profile', { cookieJar }); +await websocket('wss://example.com/ws', { cookieJar }); +``` + +### `tough-cookie` + +```bash +npm install tough-cookie ``` -### With Proxy +```ts +import { CookieJar as ToughCookieJar } from 'tough-cookie'; +import { createClient } from 'node-wreq'; + +const toughJar = new ToughCookieJar(); -```typescript -import { request } from 'node-wreq'; +const cookieJar = { + async getCookies(url: string) { + const cookies = await toughJar.getCookies(url); -const response = await request({ - url: 'https://example.com', + return cookies.map((cookie) => ({ + name: cookie.key, + value: cookie.value, + })); + }, + async setCookie(cookie: string, url: string) { + await toughJar.setCookie(cookie, url); + }, +}; + +const client = createClient({ browser: 'chrome_137', - proxy?: 'http://proxy.example.com:8080', - // or with authentication: - // proxy: 'http://username:password@proxy.example.com:8080', + cookieJar, }); + +await client.fetch('https://example.com/login'); +await client.fetch('https://example.com/profile'); +``` + +### Inspect cookies on a response + +```ts +import { fetch } from 'node-wreq'; + +const response = await fetch('https://example.com/login', { cookieJar }); + +console.log(response.wreq.setCookies); +console.log(response.wreq.cookies); ``` -## 📚 API Reference +## Redirects and Retries -### `request(options:` [`RequestOptions`](#requestoptions)`): Promise<`[`Response`](#response)`>` +Both are opt-in controls on top of the normal request pipeline. -Main function for making HTTP requests with browser impersonation. +### Manual redirects -**Options:** - +```ts +const response = await fetch('https://example.com/login', { + redirect: 'manual', +}); -```typescript -interface RequestOptions { - url: string; // Required: URL to request - browser?: BrowserProfile; // Default: 'chrome_137' - method?: HttpMethod; // Default: 'GET' - headers?: Record; - body?: string; - proxy?: string; // HTTP/HTTPS/SOCKS5 proxy URL - timeout?: number; // Default: 30000ms -} +console.log(response.status); +console.log(response.headers.get('location')); +console.log(response.redirected); +``` + +Modes: + +- `follow` - default redirect following +- `manual` - return the redirect response as-is +- `error` - throw on the first redirect + +Useful redirect facts: + +- `response.wreq.redirectChain` records followed hops +- `301` / `302` rewrite `POST` to `GET` +- `303` rewrites to `GET` unless current method is `HEAD` +- `307` / `308` preserve method and body +- `authorization` is stripped on cross-origin redirect + +### Simple retries + +```ts +const response = await fetch('https://example.com', { + retry: 2, +}); ``` -**Response:** - +### Explicit retry policy + +```ts +const response = await fetch('https://example.com', { + retry: { + limit: 3, + statusCodes: [429, 503], + backoff: ({ attempt }) => attempt * 500, + }, +}); +``` + +### Custom retry decision + +```ts +import { TimeoutError, fetch } from 'node-wreq'; + +const response = await fetch('https://example.com', { + retry: { + limit: 5, + shouldRetry: ({ error, response }) => { + if (response?.status === 429) { + return true; + } + + return error instanceof TimeoutError; + }, + }, +}); +``` + +Defaults: + +- retry is off unless you enable it +- default retry methods are `GET` and `HEAD` +- default status codes include `408`, `425`, `429`, `500`, `502`, `503`, `504` +- default error codes include `ECONNRESET`, `ECONNREFUSED`, `ETIMEDOUT`, `ERR_TIMEOUT` + +## Observability + +Two main surfaces: + +- `response.wreq.timings` +- `onStats(stats)` + +### Per-request stats callback + +```ts +await fetch('https://example.com', { + onStats: ({ attempt, timings, response, error }) => { + console.log({ + attempt, + wait: timings.wait, + total: timings.total, + status: response?.status, + error, + }); + }, +}); +``` + +### Read timings from the final response + +```ts +const response = await fetch('https://example.com', { + browser: 'chrome_137', +}); + +console.log(response.wreq.timings); +``` + +Current timings are wrapper-level timings that are still useful in practice: + +- request start +- response available +- total time when body consumption is known + +## Errors + +Main error classes: + +- `RequestError` +- `HTTPError` +- `TimeoutError` +- `AbortError` +- `WebSocketError` + +Typical patterns: -```typescript -interface Response { - status: number; - headers: Record; - body: string; - cookies: Record; - url: string; // Final URL after redirects +```ts +import { HTTPError, TimeoutError, fetch } from 'node-wreq'; + +try { + await fetch('https://example.com', { + timeout: 1_000, + throwHttpErrors: true, + }); +} catch (error) { + if (error instanceof TimeoutError) { + console.error('request timed out'); + } else if (error instanceof HTTPError) { + console.error('bad status', error.statusCode); + } else { + console.error(error); + } } ``` -### `get(url: string, options?): Promise<`[`Response`](#response)`>` +## WebSockets -### `post(url: string, body?: string, options?): Promise<`[`Response`](#response)`>` +You can use either: -### `getProfiles():` [`BrowserProfile[]`](#browser-profiles) +- `await websocket(url, init?)` +- `new WebSocket(url, init?)` -Get list of available browser profiles. +### Simple helper -```typescript -import { getProfiles } from 'node-wreq'; +```ts +import { websocket } from 'node-wreq'; + +const socket = await websocket('wss://echo.websocket.events', { + browser: 'chrome_137', + protocols: ['chat'], +}); + +socket.addEventListener('message', (event) => { + console.log('message:', event.data); +}); + +socket.send('hello'); +``` + +### WHATWG-like constructor + +```ts +import { WebSocket } from 'node-wreq'; + +const socket = new WebSocket('wss://example.com/ws', { + binaryType: 'arraybuffer', +}); + +await socket.opened; + +socket.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + console.log(new Uint8Array(event.data)); + } +}; + +socket.send(new Uint8Array([1, 2, 3])); +socket.close(1000, 'done'); +``` + +### WebSocket from a client + +Useful when you want shared defaults like browser, proxy, or cookies: + +```ts +const client = createClient({ + browser: 'chrome_137', + cookieJar: yourCookieJar, +}); + +const socket = await client.websocket('wss://example.com/ws'); +``` + +Notes: -const profiles = getProfiles(); -console.log(profiles); -// ['chrome_100', 'chrome_101', ..., 'chrome_137', 'edge_101', ..., 'safari_18', ...] +- cookies from `cookieJar` are sent during handshake +- duplicate subprotocols are rejected + +## Networking / Transport Knobs + +This is the "transport nerd" section. + +Everything else here is for debugging request shape, fingerprint-sensitive targets, or testing transport hypotheses. + +### Browser profile + proxy + timeout + +```ts +const response = await fetch('https://httpbin.org/anything', { + browser: 'chrome_137', + proxy: 'http://username:password@proxy.example.com:8080', + timeout: 10_000, +}); ``` -## 🎭 Browser Profiles - +### Disable default browser-like headers -Available browser profiles (78+ profiles): +By default, `node-wreq` may apply profile-appropriate default headers. -### Chrome -29 versions from Chrome 100 to Chrome 137: -- `chrome_100`, `chrome_101`, `chrome_104`, `chrome_105`, `chrome_106`, `chrome_107`, `chrome_108`, `chrome_109`, `chrome_110` -- `chrome_114`, `chrome_116`, `chrome_117`, `chrome_118`, `chrome_119`, `chrome_120`, `chrome_123`, `chrome_124`, `chrome_126` -- `chrome_127`, `chrome_128`, `chrome_129`, `chrome_130`, `chrome_131`, `chrome_132`, `chrome_133`, `chrome_134`, `chrome_135`, `chrome_136`, `chrome_137` +If you want full manual control: -### Edge -5 versions: `edge_101`, `edge_122`, `edge_127`, `edge_131`, `edge_134` +```ts +await fetch('https://example.com', { + disableDefaultHeaders: true, + headers: { + accept: '*/*', + 'user-agent': 'custom-client', + }, +}); +``` -### Safari -19 versions including iOS and iPad: -- Desktop: `safari_15_3`, `safari_15_5`, `safari_15_6_1`, `safari_16`, `safari_16_5`, `safari_17_0`, `safari_17_2_1`, `safari_17_4_1`, `safari_17_5`, `safari_18`, `safari_18_2`, `safari_18_3`, `safari_18_3_1`, `safari_18_5` -- iOS: `safari_ios_16_5`, `safari_ios_17_2`, `safari_ios_17_4_1`, `safari_ios_18_1_1` -- iPad: `safari_ipad_18` +### Exact header order -### Firefox -10 versions including private and Android: -- `firefox_109`, `firefox_117`, `firefox_128`, `firefox_133`, `firefox_135`, `firefox_136`, `firefox_139` -- Private: `firefox_private_135`, `firefox_private_136` -- Android: `firefox_android_135` +Use tuples when header order matters: -### Opera -4 versions: `opera_116`, `opera_117`, `opera_118`, `opera_119` +```ts +await fetch('https://example.com', { + headers: [ + ['x-lower', 'one'], + ['X-Mixed', 'two'], + ], +}); +``` -### OkHttp (Android HTTP client) -8 versions: `okhttp_3_9`, `okhttp_3_11`, `okhttp_3_13`, `okhttp_3_14`, `okhttp_4_9`, `okhttp_4_10`, `okhttp_4_12`, `okhttp_5` +### Exact original header names on the wire -> Use `getProfiles()` to get the complete list programmatically. +Use this only if you really need exact casing / spelling preservation: -## 📖 Documentation +```ts +await fetch('https://example.com', { + disableDefaultHeaders: true, + keepOriginalHeaderNames: true, + headers: [ + ['x-lower', 'one'], + ['X-Mixed', 'two'], + ], +}); +``` -- **[Architecture Guide](docs/ARCHITECTURE.md)** — Technical details about TLS/HTTP2 fingerprinting, how browser impersonation works -- **[Build Instructions](docs/BUILD.md)** — Developer guide for building from source -- **[Publishing Guide](docs/PUBLISHING.md)** — How to publish the package +### Lower-level transport tuning -## 🤝 Contributions are welcome! +If a browser preset gets you close but not all the way there: -Please read [Contributing Guide](CONTRIBUTING.md). +```ts +await fetch('https://example.com', { + browser: 'chrome_137', + tlsOptions: { + greaseEnabled: true, + }, + http1Options: { + writev: true, + }, + http2Options: { + adaptiveWindow: false, + maxConcurrentStreams: 64, + }, +}); +``` -## 🙏 Acknowledgments +Use these only when: -Built with: -- [wreq](https://github.com/0x676e67/wreq) — Rust HTTP client with browser impersonation -- [Neon](https://neon-bindings.com/) — Rust ↔ Node.js bindings +- a target is still picky after choosing a browser profile +- you are comparing transport behavior +- you want to debug fingerprint mismatches + +### Compression + +Compression is enabled by default. + +Disable it if you need stricter control over response handling: + +```ts +await fetch('https://example.com/archive', { + compress: false, +}); +``` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2584879..63dec09 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -5,7 +5,7 @@ ``` ┌─────────────────────────────────────┐ │ JavaScript/TypeScript Code │ -│ request({ url, browser }) │ +│ fetch(url, init) / createClient() │ └──────────────┬──────────────────────┘ ↓ ┌──────────────────────────────────────┐ @@ -58,18 +58,18 @@ The client automatically reuses connections (HTTP/2 keep-alive). ### Parallel Requests -```typescript -// Good: Parallel requests +```ts +// Good: parallel requests const [res1, res2, res3] = await Promise.all([ - request({ url: url1 }), - request({ url: url2 }), - request({ url: url3 }), + fetch(url1), + fetch(url2), + fetch(url3), ]); -// Bad: Sequential requests -const res1 = await request({ url: url1 }); -const res2 = await request({ url: url2 }); -const res3 = await request({ url: url3 }); +// Slower: sequential requests +const res1 = await fetch(url1); +const res2 = await fetch(url2); +const res3 = await fetch(url3); ``` ## References diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index b1eeadb..ca5927e 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -2,19 +2,30 @@ ## How to Publish node-wreq -This package uses a **single npm package** that includes pre-built native binaries for all supported platforms. +Publishing is split into two layers: + +- scoped platform packages with the native `.node` binaries +- the main `node-wreq` package with JS, types, loader logic, and `optionalDependencies` ### Package Structure When published, the package includes: -- TypeScript compiled code (`dist/`) -- Native binaries for all platforms (`rust/*.node`) - -Supported platforms: -- 🍎 macOS Intel (x64) -- 🍎 macOS Apple Silicon (arm64) -- 🐧 Linux x64 -- 🪟 Windows x64 +- main package: + - CommonJS output in `dist/` + - ESM wrapper output in `dist/` + - Type declarations in `dist/` + - `optionalDependencies` pointing at the scoped native packages +- scoped platform packages: + - one native `.node` binary each + +Scoped package names: + +- `@node-wreq/darwin-x64` +- `@node-wreq/darwin-arm64` +- `@node-wreq/linux-x64-gnu` +- `@node-wreq/linux-arm64-gnu` +- `@node-wreq/linux-x64-musl` +- `@node-wreq/win32-x64-msvc` ### Publishing Process @@ -49,10 +60,11 @@ Then create a GitHub Release from this tag. This will trigger the build workflow #### 3. Automated Build & Publish GitHub Actions will automatically: -1. Build native binaries for all platforms (macOS, Linux, Windows) -2. Collect all binaries into the `rust/` directory -3. Build TypeScript code -4. Publish the package to npm with all binaries included +1. Build native binaries for all configured platforms +2. Publish one scoped package per platform artifact +3. Build the JS outputs +4. Stage the main package with generated `optionalDependencies` +5. Publish the main package to npm ### Local Testing Before Publishing @@ -63,33 +75,42 @@ npm run build # Run tests npm test -# Pack to see what will be published -npm pack - -# Extract and inspect -tar -tzf node-wreq-*.tgz +# Stage the publishable main package +npm run prepare:publish:main -- .release/main-package -# Test in another project -cd /path/to/test-project -npm install /path/to/node-wreq/node-wreq-*.tgz +# Inspect staged files +find .release/main-package -maxdepth 3 -type f | sort ``` -### Manual Publishing (Not Recommended) +### Manual Publishing (Platform Packages) -If you need to publish manually: +If you really need to publish a scoped platform package manually: ```bash -# Build TypeScript -npm run build:ts +# Example: build a target +npm run build:rust -- --target x86_64-unknown-linux-musl + +# Stage the scoped package +node ./scripts/prepare-platform-package.mjs \ + --target x86_64-unknown-linux-musl \ + --binary rust/node-wreq.linux-x64-musl.node \ + --outDir .release/linux-x64-musl -# Ensure all platform binaries are in rust/ directory -ls rust/*.node +# Publish it +npm publish .release/linux-x64-musl --access public +``` + +### Manual Publishing (Main Package) + +After the platform packages for the same version exist: -# Publish -npm publish --access public +```bash +npm run build:ts +npm run prepare:publish:main -- .release/main-package +npm publish .release/main-package --access public ``` -**Note:** Manual publishing requires you to have all platform binaries built locally, which is difficult without cross-compilation setup. Use GitHub Actions instead. +Use GitHub Actions unless you have a specific reason not to. ## Troubleshooting @@ -101,6 +122,6 @@ npm publish --access public ### Module Load Error After Install -- Verify `rust/*.node` files are included in the published package -- Check that the binary was built for the user's platform -- Ensure file permissions are correct +- Verify the matching scoped package for the user's platform was published +- Verify the main package version and platform package versions match +- Check that the loader can resolve the correct package for the user's platform diff --git a/package-lock.json b/package-lock.json index eda80e5..e2341e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ ], "devDependencies": { "@napi-rs/cli": "^2.18.0", + "@stylistic/eslint-plugin": "^5.10.0", "@types/node": "^20.0.0", "@types/ws": "^8.18.1", "oxfmt": "^0.41.0", @@ -30,6 +31,175 @@ "node": ">=18.0.0" } }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.5.tgz", + "integrity": "sha512-eIJYKTCECbP/nsKaaruF6LW967mtbQbsw4JTtSVkUQc9MneSkbrgPJAbKl9nWr0ZeowV8BfsarBmPpBzGelA2w==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.1.tgz", + "integrity": "sha512-rZAP3aVgB9ds9KOeUSL+zZ21hPmo8dh6fnIFwRQj5EAZl9gzR7wxYbYXYysAM8CTqGmUGyp2S4kUdV17MnGuWQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, "node_modules/@napi-rs/cli": { "version": "2.18.4", "dev": true, @@ -691,6 +861,51 @@ "node": "^20.19.0 || >=22.12.0" } }, + "node_modules/@stylistic/eslint-plugin": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-5.10.0.tgz", + "integrity": "sha512-nPK52ZHvot8Ju/0A4ucSX1dcPV2/1clx0kLcH5wDmrE4naKso7TUC/voUyU1O9OTKTrR6MYip6LP0ogEMQ9jPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/types": "^8.56.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "estraverse": "^5.3.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "peerDependencies": { + "eslint": "^9.0.0 || ^10.0.0" + } + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/@types/node": { "version": "20.19.21", "dev": true, @@ -709,6 +924,600 @@ "@types/node": "*" } }, + "node_modules/@typescript-eslint/types": { + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.1.tgz", + "integrity": "sha512-io/dV5Aw5ezwzfPBBWLoT+5QfVtP8O7q4Kftjn5azJ88bYyp/ZMCsyW1lpKK46EXJcaYMZ1JtYj+s/7TdzmQMw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", + "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.0.tgz", + "integrity": "sha512-+L0vBFYGIpSNIt/KWTpFonPrqYvgKw1eUI5Vn7mEogrQcWtWYtNQ7dNqC+px/J0idT3BAkiWrhfS7k+Tum8TUA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.4", + "@eslint/config-helpers": "^0.5.4", + "@eslint/core": "^1.2.0", + "@eslint/plugin-kit": "^0.7.0", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "peer": true, + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/oxfmt": { "version": "0.41.0", "resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.41.0.tgz", @@ -794,6 +1603,122 @@ } } }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/tinypool": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-2.1.0.tgz", @@ -804,6 +1729,20 @@ "node": "^20.0.0 || >=22.0.0" } }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/typescript": { "version": "5.9.3", "dev": true, @@ -821,6 +1760,45 @@ "dev": true, "license": "MIT" }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ws": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", @@ -842,6 +1820,20 @@ "optional": true } } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/package.json b/package.json index ace4182..eb89c96 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,25 @@ { "name": "node-wreq", "version": "0.1.2", - "description": "Browser fingerprint bypass library using Rust for TLS/HTTP2 impersonation", - "main": "dist/node-wreq.js", + "description": "Node HTTP client with native browser impersonation, hooks, cookies, redirects, websockets, and timing stats", + "main": "./dist/node-wreq.js", + "module": "./dist/node-wreq.mjs", "types": "dist/node-wreq.d.ts", + "exports": { + ".": { + "types": "./dist/node-wreq.d.ts", + "require": "./dist/node-wreq.js", + "import": "./dist/node-wreq.mjs", + "default": "./dist/node-wreq.js" + }, + "./package.json": "./package.json" + }, "scripts": { "build": "npm run build:rust && npm run build:ts", "build:rust": "napi build --platform --release --cargo-cwd rust rust", - "build:ts": "tsc", + "build:ts": "node ./scripts/generate-browser-profiles.mjs && tsc && node ./scripts/postbuild.mjs", + "prepare:publish:main": "node ./scripts/prepare-main-package.mjs", + "prepare:publish:platform": "node ./scripts/prepare-platform-package.mjs", "artifacts": "napi artifacts", "clean": "rm -rf dist rust/target rust/*.node", "test": "npm run build && node --test dist/test/node-wreq.spec.js", @@ -36,14 +48,14 @@ "url": "https://github.com/StopMakingThatBigFace/node-wreq/issues" }, "homepage": "https://github.com/StopMakingThatBigFace/node-wreq#readme", - "dependencies": {}, "devDependencies": { + "@napi-rs/cli": "^2.18.0", + "@stylistic/eslint-plugin": "^5.10.0", "@types/node": "^20.0.0", "@types/ws": "^8.18.1", - "typescript": "^5.0.0", - "@napi-rs/cli": "^2.18.0", "oxfmt": "^0.41.0", "oxlint": "^1.56.0", + "typescript": "^5.0.0", "ws": "^8.18.3" }, "engines": { @@ -70,6 +82,8 @@ "x86_64-apple-darwin", "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc" ] } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 41d2a12..5805c06 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -78,27 +69,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -134,11 +104,10 @@ dependencies = [ [[package]] name = "boring-sys2" -version = "5.0.0-alpha.10" +version = "5.0.0-alpha.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc6cf07c95de5849d3e42032e2bc35242f2b9cffdd1835ee4e5d2b7a0ec65c7" +checksum = "455d79965f5155dcc88a7abce112c3590883889131b799beda10bf9a813ed669" dependencies = [ - "autocfg", "bindgen", "cmake", "fs_extra", @@ -147,9 +116,9 @@ dependencies = [ [[package]] name = "boring2" -version = "5.0.0-alpha.10" +version = "5.0.0-alpha.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a24a6ad62dc302105299550ef2f09ebaf82be4d60bff4ceabc36146a35322b" +checksum = "183ccc3854411c035410dcdbffafca62084f3a6c33f013c77e83c025d2a08a28" dependencies = [ "bitflags", "boring-sys2", @@ -187,9 +156,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.41" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -225,9 +194,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.54" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] @@ -242,8 +211,6 @@ dependencies = [ "compression-core", "flate2", "memchr", - "zstd", - "zstd-safe", ] [[package]] @@ -315,6 +282,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -329,9 +307,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" @@ -464,12 +442,6 @@ dependencies = [ "wasi 0.14.7+wasi-0.2.4", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "glob" version = "0.3.3" @@ -488,6 +460,12 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "http" version = "1.3.1" @@ -524,9 +502,9 @@ dependencies = [ [[package]] name = "http2" -version = "0.5.8" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ed027cf2bcf07fcd4e27cfba91b8e5fbae47151283406b16391c463e3a6db" +checksum = "a7349aa548b6a413a7e7146d5d7e3db4565b3d0fbc8ff53c72e46b44b083cf9f" dependencies = [ "atomic-waker", "bytes", @@ -535,6 +513,7 @@ dependencies = [ "futures-sink", "http", "indexmap", + "parking_lot", "slab", "smallvec", "tokio", @@ -548,42 +527,124 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] -name = "indexmap" -version = "2.11.4" +name = "icu_collections" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "equivalent", - "hashbrown 0.16.0", + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "io-uring" -version = "0.7.10" +name = "icu_locale_core" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ - "bitflags", - "cfg-if", - "libc", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "ipnet" -version = "2.11.0" +name = "icu_normalizer" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] -name = "iri-string" -version = "0.7.8" +name = "icu_properties" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "memchr", - "serde", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", ] +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +dependencies = [ + "equivalent", + "hashbrown 0.16.0", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + [[package]] name = "itertools" version = "0.13.0" @@ -611,9 +672,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "libloading" @@ -645,6 +706,12 @@ dependencies = [ "syn", ] +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -684,9 +751,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.4" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -728,6 +795,7 @@ dependencies = [ "neon", "serde", "serde_json", + "strum", "thiserror 1.0.69", "tokio", "wreq", @@ -750,15 +818,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -823,6 +882,15 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -849,9 +917,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -929,12 +997,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3160422bbd54dd5ecfdca71e5fd59b7b8fe2b1697ab2baf64f6d05dcc66d298" -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -1025,18 +1087,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "sha1" version = "0.10.6" @@ -1083,14 +1133,38 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", "windows-sys", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "2.0.106" @@ -1108,6 +1182,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1179,21 +1264,28 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" -version = "1.47.1" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", "socket2", "tokio-macros", "windows-sys", @@ -1201,20 +1293,19 @@ dependencies = [ [[package]] name = "tokio-boring2" -version = "5.0.0-alpha.10" +version = "5.0.0-alpha.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e756617f510b6822897b6fe9c40d643585e524ed11de2e308bff1ee837eefe2" +checksum = "0f81df1210d791f31d72d840de8fbd80b9c3cb324956523048b1413e2bd55756" dependencies = [ - "boring-sys2", "boring2", "tokio", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -1235,9 +1326,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", @@ -1260,9 +1351,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -1275,9 +1366,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", "bitflags", @@ -1313,9 +1404,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", @@ -1330,18 +1421,18 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.22.0" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" dependencies = [ "typed-builder-macro", ] [[package]] name = "typed-builder-macro" -version = "0.22.0" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" dependencies = [ "proc-macro2", "quote", @@ -1360,12 +1451,30 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "version_check" version = "0.9.5" @@ -1444,77 +1553,13 @@ checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-link", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.46.0" @@ -1523,9 +1568,9 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "wreq" -version = "6.0.0-rc.20" +version = "6.0.0-rc.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776885d1475f8358bbde83b65967dae6408de22ef9efa96432fb2b24ab054110" +checksum = "f79937f6c4df65b3f6f78715b9de2977afe9ee3b3436483c7949a24511e25935" dependencies = [ "ahash", "boring2", @@ -1541,14 +1586,12 @@ dependencies = [ "http2", "httparse", "ipnet", - "iri-string", "libc", "percent-encoding", "pin-project-lite", "schnellru", "serde", "serde_json", - "serde_urlencoded", "smallvec", "socket2", "tokio", @@ -1557,6 +1600,7 @@ dependencies = [ "tokio-tungstenite", "tower", "tower-http", + "url", "want", "webpki-root-certs", "zstd", @@ -1564,14 +1608,46 @@ dependencies = [ [[package]] name = "wreq-util" -version = "3.0.0-rc.5" +version = "3.0.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a38c2ade0a913ec9e1f7f394de8ebdb78d7123e45f015f9d02fed5bdc9db67b" +checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04" dependencies = [ + "serde", + "strum", + "strum_macros", "typed-builder", "wreq", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.27" @@ -1592,6 +1668,60 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 718accc..0977cc6 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,8 +9,9 @@ crate-type = ["cdylib"] [dependencies] # HTTP client with browser impersonation -wreq = { version = "6.0.0-rc.20", default-features = false, features = ["gzip", "deflate", "socks", "cookies", "json", "webpki-roots", "ws"] } -wreq-util = "3.0.0-rc.1" +wreq = { version = "6.0.0-rc.28", default-features = false, features = ["gzip", "brotli", "deflate", "socks", "cookies", "json", "webpki-roots", "ws"] } +wreq-util = { version = "3.0.0-rc.10", features = ["emulation-rand", "emulation-serde"] } +strum = "0.27.2" # Neon for Node.js bindings neon = { version = "1.0", default-features = false, features = ["napi-6"] } diff --git a/rust/src/client.rs b/rust/src/client.rs deleted file mode 100644 index 4e0bbca..0000000 --- a/rust/src/client.rs +++ /dev/null @@ -1,511 +0,0 @@ -use anyhow::{Context, Result}; -use std::collections::HashMap; -use std::sync::{ - Arc, Mutex, OnceLock, - atomic::{AtomicU64, Ordering}, -}; -use std::time::Duration; -use wreq::redirect; -use wreq::ws::WebSocket; -use wreq::ws::message::{CloseCode, CloseFrame, Message}; -use wreq_util::Emulation; - -#[derive(Debug, Clone)] -pub struct RequestOptions { - pub url: String, - pub emulation: Emulation, - pub headers: HashMap, - pub method: String, - pub body: Option, - pub proxy: Option, - pub timeout: u64, -} - -#[derive(Debug, Clone)] -pub struct Response { - pub status: u16, - pub headers: HashMap, - pub body_handle: u64, - pub cookies: HashMap, - pub set_cookies: Vec, - pub url: String, -} - -#[derive(Debug, Clone)] -pub struct WebSocketConnectOptions { - pub url: String, - pub emulation: Emulation, - pub headers: HashMap, - pub proxy: Option, - pub timeout: u64, - pub protocols: Vec, -} - -#[derive(Debug, Clone)] -pub struct WebSocketConnection { - pub handle: u64, - pub protocol: Option, - pub extensions: Option, - pub url: String, -} - -#[derive(Debug, Clone)] -pub enum WebSocketReadResult { - Text(String), - Binary(Vec), - Close { - code: u16, - reason: String, - was_clean: bool, - }, -} - -#[derive(Debug)] -enum WebSocketCommand { - Text(String), - Binary(Vec), - Close { - code: Option, - reason: Option, - }, -} - -#[derive(Debug)] -struct StoredBody { - response: wreq::Response, -} - -#[derive(Debug)] -struct StoredWebSocket { - commands: tokio::sync::mpsc::UnboundedSender, - events: tokio::sync::Mutex>, -} - -type SharedWebSocket = Arc; - -static NEXT_BODY_HANDLE: AtomicU64 = AtomicU64::new(1); -static NEXT_WEBSOCKET_HANDLE: AtomicU64 = AtomicU64::new(1); -static BODY_STORE: OnceLock>> = OnceLock::new(); -static WEBSOCKET_STORE: OnceLock>> = OnceLock::new(); -static TOKIO_RUNTIME: OnceLock = OnceLock::new(); - -fn body_store() -> &'static Mutex> { - BODY_STORE.get_or_init(|| Mutex::new(HashMap::new())) -} - -fn websocket_store() -> &'static Mutex> { - WEBSOCKET_STORE.get_or_init(|| Mutex::new(HashMap::new())) -} - -fn runtime() -> &'static tokio::runtime::Runtime { - TOKIO_RUNTIME.get_or_init(|| { - tokio::runtime::Runtime::new().expect("Failed to create Tokio runtime") - }) -} - -pub fn execute_request(options: RequestOptions) -> Result { - runtime().block_on(make_request(options)) -} - -pub fn connect_websocket(options: WebSocketConnectOptions) -> Result { - runtime().block_on(make_websocket(options)) -} - -fn store_body(response: wreq::Response) -> u64 { - let handle = NEXT_BODY_HANDLE.fetch_add(1, Ordering::Relaxed); - body_store() - .lock() - .expect("body store poisoned") - .insert(handle, StoredBody { response }); - handle -} - -fn store_websocket(websocket: WebSocket) -> u64 { - let handle = NEXT_WEBSOCKET_HANDLE.fetch_add(1, Ordering::Relaxed); - let (command_tx, command_rx) = tokio::sync::mpsc::unbounded_channel(); - let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel(); - - runtime().spawn(run_websocket_task(websocket, command_rx, event_tx)); - - websocket_store() - .lock() - .expect("websocket store poisoned") - .insert( - handle, - Arc::new(StoredWebSocket { - commands: command_tx, - events: tokio::sync::Mutex::new(event_rx), - }), - ); - - handle -} - -fn get_websocket(handle: u64) -> Result { - let store = websocket_store() - .lock() - .map_err(|_| anyhow::anyhow!("websocket store poisoned"))?; - - store - .get(&handle) - .cloned() - .ok_or_else(|| anyhow::anyhow!("Unknown websocket handle: {}", handle)) -} - -pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { - let mut store = body_store() - .lock() - .map_err(|_| anyhow::anyhow!("body store poisoned"))?; - let Some(body) = store.get_mut(&handle) else { - return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); - }; - - let chunk = runtime() - .block_on(body.response.chunk()) - .context("Failed to read response body chunk")?; - - let Some(chunk) = chunk else { - store.remove(&handle); - return Ok((Vec::new(), true)); - }; - - Ok((chunk.to_vec(), false)) -} - -pub fn read_body_all(handle: u64) -> Result> { - let mut store = body_store() - .lock() - .map_err(|_| anyhow::anyhow!("body store poisoned"))?; - let Some(body) = store.remove(&handle) else { - return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); - }; - - let mut bytes = Vec::new(); - let mut response = body.response; - - runtime().block_on(async { - while let Some(chunk) = response - .chunk() - .await - .context("Failed to read response body chunk")? - { - bytes.extend_from_slice(&chunk); - } - - Ok::<(), anyhow::Error>(()) - })?; - - Ok(bytes) -} - -pub fn cancel_body(handle: u64) -> bool { - body_store() - .lock() - .expect("body store poisoned") - .remove(&handle) - .is_some() -} - -pub fn read_websocket_message(handle: u64) -> Result { - let websocket = get_websocket(handle)?; - - let result = runtime().block_on(async { - let mut events = websocket.events.lock().await; - events - .recv() - .await - .ok_or_else(|| anyhow::anyhow!("WebSocket event stream is closed")) - }); - - if matches!(result, Ok(WebSocketReadResult::Close { .. })) { - websocket_store() - .lock() - .expect("websocket store poisoned") - .remove(&handle); - } - - result -} - -pub fn send_websocket_text(handle: u64, text: String) -> Result<()> { - let websocket = get_websocket(handle)?; - websocket - .commands - .send(WebSocketCommand::Text(text)) - .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) -} - -pub fn send_websocket_binary(handle: u64, bytes: Vec) -> Result<()> { - let websocket = get_websocket(handle)?; - websocket - .commands - .send(WebSocketCommand::Binary(bytes)) - .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) -} - -pub fn close_websocket(handle: u64, code: Option, reason: Option) -> Result<()> { - let websocket = get_websocket(handle)?; - websocket - .commands - .send(WebSocketCommand::Close { code, reason }) - .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) -} - -fn parse_cookie_pair(set_cookie: &str) -> Option<(String, String)> { - let pair = set_cookie.split(';').next()?.trim(); - let (name, value) = pair.split_once('=')?; - - Some((name.to_string(), value.to_string())) -} - -async fn run_websocket_task( - mut websocket: WebSocket, - mut commands: tokio::sync::mpsc::UnboundedReceiver, - events: tokio::sync::mpsc::UnboundedSender, -) { - let mut close_requested = false; - let mut requested_close_code = 1000; - let mut requested_close_reason = String::new(); - - loop { - tokio::select! { - command = commands.recv() => { - match command { - Some(WebSocketCommand::Text(text)) => { - if websocket.send(Message::Text(text.into())).await.is_err() { - let _ = events.send(WebSocketReadResult::Close { - code: 1006, - reason: String::new(), - was_clean: false, - }); - break; - } - } - Some(WebSocketCommand::Binary(bytes)) => { - if websocket.send(Message::Binary(bytes.into())).await.is_err() { - let _ = events.send(WebSocketReadResult::Close { - code: 1006, - reason: String::new(), - was_clean: false, - }); - break; - } - } - Some(WebSocketCommand::Close { code, reason }) => { - close_requested = true; - requested_close_code = code.unwrap_or(1000); - requested_close_reason = reason.unwrap_or_default(); - - let frame = Message::Close(Some(CloseFrame { - code: CloseCode::from(requested_close_code), - reason: requested_close_reason.clone().into(), - })); - - if websocket.send(frame).await.is_err() { - let _ = events.send(WebSocketReadResult::Close { - code: 1006, - reason: String::new(), - was_clean: false, - }); - break; - } - } - None => { - break; - } - } - } - message = websocket.recv() => { - match message { - Some(Ok(Message::Text(text))) => { - if events.send(WebSocketReadResult::Text(text.to_string())).is_err() { - break; - } - } - Some(Ok(Message::Binary(bytes))) => { - if events.send(WebSocketReadResult::Binary(bytes.to_vec())).is_err() { - break; - } - } - Some(Ok(Message::Close(frame))) => { - let (code, reason) = match frame { - Some(frame) => (u16::from(frame.code), frame.reason.to_string()), - None => { - if close_requested { - (requested_close_code, requested_close_reason.clone()) - } else { - (1005, String::new()) - } - } - }; - - let _ = events.send(WebSocketReadResult::Close { - code, - reason, - was_clean: true, - }); - break; - } - Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => {} - Some(Err(_)) => { - let _ = events.send(WebSocketReadResult::Close { - code: 1006, - reason: String::new(), - was_clean: false, - }); - break; - } - None => { - let _ = events.send(WebSocketReadResult::Close { - code: if close_requested { - requested_close_code - } else { - 1006 - }, - reason: if close_requested { - requested_close_reason.clone() - } else { - String::new() - }, - was_clean: close_requested, - }); - break; - } - } - } - } - } -} - -pub async fn make_request(options: RequestOptions) -> Result { - let mut client_builder = wreq::Client::builder() - .emulation(options.emulation) - .cookie_store(true); - - if let Some(proxy_url) = &options.proxy { - let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; - client_builder = client_builder.proxy(proxy); - } - - let client = client_builder.build().context("Failed to build HTTP client")?; - - let method = if options.method.is_empty() { - "GET" - } else { - &options.method - }; - - let mut request = match method.to_uppercase().as_str() { - "GET" => client.get(&options.url), - "POST" => client.post(&options.url), - "PUT" => client.put(&options.url), - "DELETE" => client.delete(&options.url), - "PATCH" => client.patch(&options.url), - "HEAD" => client.head(&options.url), - _ => return Err(anyhow::anyhow!("Unsupported HTTP method: {}", method)), - }; - - for (key, value) in &options.headers { - request = request.header(key, value); - } - - if let Some(body) = options.body { - request = request.body(body); - } - - request = request.timeout(Duration::from_millis(options.timeout)); - request = request.redirect(redirect::Policy::none()); - - let response = request - .send() - .await - .with_context(|| format!("{} {}", method, options.url))?; - - let status = response.status().as_u16(); - let final_url = response.uri().to_string(); - - let mut response_headers = HashMap::new(); - for (key, value) in response.headers() { - if let Ok(value_str) = value.to_str() { - response_headers.insert(key.to_string(), value_str.to_string()); - } - } - - let mut cookies = HashMap::new(); - let mut set_cookies = Vec::new(); - for cookie_header in response.headers().get_all("set-cookie") { - if let Ok(cookie_str) = cookie_header.to_str() { - set_cookies.push(cookie_str.to_string()); - - if let Some((key, value)) = parse_cookie_pair(cookie_str) { - cookies.insert(key, value); - } - } - } - - let body_handle = store_body(response); - - Ok(Response { - status, - headers: response_headers, - body_handle, - cookies, - set_cookies, - url: final_url, - }) -} - -async fn make_websocket(options: WebSocketConnectOptions) -> Result { - let mut client_builder = wreq::Client::builder() - .emulation(options.emulation) - .cookie_store(true) - .timeout(Duration::from_millis(options.timeout)); - - if let Some(proxy_url) = &options.proxy { - let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; - client_builder = client_builder.proxy(proxy); - } - - let client = client_builder - .build() - .context("Failed to build WebSocket client")?; - - let mut request = client.websocket(&options.url); - for (key, value) in &options.headers { - request = request.header(key, value); - } - - if !options.protocols.is_empty() { - request = request.protocols(options.protocols.iter().cloned()); - } - - let response = request - .send() - .await - .with_context(|| format!("WS {}", options.url))?; - - let extensions = response - .headers() - .get("sec-websocket-extensions") - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - - let websocket = response - .into_websocket() - .await - .with_context(|| format!("WS upgrade {}", options.url))?; - - let protocol = websocket - .protocol() - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - - let handle = store_websocket(websocket); - - Ok(WebSocketConnection { - handle, - protocol, - extensions, - url: options.url, - }) -} diff --git a/rust/src/emulation/builders.rs b/rust/src/emulation/builders.rs new file mode 100644 index 0000000..31ec2e3 --- /dev/null +++ b/rust/src/emulation/builders.rs @@ -0,0 +1,346 @@ +use crate::emulation::parse::{ + parse_alpn_protocol, parse_alps_protocol, parse_certificate_compression_algorithm, + parse_http2_setting_id, parse_pseudo_id, parse_tls_version, +}; +use crate::emulation::payload::{ + CustomHttp1Options, CustomHttp2ExperimentalSetting, CustomHttp2Options, CustomHttp2Priority, + CustomTlsOptions, +}; +use anyhow::{anyhow, bail, Result}; +use std::collections::HashSet; +use std::time::Duration; +use wreq::{ + http1::Http1Options, + http2::{ + ExperimentalSettings, Http2Options, Priorities, Priority, PseudoOrder, Setting, SettingId, + SettingsOrder, StreamDependency, StreamId, + }, + tls::{ExtensionType, TlsOptions}, +}; + +pub fn build_tls_options(options: CustomTlsOptions) -> Result { + let mut builder = TlsOptions::builder(); + + if let Some(alpn_protocols) = options.alpn_protocols { + builder = builder.alpn_protocols( + alpn_protocols + .into_iter() + .map(|protocol| parse_alpn_protocol(&protocol)) + .collect::>>()?, + ); + } + + if let Some(alps_protocols) = options.alps_protocols { + builder = builder.alps_protocols( + alps_protocols + .into_iter() + .map(|protocol| parse_alps_protocol(&protocol)) + .collect::>>()?, + ); + } + + if let Some(value) = options.alps_use_new_codepoint { + builder = builder.alps_use_new_codepoint(value); + } + if let Some(value) = options.session_ticket { + builder = builder.session_ticket(value); + } + if let Some(value) = options.min_tls_version { + builder = builder.min_tls_version(Some(parse_tls_version(&value)?)); + } + if let Some(value) = options.max_tls_version { + builder = builder.max_tls_version(Some(parse_tls_version(&value)?)); + } + if let Some(value) = options.pre_shared_key { + builder = builder.pre_shared_key(value); + } + if let Some(value) = options.enable_ech_grease { + builder = builder.enable_ech_grease(value); + } + if let Some(value) = options.permute_extensions { + builder = builder.permute_extensions(Some(value)); + } + if let Some(value) = options.grease_enabled { + builder = builder.grease_enabled(Some(value)); + } + if let Some(value) = options.enable_ocsp_stapling { + builder = builder.enable_ocsp_stapling(value); + } + if let Some(value) = options.enable_signed_cert_timestamps { + builder = builder.enable_signed_cert_timestamps(value); + } + if let Some(value) = options.record_size_limit { + builder = builder.record_size_limit(Some(value)); + } + if let Some(value) = options.psk_skip_session_ticket { + builder = builder.psk_skip_session_ticket(value); + } + if let Some(value) = options.key_shares_limit { + builder = builder.key_shares_limit(Some(value)); + } + if let Some(value) = options.psk_dhe_ke { + builder = builder.psk_dhe_ke(value); + } + if let Some(value) = options.renegotiation { + builder = builder.renegotiation(value); + } + if let Some(value) = options.delegated_credentials { + builder = builder.delegated_credentials(value); + } + if let Some(value) = options.curves_list { + builder = builder.curves_list(value); + } + if let Some(value) = options.cipher_list { + builder = builder.cipher_list(value); + } + if let Some(value) = options.sigalgs_list { + builder = builder.sigalgs_list(value); + } + if let Some(value) = options.certificate_compression_algorithms { + builder = builder.certificate_compression_algorithms( + value + .into_iter() + .map(|algorithm| parse_certificate_compression_algorithm(&algorithm)) + .collect::>>()?, + ); + } + if let Some(value) = options.extension_permutation { + builder = builder.extension_permutation( + value + .into_iter() + .map(ExtensionType::from) + .collect::>(), + ); + } + if let Some(value) = options.aes_hw_override { + builder = builder.aes_hw_override(Some(value)); + } + if let Some(value) = options.preserve_tls13_cipher_list { + builder = builder.preserve_tls13_cipher_list(Some(value)); + } + if let Some(value) = options.random_aes_hw_override { + builder = builder.random_aes_hw_override(value); + } + + Ok(builder.build()) +} + +pub fn build_http1_options(options: CustomHttp1Options) -> Result { + let mut builder = Http1Options::builder(); + + if let Some(value) = options.http09_responses { + builder = builder.http09_responses(value); + } + if let Some(value) = options.writev { + builder = builder.writev(Some(value)); + } + if let Some(value) = options.max_headers { + builder = builder.max_headers(value); + } + if let Some(value) = options.read_buf_exact_size { + builder = builder.read_buf_exact_size(Some(value)); + } + if let Some(value) = options.max_buf_size { + if value < 8192 { + bail!("Invalid emulation http1Options.maxBufSize: must be at least 8192"); + } + builder = builder.max_buf_size(value); + } + if options.read_buf_exact_size.is_some() && options.max_buf_size.is_some() { + bail!("Invalid emulation http1Options: readBufExactSize and maxBufSize cannot both be set"); + } + if let Some(value) = options.ignore_invalid_headers_in_responses { + builder = builder.ignore_invalid_headers_in_responses(value); + } + if let Some(value) = options.allow_spaces_after_header_name_in_responses { + builder = builder.allow_spaces_after_header_name_in_responses(value); + } + if let Some(value) = options.allow_obsolete_multiline_headers_in_responses { + builder = builder.allow_obsolete_multiline_headers_in_responses(value); + } + + Ok(builder.build()) +} + +pub fn build_http2_options(options: CustomHttp2Options) -> Result { + let mut builder = Http2Options::builder(); + + if let Some(value) = options.adaptive_window { + builder = builder.adaptive_window(value); + } + if let Some(value) = options.initial_stream_id { + builder = builder.initial_stream_id(Some(value)); + } + if let Some(value) = options.initial_connection_window_size { + builder = builder.initial_connection_window_size(Some(value)); + } + if let Some(value) = options.initial_window_size { + builder = builder.initial_window_size(Some(value)); + } + if let Some(value) = options.initial_max_send_streams { + builder = builder.initial_max_send_streams(Some(value)); + } + if let Some(value) = options.max_frame_size { + builder = builder.max_frame_size(Some(value)); + } + if let Some(value) = options.keep_alive_interval { + builder = builder.keep_alive_interval(Some(Duration::from_millis(value))); + } + if let Some(value) = options.keep_alive_timeout { + builder = builder.keep_alive_timeout(Duration::from_millis(value)); + } + if let Some(value) = options.keep_alive_while_idle { + builder = builder.keep_alive_while_idle(value); + } + if let Some(value) = options.max_concurrent_reset_streams { + builder = builder.max_concurrent_reset_streams(value); + } + if let Some(value) = options.max_send_buffer_size { + builder = builder.max_send_buf_size(value); + } + if let Some(value) = options.max_concurrent_streams { + builder = builder.max_concurrent_streams(Some(value)); + } + if let Some(value) = options.max_header_list_size { + builder = builder.max_header_list_size(value); + } + if let Some(value) = options.max_pending_accept_reset_streams { + builder = builder.max_pending_accept_reset_streams(Some(value)); + } + if let Some(value) = options.enable_push { + builder = builder.enable_push(value); + } + if let Some(value) = options.header_table_size { + builder = builder.header_table_size(Some(value)); + } + if let Some(value) = options.enable_connect_protocol { + builder = builder.enable_connect_protocol(value); + } + if let Some(value) = options.no_rfc7540_priorities { + builder = builder.no_rfc7540_priorities(value); + } + if let Some(settings_order) = options.settings_order { + builder = builder.settings_order(Some(build_settings_order(settings_order)?)); + } + if let Some(pseudo_order) = options.headers_pseudo_order { + builder = builder.headers_pseudo_order(Some(build_pseudo_order(pseudo_order)?)); + } + if let Some(dep) = options.headers_stream_dependency { + builder = builder.headers_stream_dependency(Some(StreamDependency::new( + StreamId::from(dep.dependency_id), + dep.weight, + dep.exclusive, + ))); + } + if let Some(priorities) = options.priorities { + builder = builder.priorities(Some(build_priorities(priorities)?)); + } + if let Some(experimental_settings) = options.experimental_settings { + builder = builder + .experimental_settings(Some(build_experimental_settings(experimental_settings)?)); + } + + Ok(builder.build()) +} + +fn build_pseudo_order(pseudo_order: Vec) -> Result { + let mut builder = PseudoOrder::builder(); + let mut seen = HashSet::with_capacity(pseudo_order.len()); + + for pseudo_id in &pseudo_order { + let id = parse_pseudo_id(pseudo_id)?; + if !seen.insert(pseudo_id.clone()) { + bail!("Duplicate emulation http2Options.headersPseudoOrder entry: {pseudo_id}"); + } + builder = builder.push(id); + } + + Ok(builder.build()) +} + +fn build_settings_order(settings_order: Vec) -> Result { + let mut builder = SettingsOrder::builder(); + let mut seen = HashSet::with_capacity(settings_order.len()); + + for setting in settings_order { + let setting_id = parse_http2_setting_id(&setting)?; + if !seen.insert(setting_id.clone()) { + bail!("Duplicate emulation http2Options.settingsOrder entry: {setting}"); + } + builder = builder.push(setting_id); + } + + Ok(builder.build()) +} + +fn build_priorities(priorities: Vec) -> Result { + let mut builder = Priorities::builder(); + let mut seen_stream_ids = HashSet::with_capacity(priorities.len()); + + for priority in priorities { + if priority.stream_id == 0 { + bail!( + "Invalid emulation http2Options.priorities entry: streamId must be greater than 0" + ); + } + if !seen_stream_ids.insert(priority.stream_id) { + bail!( + "Duplicate emulation http2Options.priorities streamId: {}", + priority.stream_id + ); + } + + let dependency = StreamDependency::new( + StreamId::from(priority.dependency.dependency_id), + priority.dependency.weight, + priority.dependency.exclusive, + ); + + builder = builder.push(Priority::new( + StreamId::from(priority.stream_id), + dependency, + )); + } + + Ok(builder.build()) +} + +fn build_experimental_settings( + experimental_settings: Vec, +) -> Result { + let mut builder = ExperimentalSettings::builder(); + let mut seen_ids = HashSet::with_capacity(experimental_settings.len()); + let max_id = 15u16; + + for setting in experimental_settings { + if setting.id == 0 || setting.id > max_id { + bail!( + "Invalid emulation http2Options.experimentalSettings entry: id must be between 1 and {}", + max_id + ); + } + if !matches!(SettingId::from(setting.id), SettingId::Unknown(_)) { + bail!( + "Invalid emulation http2Options.experimentalSettings entry: {} is a standard HTTP/2 setting id", + setting.id + ); + } + if !seen_ids.insert(setting.id) { + bail!( + "Duplicate emulation http2Options.experimentalSettings id: {}", + setting.id + ); + } + + let setting = + Setting::from_id(SettingId::Unknown(setting.id), setting.value).ok_or_else(|| { + anyhow!( + "Invalid emulation http2Options.experimentalSettings id: {}", + setting.id + ) + })?; + builder = builder.push(setting); + } + + Ok(builder.build()) +} diff --git a/rust/src/emulation/mod.rs b/rust/src/emulation/mod.rs new file mode 100644 index 0000000..9c41762 --- /dev/null +++ b/rust/src/emulation/mod.rs @@ -0,0 +1,6 @@ +mod builders; +mod parse; +mod payload; +mod resolve; + +pub use resolve::resolve_emulation; diff --git a/rust/src/emulation/parse.rs b/rust/src/emulation/parse.rs new file mode 100644 index 0000000..b72fd1d --- /dev/null +++ b/rust/src/emulation/parse.rs @@ -0,0 +1,69 @@ +use anyhow::{bail, Result}; +use wreq::{ + http2::{PseudoId, SettingId}, + tls::{AlpnProtocol, AlpsProtocol, CertificateCompressionAlgorithm, TlsVersion}, +}; + +pub fn parse_tls_version(value: &str) -> Result { + match value { + "1.0" | "TLS1.0" => Ok(TlsVersion::TLS_1_0), + "1.1" | "TLS1.1" => Ok(TlsVersion::TLS_1_1), + "1.2" | "TLS1.2" => Ok(TlsVersion::TLS_1_2), + "1.3" | "TLS1.3" => Ok(TlsVersion::TLS_1_3), + other => bail!("Invalid TLS version: {other}"), + } +} + +pub fn parse_alpn_protocol(value: &str) -> Result { + match value { + "HTTP1" => Ok(AlpnProtocol::HTTP1), + "HTTP2" => Ok(AlpnProtocol::HTTP2), + "HTTP3" => Ok(AlpnProtocol::HTTP3), + other => bail!("Invalid ALPN protocol: {other}"), + } +} + +pub fn parse_alps_protocol(value: &str) -> Result { + match value { + "HTTP1" => Ok(AlpsProtocol::HTTP1), + "HTTP2" => Ok(AlpsProtocol::HTTP2), + "HTTP3" => Ok(AlpsProtocol::HTTP3), + other => bail!("Invalid ALPS protocol: {other}"), + } +} + +pub fn parse_certificate_compression_algorithm( + value: &str, +) -> Result { + match value { + "zlib" => Ok(CertificateCompressionAlgorithm::ZLIB), + "brotli" => Ok(CertificateCompressionAlgorithm::BROTLI), + "zstd" => Ok(CertificateCompressionAlgorithm::ZSTD), + other => bail!("Invalid certificate compression algorithm: {other}"), + } +} + +pub fn parse_pseudo_id(value: &str) -> Result { + match value { + "Method" => Ok(PseudoId::Method), + "Scheme" => Ok(PseudoId::Scheme), + "Authority" => Ok(PseudoId::Authority), + "Path" => Ok(PseudoId::Path), + "Protocol" => Ok(PseudoId::Protocol), + other => bail!("Invalid HTTP/2 pseudo-header id: {other}"), + } +} + +pub fn parse_http2_setting_id(value: &str) -> Result { + match value { + "HeaderTableSize" => Ok(SettingId::HeaderTableSize), + "EnablePush" => Ok(SettingId::EnablePush), + "MaxConcurrentStreams" => Ok(SettingId::MaxConcurrentStreams), + "InitialWindowSize" => Ok(SettingId::InitialWindowSize), + "MaxFrameSize" => Ok(SettingId::MaxFrameSize), + "MaxHeaderListSize" => Ok(SettingId::MaxHeaderListSize), + "EnableConnectProtocol" => Ok(SettingId::EnableConnectProtocol), + "NoRfc7540Priorities" => Ok(SettingId::NoRfc7540Priorities), + other => bail!("Invalid HTTP/2 setting id: {other}"), + } +} diff --git a/rust/src/emulation/payload.rs b/rust/src/emulation/payload.rs new file mode 100644 index 0000000..1029def --- /dev/null +++ b/rust/src/emulation/payload.rs @@ -0,0 +1,163 @@ +use serde::Deserialize; + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct CustomEmulationPayload { + #[serde(default)] + pub tls_options: Option, + #[serde(default)] + pub http1_options: Option, + #[serde(default)] + pub http2_options: Option, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct CustomTlsOptions { + #[serde(default)] + pub alpn_protocols: Option>, + #[serde(default)] + pub alps_protocols: Option>, + #[serde(default)] + pub alps_use_new_codepoint: Option, + #[serde(default)] + pub session_ticket: Option, + #[serde(default)] + pub min_tls_version: Option, + #[serde(default)] + pub max_tls_version: Option, + #[serde(default)] + pub pre_shared_key: Option, + #[serde(default)] + pub enable_ech_grease: Option, + #[serde(default)] + pub permute_extensions: Option, + #[serde(default)] + pub grease_enabled: Option, + #[serde(default)] + pub enable_ocsp_stapling: Option, + #[serde(default)] + pub enable_signed_cert_timestamps: Option, + #[serde(default)] + pub record_size_limit: Option, + #[serde(default)] + pub psk_skip_session_ticket: Option, + #[serde(default)] + pub key_shares_limit: Option, + #[serde(default)] + pub psk_dhe_ke: Option, + #[serde(default)] + pub renegotiation: Option, + #[serde(default)] + pub delegated_credentials: Option, + #[serde(default)] + pub curves_list: Option, + #[serde(default)] + pub cipher_list: Option, + #[serde(default)] + pub sigalgs_list: Option, + #[serde(default)] + pub certificate_compression_algorithms: Option>, + #[serde(default)] + pub extension_permutation: Option>, + #[serde(default)] + pub aes_hw_override: Option, + #[serde(default)] + pub preserve_tls13_cipher_list: Option, + #[serde(default)] + pub random_aes_hw_override: Option, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct CustomHttp1Options { + #[serde(default)] + pub http09_responses: Option, + #[serde(default)] + pub writev: Option, + #[serde(default)] + pub max_headers: Option, + #[serde(default)] + pub read_buf_exact_size: Option, + #[serde(default)] + pub max_buf_size: Option, + #[serde(default)] + pub ignore_invalid_headers_in_responses: Option, + #[serde(default)] + pub allow_spaces_after_header_name_in_responses: Option, + #[serde(default)] + pub allow_obsolete_multiline_headers_in_responses: Option, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct CustomHttp2Options { + #[serde(default)] + pub adaptive_window: Option, + #[serde(default)] + pub initial_stream_id: Option, + #[serde(default)] + pub initial_connection_window_size: Option, + #[serde(default)] + pub initial_window_size: Option, + #[serde(default)] + pub initial_max_send_streams: Option, + #[serde(default)] + pub max_frame_size: Option, + #[serde(default)] + pub keep_alive_interval: Option, + #[serde(default)] + pub keep_alive_timeout: Option, + #[serde(default)] + pub keep_alive_while_idle: Option, + #[serde(default)] + pub max_concurrent_reset_streams: Option, + #[serde(default)] + pub max_send_buffer_size: Option, + #[serde(default)] + pub max_concurrent_streams: Option, + #[serde(default)] + pub max_header_list_size: Option, + #[serde(default)] + pub max_pending_accept_reset_streams: Option, + #[serde(default)] + pub enable_push: Option, + #[serde(default)] + pub header_table_size: Option, + #[serde(default)] + pub enable_connect_protocol: Option, + #[serde(default)] + pub no_rfc7540_priorities: Option, + #[serde(default)] + pub settings_order: Option>, + #[serde(default)] + pub headers_pseudo_order: Option>, + #[serde(default)] + pub headers_stream_dependency: Option, + #[serde(default)] + pub priorities: Option>, + #[serde(default)] + pub experimental_settings: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CustomHttp2Priority { + pub stream_id: u32, + pub dependency: CustomHttp2StreamDependency, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CustomHttp2StreamDependency { + pub dependency_id: u32, + pub weight: u8, + #[serde(default)] + pub exclusive: bool, +} + +#[derive(Debug, Deserialize)] +pub struct CustomHttp2ExperimentalSetting { + pub id: u16, + pub value: u32, +} diff --git a/rust/src/emulation/resolve.rs b/rust/src/emulation/resolve.rs new file mode 100644 index 0000000..3e57ca7 --- /dev/null +++ b/rust/src/emulation/resolve.rs @@ -0,0 +1,39 @@ +use crate::emulation::builders::{build_http1_options, build_http2_options, build_tls_options}; +use crate::emulation::payload::CustomEmulationPayload; +use anyhow::{Context, Result}; +use wreq::{Emulation as WreqEmulation, EmulationFactory}; +use wreq_util::Emulation as BrowserEmulation; + +pub fn resolve_emulation( + browser: BrowserEmulation, + emulation_json: Option<&str>, +) -> Result { + let mut emulation = browser.emulation(); + + if let Some(emulation_json) = emulation_json { + let payload = parse_payload(emulation_json)?; + apply_payload(&mut emulation, payload)?; + } + + Ok(emulation) +} + +fn parse_payload(emulation_json: &str) -> Result { + serde_json::from_str(emulation_json).context("Failed to parse emulation JSON") +} + +fn apply_payload(emulation: &mut WreqEmulation, payload: CustomEmulationPayload) -> Result<()> { + if let Some(tls_options) = payload.tls_options { + *emulation.tls_options_mut() = Some(build_tls_options(tls_options)?); + } + + if let Some(http1_options) = payload.http1_options { + *emulation.http1_options_mut() = Some(build_http1_options(http1_options)?); + } + + if let Some(http2_options) = payload.http2_options { + *emulation.http2_options_mut() = Some(build_http2_options(http2_options)?); + } + + Ok(()) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 71605bf..bf681ad 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,583 +1,11 @@ -mod client; +mod emulation; +mod napi; +mod store; +mod transport; use neon::prelude::*; -use neon::types::buffer::TypedArray; -use neon::types::JsBuffer; -use client::{ - RequestOptions, Response, WebSocketConnectOptions, WebSocketConnection, WebSocketReadResult, - cancel_body, close_websocket, connect_websocket, execute_request, read_body_all, - read_body_chunk, read_websocket_message, send_websocket_binary, send_websocket_text, -}; -use std::collections::HashMap; -use wreq_util::Emulation; -// Parse browser string to Emulation enum -fn parse_emulation(browser: &str) -> Emulation { - match browser { - // Chrome - "chrome_100" => Emulation::Chrome100, - "chrome_101" => Emulation::Chrome101, - "chrome_104" => Emulation::Chrome104, - "chrome_105" => Emulation::Chrome105, - "chrome_106" => Emulation::Chrome106, - "chrome_107" => Emulation::Chrome107, - "chrome_108" => Emulation::Chrome108, - "chrome_109" => Emulation::Chrome109, - "chrome_110" => Emulation::Chrome110, - "chrome_114" => Emulation::Chrome114, - "chrome_116" => Emulation::Chrome116, - "chrome_117" => Emulation::Chrome117, - "chrome_118" => Emulation::Chrome118, - "chrome_119" => Emulation::Chrome119, - "chrome_120" => Emulation::Chrome120, - "chrome_123" => Emulation::Chrome123, - "chrome_124" => Emulation::Chrome124, - "chrome_126" => Emulation::Chrome126, - "chrome_127" => Emulation::Chrome127, - "chrome_128" => Emulation::Chrome128, - "chrome_129" => Emulation::Chrome129, - "chrome_130" => Emulation::Chrome130, - "chrome_131" => Emulation::Chrome131, - "chrome_132" => Emulation::Chrome132, - "chrome_133" => Emulation::Chrome133, - "chrome_134" => Emulation::Chrome134, - "chrome_135" => Emulation::Chrome135, - "chrome_136" => Emulation::Chrome136, - "chrome_137" => Emulation::Chrome137, - // Edge - "edge_101" => Emulation::Edge101, - "edge_122" => Emulation::Edge122, - "edge_127" => Emulation::Edge127, - "edge_131" => Emulation::Edge131, - "edge_134" => Emulation::Edge134, - // Safari - "safari_ios_17_2" => Emulation::SafariIos17_2, - "safari_ios_17_4_1" => Emulation::SafariIos17_4_1, - "safari_ios_16_5" => Emulation::SafariIos16_5, - "safari_15_3" => Emulation::Safari15_3, - "safari_15_5" => Emulation::Safari15_5, - "safari_15_6_1" => Emulation::Safari15_6_1, - "safari_16" => Emulation::Safari16, - "safari_16_5" => Emulation::Safari16_5, - "safari_17_0" => Emulation::Safari17_0, - "safari_17_2_1" => Emulation::Safari17_2_1, - "safari_17_4_1" => Emulation::Safari17_4_1, - "safari_17_5" => Emulation::Safari17_5, - "safari_18" => Emulation::Safari18, - "safari_ipad_18" => Emulation::SafariIPad18, - "safari_18_2" => Emulation::Safari18_2, - "safari_ios_18_1_1" => Emulation::SafariIos18_1_1, - "safari_18_3" => Emulation::Safari18_3, - "safari_18_3_1" => Emulation::Safari18_3_1, - "safari_18_5" => Emulation::Safari18_5, - // Firefox - "firefox_109" => Emulation::Firefox109, - "firefox_117" => Emulation::Firefox117, - "firefox_128" => Emulation::Firefox128, - "firefox_133" => Emulation::Firefox133, - "firefox_135" => Emulation::Firefox135, - "firefox_private_135" => Emulation::FirefoxPrivate135, - "firefox_android_135" => Emulation::FirefoxAndroid135, - "firefox_136" => Emulation::Firefox136, - "firefox_private_136" => Emulation::FirefoxPrivate136, - "firefox_139" => Emulation::Firefox139, - // Opera - "opera_116" => Emulation::Opera116, - "opera_117" => Emulation::Opera117, - "opera_118" => Emulation::Opera118, - "opera_119" => Emulation::Opera119, - // OkHttp - "okhttp_3_9" => Emulation::OkHttp3_9, - "okhttp_3_11" => Emulation::OkHttp3_11, - "okhttp_3_13" => Emulation::OkHttp3_13, - "okhttp_3_14" => Emulation::OkHttp3_14, - "okhttp_4_9" => Emulation::OkHttp4_9, - "okhttp_4_10" => Emulation::OkHttp4_10, - "okhttp_4_12" => Emulation::OkHttp4_12, - "okhttp_5" => Emulation::OkHttp5, - // Default to Chrome 137 - _ => Emulation::Chrome137, - } -} - -// Convert JS object to RequestOptions -fn js_object_to_request_options(cx: &mut FunctionContext, obj: Handle) -> NeonResult { - // Get URL (required) - let url: Handle = obj.get(cx, "url")?; - let url = url.value(cx); - - // Get browser (optional, defaults to chrome_137) - let browser_str = obj - .get_opt(cx, "browser")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx)) - .unwrap_or_else(|| "chrome_137".to_string()); - - let emulation = parse_emulation(&browser_str); - - // Get method (optional, defaults to GET) - let method = obj - .get_opt(cx, "method")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx)) - .unwrap_or_else(|| "GET".to_string()); - - // Get headers (optional) - let mut headers = HashMap::new(); - if let Ok(Some(headers_obj)) = obj.get_opt::(cx, "headers") { - let keys = headers_obj.get_own_property_names(cx)?; - let keys_vec = keys.to_vec(cx)?; - - for key_val in keys_vec { - if let Ok(key_str) = key_val.downcast::(cx) { - let key = key_str.value(cx); - if let Ok(value) = headers_obj.get::(cx, key.as_str()) { - headers.insert(key, value.value(cx)); - } - } - } - } - - // Get body (optional) - let body = obj - .get_opt(cx, "body")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx)); - - // Get proxy (optional) - let proxy = obj - .get_opt(cx, "proxy")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx)); - - // Get timeout (optional, defaults to 30000ms) - let timeout = obj - .get_opt(cx, "timeout")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx) as u64) - .unwrap_or(30000); - - Ok(RequestOptions { - url, - emulation, - headers, - method, - body, - proxy, - timeout, - }) -} - -fn js_object_to_websocket_options( - cx: &mut FunctionContext, - obj: Handle, -) -> NeonResult { - let url: Handle = obj.get(cx, "url")?; - let url = url.value(cx); - - let browser_str = obj - .get_opt(cx, "browser")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx)) - .unwrap_or_else(|| "chrome_137".to_string()); - - let emulation = parse_emulation(&browser_str); - - let mut headers = HashMap::new(); - if let Ok(Some(headers_obj)) = obj.get_opt::(cx, "headers") { - let keys = headers_obj.get_own_property_names(cx)?; - let keys_vec = keys.to_vec(cx)?; - - for key_val in keys_vec { - if let Ok(key_str) = key_val.downcast::(cx) { - let key = key_str.value(cx); - if let Ok(value) = headers_obj.get::(cx, key.as_str()) { - headers.insert(key, value.value(cx)); - } - } - } - } - - let proxy = obj - .get_opt(cx, "proxy")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx)); - - let timeout = obj - .get_opt(cx, "timeout")? - .and_then(|v: Handle| v.downcast::(cx).ok()) - .map(|v| v.value(cx) as u64) - .unwrap_or(30000); - - let mut protocols = Vec::new(); - if let Some(values) = obj.get_opt::(cx, "protocols")? { - for value in values.to_vec(cx)? { - if let Ok(value) = value.downcast::(cx) { - protocols.push(value.value(cx)); - } - } - } - - Ok(WebSocketConnectOptions { - url, - emulation, - headers, - proxy, - timeout, - protocols, - }) -} - -// Convert Response to JS object -fn response_to_js_object<'a, C: Context<'a>>(cx: &mut C, response: Response) -> JsResult<'a, JsObject> { - let obj = cx.empty_object(); - - // Status - let status = cx.number(response.status as f64); - obj.set(cx, "status", status)?; - - // URL - let url = cx.string(&response.url); - obj.set(cx, "url", url)?; - - // Headers - let headers_obj = cx.empty_object(); - for (key, value) in response.headers { - let value_str = cx.string(&value); - headers_obj.set(cx, key.as_str(), value_str)?; - } - obj.set(cx, "headers", headers_obj)?; - - // Cookies - let cookies_obj = cx.empty_object(); - for (key, value) in response.cookies { - let value_str = cx.string(&value); - cookies_obj.set(cx, key.as_str(), value_str)?; - } - obj.set(cx, "cookies", cookies_obj)?; - - // Raw Set-Cookie headers - let set_cookies = JsArray::new(cx, response.set_cookies.len()); - for (index, value) in response.set_cookies.into_iter().enumerate() { - let value_str = cx.string(&value); - set_cookies.set(cx, index as u32, value_str)?; - } - obj.set(cx, "setCookies", set_cookies)?; - - // Body handle - let body_handle = cx.number(response.body_handle as f64); - obj.set(cx, "bodyHandle", body_handle)?; - - Ok(obj) -} - -fn websocket_to_js_object<'a, C: Context<'a>>( - cx: &mut C, - websocket: WebSocketConnection, -) -> JsResult<'a, JsObject> { - let obj = cx.empty_object(); - let handle = cx.number(websocket.handle as f64); - let url = cx.string(&websocket.url); - - obj.set(cx, "handle", handle)?; - obj.set(cx, "url", url)?; - - match websocket.protocol { - Some(protocol) => { - let value = cx.string(protocol); - obj.set(cx, "protocol", value)?; - } - None => { - let value = cx.null(); - obj.set(cx, "protocol", value)?; - } - }; - - match websocket.extensions { - Some(extensions) => { - let value = cx.string(extensions); - obj.set(cx, "extensions", value)?; - } - None => { - let value = cx.null(); - obj.set(cx, "extensions", value)?; - } - }; - - Ok(obj) -} - -// Main request function exported to Node.js -fn request(mut cx: FunctionContext) -> JsResult { - // Get the options object - let options_obj = cx.argument::(0)?; - - // Convert JS object to Rust struct - let options = js_object_to_request_options(&mut cx, options_obj)?; - - // Create a promise - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - // Create a new Tokio runtime for this request - std::thread::spawn(move || { - let result = execute_request(options); - - // Send result back to JS - deferred.settle_with(&channel, move |mut cx| { - match result { - Ok(response) => response_to_js_object(&mut cx, response), - Err(e) => { - // Format error with full chain for better debugging - let error_msg = format!("{:#}", e); - cx.throw_error(error_msg) - } - } - }); - }); - - Ok(promise) -} - -fn read_body_chunk_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - let size = cx.argument_opt(1) - .and_then(|value| value.downcast::(&mut cx).ok()) - .map(|value| value.value(&mut cx) as usize) - .unwrap_or(65_536); - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = read_body_chunk(handle, size); - - deferred.settle_with(&channel, move |mut cx| { - match result { - Ok((chunk, done)) => { - let obj = cx.empty_object(); - let chunk_buffer = JsBuffer::from_slice(&mut cx, &chunk)?; - let done_value = cx.boolean(done); - obj.set(&mut cx, "chunk", chunk_buffer)?; - obj.set(&mut cx, "done", done_value)?; - Ok(obj) - } - Err(error) => cx.throw_error(format!("{:#}", error)), - } - }); - }); - - Ok(promise) -} - -fn read_body_all_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = read_body_all(handle); - - deferred.settle_with(&channel, move |mut cx| { - match result { - Ok(bytes) => JsBuffer::from_slice(&mut cx, &bytes), - Err(error) => cx.throw_error(format!("{:#}", error)), - } - }); - }); - - Ok(promise) -} - -fn cancel_body_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - Ok(cx.boolean(cancel_body(handle))) -} - -fn websocket_connect_js(mut cx: FunctionContext) -> JsResult { - let options_obj = cx.argument::(0)?; - let options = js_object_to_websocket_options(&mut cx, options_obj)?; - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = connect_websocket(options); - - deferred.settle_with(&channel, move |mut cx| match result { - Ok(websocket) => websocket_to_js_object(&mut cx, websocket), - Err(error) => cx.throw_error(format!("{:#}", error)), - }); - }); - - Ok(promise) -} - -fn websocket_read_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = read_websocket_message(handle); - - deferred.settle_with(&channel, move |mut cx| match result { - Ok(WebSocketReadResult::Text(text)) => { - let obj = cx.empty_object(); - let type_value = cx.string("text"); - let data_value = cx.string(text); - obj.set(&mut cx, "type", type_value)?; - obj.set(&mut cx, "data", data_value)?; - Ok(obj) - } - Ok(WebSocketReadResult::Binary(bytes)) => { - let obj = cx.empty_object(); - let type_value = cx.string("binary"); - let data_value = JsBuffer::from_slice(&mut cx, &bytes)?; - obj.set(&mut cx, "type", type_value)?; - obj.set(&mut cx, "data", data_value)?; - Ok(obj) - } - Ok(WebSocketReadResult::Close { - code, - reason, - was_clean, - }) => { - let obj = cx.empty_object(); - let type_value = cx.string("close"); - let code_value = cx.number(code as f64); - let reason_value = cx.string(reason); - let was_clean_value = cx.boolean(was_clean); - obj.set(&mut cx, "type", type_value)?; - obj.set(&mut cx, "code", code_value)?; - obj.set(&mut cx, "reason", reason_value)?; - obj.set(&mut cx, "wasClean", was_clean_value)?; - Ok(obj) - } - Err(error) => cx.throw_error(format!("{:#}", error)), - }); - }); - - Ok(promise) -} - -fn websocket_send_text_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - let text = cx.argument::(1)?.value(&mut cx); - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = send_websocket_text(handle, text); - - deferred.settle_with(&channel, move |mut cx| match result { - Ok(()) => Ok(cx.undefined()), - Err(error) => cx.throw_error(format!("{:#}", error)), - }); - }); - - Ok(promise) -} - -fn websocket_send_binary_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - let buffer = cx.argument::(1)?; - let bytes = buffer.as_slice(&cx).to_vec(); - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = send_websocket_binary(handle, bytes); - - deferred.settle_with(&channel, move |mut cx| match result { - Ok(()) => Ok(cx.undefined()), - Err(error) => cx.throw_error(format!("{:#}", error)), - }); - }); - - Ok(promise) -} - -fn websocket_close_js(mut cx: FunctionContext) -> JsResult { - let handle = cx.argument::(0)?.value(&mut cx) as u64; - let code = cx - .argument_opt(1) - .and_then(|value| value.downcast::(&mut cx).ok()) - .map(|value| value.value(&mut cx) as u16); - let reason = cx - .argument_opt(2) - .and_then(|value| value.downcast::(&mut cx).ok()) - .map(|value| value.value(&mut cx)); - - let channel = cx.channel(); - let (deferred, promise) = cx.promise(); - - std::thread::spawn(move || { - let result = close_websocket(handle, code, reason); - - deferred.settle_with(&channel, move |mut cx| match result { - Ok(()) => Ok(cx.undefined()), - Err(error) => cx.throw_error(format!("{:#}", error)), - }); - }); - - Ok(promise) -} - -// Get list of available browser profiles -fn get_profiles(mut cx: FunctionContext) -> JsResult { - let profiles = vec![ - // Chrome - "chrome_100", "chrome_101", "chrome_104", "chrome_105", "chrome_106", "chrome_107", - "chrome_108", "chrome_109", "chrome_110", "chrome_114", "chrome_116", "chrome_117", - "chrome_118", "chrome_119", "chrome_120", "chrome_123", "chrome_124", "chrome_126", - "chrome_127", "chrome_128", "chrome_129", "chrome_130", "chrome_131", "chrome_132", - "chrome_133", "chrome_134", "chrome_135", "chrome_136", "chrome_137", - // Edge - "edge_101", "edge_122", "edge_127", "edge_131", "edge_134", - // Safari - "safari_ios_17_2", "safari_ios_17_4_1", "safari_ios_16_5", - "safari_15_3", "safari_15_5", "safari_15_6_1", "safari_16", "safari_16_5", - "safari_17_0", "safari_17_2_1", "safari_17_4_1", "safari_17_5", "safari_18", - "safari_ipad_18", "safari_18_2", "safari_ios_18_1_1", - "safari_18_3", "safari_18_3_1", "safari_18_5", - // Firefox - "firefox_109", "firefox_117", "firefox_128", "firefox_133", "firefox_135", - "firefox_private_135", "firefox_android_135", - "firefox_136", "firefox_private_136", "firefox_139", - // Opera - "opera_116", "opera_117", "opera_118", "opera_119", - // OkHttp - "okhttp_3_9", "okhttp_3_11", "okhttp_3_13", "okhttp_3_14", - "okhttp_4_9", "okhttp_4_10", "okhttp_4_12", "okhttp_5", - ]; - - let js_array = cx.empty_array(); - - for (i, profile) in profiles.iter().enumerate() { - let js_string = cx.string(*profile); - js_array.set(&mut cx, i as u32, js_string)?; - } - - Ok(js_array) -} - -// Module initialization #[neon::main] fn main(mut cx: ModuleContext) -> NeonResult<()> { - cx.export_function("request", request)?; - cx.export_function("readBodyChunk", read_body_chunk_js)?; - cx.export_function("readBodyAll", read_body_all_js)?; - cx.export_function("cancelBody", cancel_body_js)?; - cx.export_function("websocketConnect", websocket_connect_js)?; - cx.export_function("websocketRead", websocket_read_js)?; - cx.export_function("websocketSendText", websocket_send_text_js)?; - cx.export_function("websocketSendBinary", websocket_send_binary_js)?; - cx.export_function("websocketClose", websocket_close_js)?; - cx.export_function("getProfiles", get_profiles)?; - Ok(()) + napi::register(&mut cx) } diff --git a/rust/src/napi/body.rs b/rust/src/napi/body.rs new file mode 100644 index 0000000..db986ae --- /dev/null +++ b/rust/src/napi/body.rs @@ -0,0 +1,63 @@ +use crate::store::body_store::{cancel_body, read_body_all, read_body_chunk}; +use neon::prelude::*; +use neon::types::JsBuffer; + +fn read_body_chunk_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let size = cx + .argument_opt(1) + .and_then(|value| value.downcast::(&mut cx).ok()) + .map(|value| value.value(&mut cx) as usize) + .unwrap_or(65_536); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = read_body_chunk(handle, size); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok((chunk, done)) => { + let obj = cx.empty_object(); + let chunk_buffer = JsBuffer::from_slice(&mut cx, &chunk)?; + let done_value = cx.boolean(done); + obj.set(&mut cx, "chunk", chunk_buffer)?; + obj.set(&mut cx, "done", done_value)?; + Ok(obj) + } + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn read_body_all_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = read_body_all(handle); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(bytes) => JsBuffer::from_slice(&mut cx, &bytes), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn cancel_body_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + Ok(cx.boolean(cancel_body(handle))) +} + +pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { + cx.export_function("readBodyChunk", read_body_chunk_js)?; + cx.export_function("readBodyAll", read_body_all_js)?; + cx.export_function("cancelBody", cancel_body_js)?; + Ok(()) +} diff --git a/rust/src/napi/convert.rs b/rust/src/napi/convert.rs new file mode 100644 index 0000000..45fd8df --- /dev/null +++ b/rust/src/napi/convert.rs @@ -0,0 +1,275 @@ +use crate::emulation::resolve_emulation; +use crate::napi::profiles::parse_browser_emulation; +use crate::transport::types::{ + RequestOptions, Response, WebSocketConnectOptions, WebSocketConnection, +}; +use neon::prelude::*; + +pub(crate) fn js_value_to_string_array( + cx: &mut FunctionContext, + value: Handle, +) -> NeonResult> { + let array = value.downcast::(cx).or_throw(cx)?; + let mut strings = Vec::with_capacity(array.len(cx) as usize); + + for item in array.to_vec(cx)? { + let string = item.downcast::(cx).or_throw(cx)?; + strings.push(string.value(cx)); + } + + Ok(strings) +} + +pub(crate) fn js_value_to_header_tuples( + cx: &mut FunctionContext, + value: Handle, +) -> NeonResult> { + let array = value.downcast::(cx).or_throw(cx)?; + let mut tuples = Vec::with_capacity(array.len(cx) as usize); + + for item in array.to_vec(cx)? { + let tuple = item.downcast::(cx).or_throw(cx)?; + if tuple.len(cx) != 2 { + return cx.throw_type_error("Header tuple entries must contain exactly 2 items"); + } + + let name = tuple.get::(cx, 0)?.value(cx); + let value = tuple.get::(cx, 1)?.value(cx); + tuples.push((name, value)); + } + + Ok(tuples) +} + +pub(crate) fn js_object_to_request_options( + cx: &mut FunctionContext, + obj: Handle, +) -> NeonResult { + let url: Handle = obj.get(cx, "url")?; + let url = url.value(cx); + + let browser_str = obj + .get_opt(cx, "browser")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or_else(|| "chrome_137".to_string()); + + let emulation_json = obj + .get_opt(cx, "emulationJson")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)); + + let emulation = resolve_emulation( + parse_browser_emulation(&browser_str), + emulation_json.as_deref(), + ) + .or_else(|error| cx.throw_error(format!("{:#}", error)))?; + + let method = obj + .get_opt(cx, "method")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or_else(|| "GET".to_string()); + + let headers = obj + .get_opt(cx, "headers")? + .map(|v| js_value_to_header_tuples(cx, v)) + .transpose()? + .unwrap_or_default(); + + let orig_headers = obj + .get_opt(cx, "origHeaders")? + .map(|v| js_value_to_string_array(cx, v)) + .transpose()? + .unwrap_or_default(); + + let body = obj + .get_opt(cx, "body")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)); + + let proxy = obj + .get_opt(cx, "proxy")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)); + + let timeout = obj + .get_opt(cx, "timeout")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx) as u64) + .unwrap_or(30000); + + let disable_default_headers = obj + .get_opt(cx, "disableDefaultHeaders")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or(false); + + let compress = obj + .get_opt(cx, "compress")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or(true); + + Ok(RequestOptions { + url, + emulation, + headers, + orig_headers, + method, + body, + proxy, + timeout, + disable_default_headers, + compress, + }) +} + +pub(crate) fn js_object_to_websocket_options( + cx: &mut FunctionContext, + obj: Handle, +) -> NeonResult { + let url: Handle = obj.get(cx, "url")?; + let url = url.value(cx); + + let browser_str = obj + .get_opt(cx, "browser")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or_else(|| "chrome_137".to_string()); + + let emulation_json = obj + .get_opt(cx, "emulationJson")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)); + + let emulation = resolve_emulation( + parse_browser_emulation(&browser_str), + emulation_json.as_deref(), + ) + .or_else(|error| cx.throw_error(format!("{:#}", error)))?; + + let headers = obj + .get_opt(cx, "headers")? + .map(|v| js_value_to_header_tuples(cx, v)) + .transpose()? + .unwrap_or_default(); + + let orig_headers = obj + .get_opt(cx, "origHeaders")? + .map(|v| js_value_to_string_array(cx, v)) + .transpose()? + .unwrap_or_default(); + + let proxy = obj + .get_opt(cx, "proxy")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)); + + let timeout = obj + .get_opt(cx, "timeout")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx) as u64) + .unwrap_or(30000); + + let disable_default_headers = obj + .get_opt(cx, "disableDefaultHeaders")? + .and_then(|v: Handle| v.downcast::(cx).ok()) + .map(|v| v.value(cx)) + .unwrap_or(false); + + let mut protocols = Vec::new(); + if let Some(values) = obj.get_opt::(cx, "protocols")? { + for value in values.to_vec(cx)? { + if let Ok(value) = value.downcast::(cx) { + protocols.push(value.value(cx)); + } + } + } + + Ok(WebSocketConnectOptions { + url, + emulation, + headers, + orig_headers, + proxy, + timeout, + disable_default_headers, + protocols, + }) +} + +pub(crate) fn response_to_js_object<'a, C: Context<'a>>( + cx: &mut C, + response: Response, +) -> JsResult<'a, JsObject> { + let obj = cx.empty_object(); + + let status = cx.number(response.status as f64); + obj.set(cx, "status", status)?; + + let url = cx.string(&response.url); + obj.set(cx, "url", url)?; + + let headers_obj = cx.empty_object(); + for (key, value) in response.headers { + let value_str = cx.string(&value); + headers_obj.set(cx, key.as_str(), value_str)?; + } + obj.set(cx, "headers", headers_obj)?; + + let cookies_obj = cx.empty_object(); + for (key, value) in response.cookies { + let value_str = cx.string(&value); + cookies_obj.set(cx, key.as_str(), value_str)?; + } + obj.set(cx, "cookies", cookies_obj)?; + + let set_cookies = JsArray::new(cx, response.set_cookies.len()); + for (index, value) in response.set_cookies.into_iter().enumerate() { + let value_str = cx.string(&value); + set_cookies.set(cx, index as u32, value_str)?; + } + obj.set(cx, "setCookies", set_cookies)?; + + let body_handle = cx.number(response.body_handle as f64); + obj.set(cx, "bodyHandle", body_handle)?; + + Ok(obj) +} + +pub(crate) fn websocket_to_js_object<'a, C: Context<'a>>( + cx: &mut C, + websocket: WebSocketConnection, +) -> JsResult<'a, JsObject> { + let obj = cx.empty_object(); + let handle = cx.number(websocket.handle as f64); + let url = cx.string(&websocket.url); + + obj.set(cx, "handle", handle)?; + obj.set(cx, "url", url)?; + + match websocket.protocol { + Some(protocol) => { + let value = cx.string(protocol); + obj.set(cx, "protocol", value)?; + } + None => { + let value = cx.null(); + obj.set(cx, "protocol", value)?; + } + }; + + match websocket.extensions { + Some(extensions) => { + let value = cx.string(extensions); + obj.set(cx, "extensions", value)?; + } + None => { + let value = cx.null(); + obj.set(cx, "extensions", value)?; + } + }; + + Ok(obj) +} diff --git a/rust/src/napi/mod.rs b/rust/src/napi/mod.rs new file mode 100644 index 0000000..db48e6e --- /dev/null +++ b/rust/src/napi/mod.rs @@ -0,0 +1,15 @@ +mod body; +mod convert; +mod profiles; +mod request; +mod websocket; + +use neon::prelude::*; + +pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { + request::register(cx)?; + body::register(cx)?; + websocket::register(cx)?; + profiles::register(cx)?; + Ok(()) +} diff --git a/rust/src/napi/profiles.rs b/rust/src/napi/profiles.rs new file mode 100644 index 0000000..751680e --- /dev/null +++ b/rust/src/napi/profiles.rs @@ -0,0 +1,56 @@ +use neon::prelude::*; +use std::collections::HashMap; +use std::sync::OnceLock; +use strum::VariantArray; +use wreq_util::Emulation as BrowserEmulation; + +static PROFILE_NAMES: OnceLock> = OnceLock::new(); +static PROFILE_MAP: OnceLock> = OnceLock::new(); + +fn serialize_emulation_name(emulation: BrowserEmulation) -> String { + serde_json::to_string(&emulation) + .expect("failed to serialize emulation profile") + .trim_matches('"') + .replace('.', "_") +} + +fn profile_names() -> &'static Vec { + PROFILE_NAMES.get_or_init(|| { + BrowserEmulation::VARIANTS + .iter() + .map(|emulation| serialize_emulation_name(*emulation)) + .collect() + }) +} + +fn profile_map() -> &'static HashMap { + PROFILE_MAP.get_or_init(|| { + BrowserEmulation::VARIANTS + .iter() + .map(|emulation| (serialize_emulation_name(*emulation), *emulation)) + .collect() + }) +} + +pub(crate) fn parse_browser_emulation(browser: &str) -> BrowserEmulation { + profile_map() + .get(browser) + .copied() + .unwrap_or(BrowserEmulation::Chrome137) +} + +fn get_profiles(mut cx: FunctionContext) -> JsResult { + let js_array = cx.empty_array(); + + for (i, profile) in profile_names().iter().enumerate() { + let js_string = cx.string(profile); + js_array.set(&mut cx, i as u32, js_string)?; + } + + Ok(js_array) +} + +pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { + cx.export_function("getProfiles", get_profiles)?; + Ok(()) +} diff --git a/rust/src/napi/request.rs b/rust/src/napi/request.rs new file mode 100644 index 0000000..1447b2a --- /dev/null +++ b/rust/src/napi/request.rs @@ -0,0 +1,27 @@ +use crate::napi::convert::{js_object_to_request_options, response_to_js_object}; +use crate::transport::execute_request; +use neon::prelude::*; + +fn request(mut cx: FunctionContext) -> JsResult { + let options_obj = cx.argument::(0)?; + let options = js_object_to_request_options(&mut cx, options_obj)?; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = execute_request(options); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(response) => response_to_js_object(&mut cx, response), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { + cx.export_function("request", request)?; + Ok(()) +} diff --git a/rust/src/napi/websocket.rs b/rust/src/napi/websocket.rs new file mode 100644 index 0000000..bf8b4ef --- /dev/null +++ b/rust/src/napi/websocket.rs @@ -0,0 +1,150 @@ +use crate::napi::convert::{js_object_to_websocket_options, websocket_to_js_object}; +use crate::store::websocket_store::{ + close_websocket, read_websocket_message, send_websocket_binary, send_websocket_text, +}; +use crate::transport::{connect_websocket, types::WebSocketReadResult}; +use neon::prelude::*; +use neon::types::buffer::TypedArray; +use neon::types::JsBuffer; + +fn websocket_connect_js(mut cx: FunctionContext) -> JsResult { + let options_obj = cx.argument::(0)?; + let options = js_object_to_websocket_options(&mut cx, options_obj)?; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = connect_websocket(options); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(websocket) => websocket_to_js_object(&mut cx, websocket), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_read_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = read_websocket_message(handle); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(WebSocketReadResult::Text(text)) => { + let obj = cx.empty_object(); + let type_value = cx.string("text"); + let data_value = cx.string(text); + obj.set(&mut cx, "type", type_value)?; + obj.set(&mut cx, "data", data_value)?; + Ok(obj) + } + Ok(WebSocketReadResult::Binary(bytes)) => { + let obj = cx.empty_object(); + let type_value = cx.string("binary"); + let data_value = JsBuffer::from_slice(&mut cx, &bytes)?; + obj.set(&mut cx, "type", type_value)?; + obj.set(&mut cx, "data", data_value)?; + Ok(obj) + } + Ok(WebSocketReadResult::Close { + code, + reason, + was_clean, + }) => { + let obj = cx.empty_object(); + let type_value = cx.string("close"); + let code_value = cx.number(code as f64); + let reason_value = cx.string(reason); + let was_clean_value = cx.boolean(was_clean); + obj.set(&mut cx, "type", type_value)?; + obj.set(&mut cx, "code", code_value)?; + obj.set(&mut cx, "reason", reason_value)?; + obj.set(&mut cx, "wasClean", was_clean_value)?; + Ok(obj) + } + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_send_text_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let text = cx.argument::(1)?.value(&mut cx); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = send_websocket_text(handle, text); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(()) => Ok(cx.undefined()), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_send_binary_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let buffer = cx.argument::(1)?; + let bytes = buffer.as_slice(&cx).to_vec(); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = send_websocket_binary(handle, bytes); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(()) => Ok(cx.undefined()), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +fn websocket_close_js(mut cx: FunctionContext) -> JsResult { + let handle = cx.argument::(0)?.value(&mut cx) as u64; + let code = cx + .argument_opt(1) + .and_then(|value| value.downcast::(&mut cx).ok()) + .map(|value| value.value(&mut cx) as u16); + let reason = cx + .argument_opt(2) + .and_then(|value| value.downcast::(&mut cx).ok()) + .map(|value| value.value(&mut cx)); + + let channel = cx.channel(); + let (deferred, promise) = cx.promise(); + + std::thread::spawn(move || { + let result = close_websocket(handle, code, reason); + + deferred.settle_with(&channel, move |mut cx| match result { + Ok(()) => Ok(cx.undefined()), + Err(error) => cx.throw_error(format!("{:#}", error)), + }); + }); + + Ok(promise) +} + +pub fn register(cx: &mut ModuleContext) -> NeonResult<()> { + cx.export_function("websocketConnect", websocket_connect_js)?; + cx.export_function("websocketRead", websocket_read_js)?; + cx.export_function("websocketSendText", websocket_send_text_js)?; + cx.export_function("websocketSendBinary", websocket_send_binary_js)?; + cx.export_function("websocketClose", websocket_close_js)?; + Ok(()) +} diff --git a/rust/src/store/body_store.rs b/rust/src/store/body_store.rs new file mode 100644 index 0000000..e00e62e --- /dev/null +++ b/rust/src/store/body_store.rs @@ -0,0 +1,82 @@ +use crate::store::runtime::runtime; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, OnceLock, +}; + +#[derive(Debug)] +struct StoredBody { + response: wreq::Response, +} + +static NEXT_BODY_HANDLE: AtomicU64 = AtomicU64::new(1); +static BODY_STORE: OnceLock>> = OnceLock::new(); + +fn body_store() -> &'static Mutex> { + BODY_STORE.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub fn store_body(response: wreq::Response) -> u64 { + let handle = NEXT_BODY_HANDLE.fetch_add(1, Ordering::Relaxed); + body_store() + .lock() + .expect("body store poisoned") + .insert(handle, StoredBody { response }); + handle +} + +pub fn read_body_chunk(handle: u64, _size: usize) -> Result<(Vec, bool)> { + let mut store = body_store() + .lock() + .map_err(|_| anyhow::anyhow!("body store poisoned"))?; + let Some(body) = store.get_mut(&handle) else { + return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); + }; + + let chunk = runtime() + .block_on(body.response.chunk()) + .context("Failed to read response body chunk")?; + + let Some(chunk) = chunk else { + store.remove(&handle); + return Ok((Vec::new(), true)); + }; + + Ok((chunk.to_vec(), false)) +} + +pub fn read_body_all(handle: u64) -> Result> { + let mut store = body_store() + .lock() + .map_err(|_| anyhow::anyhow!("body store poisoned"))?; + let Some(body) = store.remove(&handle) else { + return Err(anyhow::anyhow!("Unknown body handle: {}", handle)); + }; + + let mut bytes = Vec::new(); + let mut response = body.response; + + runtime().block_on(async { + while let Some(chunk) = response + .chunk() + .await + .context("Failed to read response body chunk")? + { + bytes.extend_from_slice(&chunk); + } + + Ok::<(), anyhow::Error>(()) + })?; + + Ok(bytes) +} + +pub fn cancel_body(handle: u64) -> bool { + body_store() + .lock() + .expect("body store poisoned") + .remove(&handle) + .is_some() +} diff --git a/rust/src/store/mod.rs b/rust/src/store/mod.rs new file mode 100644 index 0000000..b27735a --- /dev/null +++ b/rust/src/store/mod.rs @@ -0,0 +1,3 @@ +pub mod body_store; +pub mod runtime; +pub mod websocket_store; diff --git a/rust/src/store/runtime.rs b/rust/src/store/runtime.rs new file mode 100644 index 0000000..990a0ba --- /dev/null +++ b/rust/src/store/runtime.rs @@ -0,0 +1,8 @@ +use std::sync::OnceLock; + +static TOKIO_RUNTIME: OnceLock = OnceLock::new(); + +pub fn runtime() -> &'static tokio::runtime::Runtime { + TOKIO_RUNTIME + .get_or_init(|| tokio::runtime::Runtime::new().expect("Failed to create Tokio runtime")) +} diff --git a/rust/src/store/websocket_store.rs b/rust/src/store/websocket_store.rs new file mode 100644 index 0000000..6458502 --- /dev/null +++ b/rust/src/store/websocket_store.rs @@ -0,0 +1,112 @@ +use crate::transport::types::WebSocketReadResult; +use anyhow::Result; +use std::collections::HashMap; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, OnceLock, +}; + +#[derive(Debug)] +pub(crate) enum WebSocketCommand { + Text(String), + Binary(Vec), + Close { + code: Option, + reason: Option, + }, +} + +#[derive(Debug)] +pub(crate) struct StoredWebSocket { + pub commands: tokio::sync::mpsc::UnboundedSender, + pub events: tokio::sync::Mutex>, +} + +pub(crate) type SharedWebSocket = Arc; + +static NEXT_WEBSOCKET_HANDLE: AtomicU64 = AtomicU64::new(1); +static WEBSOCKET_STORE: OnceLock>> = OnceLock::new(); + +fn websocket_store() -> &'static Mutex> { + WEBSOCKET_STORE.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub(crate) fn insert_websocket( + commands: tokio::sync::mpsc::UnboundedSender, + events: tokio::sync::mpsc::UnboundedReceiver, +) -> u64 { + let handle = NEXT_WEBSOCKET_HANDLE.fetch_add(1, Ordering::Relaxed); + + websocket_store() + .lock() + .expect("websocket store poisoned") + .insert( + handle, + Arc::new(StoredWebSocket { + commands, + events: tokio::sync::Mutex::new(events), + }), + ); + + handle +} + +fn get_websocket(handle: u64) -> Result { + let store = websocket_store() + .lock() + .map_err(|_| anyhow::anyhow!("websocket store poisoned"))?; + + store + .get(&handle) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Unknown websocket handle: {}", handle)) +} + +pub(crate) fn remove_websocket(handle: u64) { + websocket_store() + .lock() + .expect("websocket store poisoned") + .remove(&handle); +} + +pub fn read_websocket_message(handle: u64) -> Result { + let websocket = get_websocket(handle)?; + + let result = crate::store::runtime::runtime().block_on(async { + let mut events = websocket.events.lock().await; + events + .recv() + .await + .ok_or_else(|| anyhow::anyhow!("WebSocket event stream is closed")) + }); + + if matches!(result, Ok(WebSocketReadResult::Close { .. })) { + remove_websocket(handle); + } + + result +} + +pub fn send_websocket_text(handle: u64, text: String) -> Result<()> { + let websocket = get_websocket(handle)?; + websocket + .commands + .send(WebSocketCommand::Text(text)) + .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) +} + +pub fn send_websocket_binary(handle: u64, bytes: Vec) -> Result<()> { + let websocket = get_websocket(handle)?; + websocket + .commands + .send(WebSocketCommand::Binary(bytes)) + .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) +} + +pub fn close_websocket(handle: u64, code: Option, reason: Option) -> Result<()> { + let websocket = get_websocket(handle)?; + websocket + .commands + .send(WebSocketCommand::Close { code, reason }) + .map_err(|_| anyhow::anyhow!("WebSocket is already closed")) +} diff --git a/rust/src/transport/cookies.rs b/rust/src/transport/cookies.rs new file mode 100644 index 0000000..17f6d55 --- /dev/null +++ b/rust/src/transport/cookies.rs @@ -0,0 +1,6 @@ +pub fn parse_cookie_pair(set_cookie: &str) -> Option<(String, String)> { + let pair = set_cookie.split(';').next()?.trim(); + let (name, value) = pair.split_once('=')?; + + Some((name.to_string(), value.to_string())) +} diff --git a/rust/src/transport/headers.rs b/rust/src/transport/headers.rs new file mode 100644 index 0000000..f31b265 --- /dev/null +++ b/rust/src/transport/headers.rs @@ -0,0 +1,9 @@ +use wreq::header::OrigHeaderMap; + +pub fn build_orig_header_map(orig_headers: &[String]) -> OrigHeaderMap { + let mut map = OrigHeaderMap::with_capacity(orig_headers.len()); + for header in orig_headers { + map.insert(header.clone()); + } + map +} diff --git a/rust/src/transport/mod.rs b/rust/src/transport/mod.rs new file mode 100644 index 0000000..b41b8f6 --- /dev/null +++ b/rust/src/transport/mod.rs @@ -0,0 +1,8 @@ +mod cookies; +mod headers; +mod request; +pub mod types; +mod websocket; + +pub use request::execute_request; +pub use websocket::connect_websocket; diff --git a/rust/src/transport/request.rs b/rust/src/transport/request.rs new file mode 100644 index 0000000..df27cb0 --- /dev/null +++ b/rust/src/transport/request.rs @@ -0,0 +1,102 @@ +use crate::store::body_store::store_body; +use crate::store::runtime::runtime; +use crate::transport::cookies::parse_cookie_pair; +use crate::transport::headers::build_orig_header_map; +use crate::transport::types::{RequestOptions, Response}; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::time::Duration; +use wreq::redirect; + +pub fn execute_request(options: RequestOptions) -> Result { + runtime().block_on(make_request(options)) +} + +pub async fn make_request(options: RequestOptions) -> Result { + let mut client_builder = wreq::Client::builder() + .emulation(options.emulation) + .cookie_store(true); + + if let Some(proxy_url) = &options.proxy { + let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; + client_builder = client_builder.proxy(proxy); + } + + let orig_headers = build_orig_header_map(&options.orig_headers); + let client = client_builder + .build() + .context("Failed to build HTTP client")?; + + let method = if options.method.is_empty() { + "GET" + } else { + &options.method + }; + + let mut request = match method.to_uppercase().as_str() { + "GET" => client.get(&options.url), + "POST" => client.post(&options.url), + "PUT" => client.put(&options.url), + "DELETE" => client.delete(&options.url), + "PATCH" => client.patch(&options.url), + "HEAD" => client.head(&options.url), + _ => return Err(anyhow::anyhow!("Unsupported HTTP method: {}", method)), + }; + + for (key, value) in &options.headers { + request = request.header(key, value); + } + + if !orig_headers.is_empty() { + request = request.orig_headers(orig_headers); + } + + if let Some(body) = options.body { + request = request.body(body); + } + + request = request.timeout(Duration::from_millis(options.timeout)); + request = request.redirect(redirect::Policy::none()); + request = request.default_headers(!options.disable_default_headers); + request = request.gzip(options.compress); + request = request.brotli(options.compress); + request = request.deflate(options.compress); + + let response = request + .send() + .await + .with_context(|| format!("{} {}", method, options.url))?; + + let status = response.status().as_u16(); + let final_url = response.uri().to_string(); + + let mut response_headers = HashMap::new(); + for (key, value) in response.headers() { + if let Ok(value_str) = value.to_str() { + response_headers.insert(key.to_string(), value_str.to_string()); + } + } + + let mut cookies = HashMap::new(); + let mut set_cookies = Vec::new(); + for cookie_header in response.headers().get_all("set-cookie") { + if let Ok(cookie_str) = cookie_header.to_str() { + set_cookies.push(cookie_str.to_string()); + + if let Some((key, value)) = parse_cookie_pair(cookie_str) { + cookies.insert(key, value); + } + } + } + + let body_handle = store_body(response); + + Ok(Response { + status, + headers: response_headers, + body_handle, + cookies, + set_cookies, + url: final_url, + }) +} diff --git a/rust/src/transport/types.rs b/rust/src/transport/types.rs new file mode 100644 index 0000000..f4c036d --- /dev/null +++ b/rust/src/transport/types.rs @@ -0,0 +1,57 @@ +use std::collections::HashMap; +use wreq::Emulation; + +#[derive(Debug, Clone)] +pub struct RequestOptions { + pub url: String, + pub emulation: Emulation, + pub headers: Vec<(String, String)>, + pub orig_headers: Vec, + pub method: String, + pub body: Option, + pub proxy: Option, + pub timeout: u64, + pub disable_default_headers: bool, + pub compress: bool, +} + +#[derive(Debug, Clone)] +pub struct Response { + pub status: u16, + pub headers: HashMap, + pub body_handle: u64, + pub cookies: HashMap, + pub set_cookies: Vec, + pub url: String, +} + +#[derive(Debug, Clone)] +pub struct WebSocketConnectOptions { + pub url: String, + pub emulation: Emulation, + pub headers: Vec<(String, String)>, + pub orig_headers: Vec, + pub proxy: Option, + pub timeout: u64, + pub disable_default_headers: bool, + pub protocols: Vec, +} + +#[derive(Debug, Clone)] +pub struct WebSocketConnection { + pub handle: u64, + pub protocol: Option, + pub extensions: Option, + pub url: String, +} + +#[derive(Debug, Clone)] +pub enum WebSocketReadResult { + Text(String), + Binary(Vec), + Close { + code: u16, + reason: String, + was_clean: bool, + }, +} diff --git a/rust/src/transport/websocket.rs b/rust/src/transport/websocket.rs new file mode 100644 index 0000000..b93b120 --- /dev/null +++ b/rust/src/transport/websocket.rs @@ -0,0 +1,196 @@ +use crate::store::runtime::runtime; +use crate::store::websocket_store::{insert_websocket, WebSocketCommand}; +use crate::transport::headers::build_orig_header_map; +use crate::transport::types::{WebSocketConnectOptions, WebSocketConnection, WebSocketReadResult}; +use anyhow::{Context, Result}; +use std::time::Duration; +use wreq::ws::message::{CloseCode, CloseFrame, Message}; +use wreq::ws::WebSocket; + +pub fn connect_websocket(options: WebSocketConnectOptions) -> Result { + runtime().block_on(make_websocket(options)) +} + +async fn run_websocket_task( + mut websocket: WebSocket, + mut commands: tokio::sync::mpsc::UnboundedReceiver, + events: tokio::sync::mpsc::UnboundedSender, +) { + let mut close_requested = false; + let mut requested_close_code = 1000; + let mut requested_close_reason = String::new(); + + loop { + tokio::select! { + command = commands.recv() => { + match command { + Some(WebSocketCommand::Text(text)) => { + if websocket.send(Message::Text(text.into())).await.is_err() { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + } + Some(WebSocketCommand::Binary(bytes)) => { + if websocket.send(Message::Binary(bytes.into())).await.is_err() { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + } + Some(WebSocketCommand::Close { code, reason }) => { + close_requested = true; + requested_close_code = code.unwrap_or(1000); + requested_close_reason = reason.unwrap_or_default(); + + let frame = Message::Close(Some(CloseFrame { + code: CloseCode::from(requested_close_code), + reason: requested_close_reason.clone().into(), + })); + + if websocket.send(frame).await.is_err() { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + } + None => { + break; + } + } + } + message = websocket.recv() => { + match message { + Some(Ok(Message::Text(text))) => { + if events.send(WebSocketReadResult::Text(text.to_string())).is_err() { + break; + } + } + Some(Ok(Message::Binary(bytes))) => { + if events.send(WebSocketReadResult::Binary(bytes.to_vec())).is_err() { + break; + } + } + Some(Ok(Message::Close(frame))) => { + let (code, reason) = match frame { + Some(frame) => (u16::from(frame.code), frame.reason.to_string()), + None => { + if close_requested { + (requested_close_code, requested_close_reason.clone()) + } else { + (1005, String::new()) + } + } + }; + + let _ = events.send(WebSocketReadResult::Close { + code, + reason, + was_clean: true, + }); + break; + } + Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => {} + Some(Err(_)) => { + let _ = events.send(WebSocketReadResult::Close { + code: 1006, + reason: String::new(), + was_clean: false, + }); + break; + } + None => { + let _ = events.send(WebSocketReadResult::Close { + code: if close_requested { + requested_close_code + } else { + 1006 + }, + reason: if close_requested { + requested_close_reason.clone() + } else { + String::new() + }, + was_clean: close_requested, + }); + break; + } + } + } + } + } +} + +async fn make_websocket(options: WebSocketConnectOptions) -> Result { + let mut client_builder = wreq::Client::builder() + .emulation(options.emulation) + .cookie_store(true) + .timeout(Duration::from_millis(options.timeout)); + + if let Some(proxy_url) = &options.proxy { + let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; + client_builder = client_builder.proxy(proxy); + } + + let client = client_builder + .build() + .context("Failed to build WebSocket client")?; + + let mut request = client.websocket(&options.url); + let orig_headers = build_orig_header_map(&options.orig_headers); + for (key, value) in &options.headers { + request = request.header(key, value); + } + + if !orig_headers.is_empty() { + request = request.orig_headers(orig_headers); + } + + request = request.default_headers(!options.disable_default_headers); + + if !options.protocols.is_empty() { + request = request.protocols(options.protocols.iter().cloned()); + } + + let response = request + .send() + .await + .with_context(|| format!("WS {}", options.url))?; + + let extensions = response + .headers() + .get("sec-websocket-extensions") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + + let websocket = response + .into_websocket() + .await + .with_context(|| format!("WS upgrade {}", options.url))?; + + let protocol = websocket + .protocol() + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + + let (command_tx, command_rx) = tokio::sync::mpsc::unbounded_channel(); + let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel(); + runtime().spawn(run_websocket_task(websocket, command_rx, event_tx)); + let handle = insert_websocket(command_tx, event_rx); + + Ok(WebSocketConnection { + handle, + protocol, + extensions, + url: options.url, + }) +} diff --git a/scripts/generate-browser-profiles.mjs b/scripts/generate-browser-profiles.mjs new file mode 100644 index 0000000..0b74e68 --- /dev/null +++ b/scripts/generate-browser-profiles.mjs @@ -0,0 +1,75 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { execFile } from 'node:child_process'; +import { dirname, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { platformTargets } from './platform-targets.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(__dirname, '..'); +const outputPath = resolve(repoRoot, 'src/config/generated/browser-profiles.ts'); +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); + +function isMuslRuntime() { + if (process.platform !== 'linux') { + return false; + } + + if (typeof process.report?.getReport === 'function') { + const report = process.report.getReport(); + if (report?.header?.glibcVersionRuntime) { + return false; + } + } + + return true; +} + +function resolveLocalBinary() { + const target = platformTargets.find(({ os, cpu, libc }) => { + if (!os.includes(process.platform) || !cpu.includes(process.arch)) { + return false; + } + + if (!libc) { + return true; + } + + return libc.includes(isMuslRuntime() ? 'musl' : 'glibc'); + }); + + return target ? resolve(repoRoot, 'rust', target.binaryName) : null; +} + +async function loadProfiles() { + const localBinary = resolveLocalBinary(); + if (!localBinary) { + throw new Error(`Unsupported host platform for profile generation: ${process.platform}-${process.arch}`); + } + + const binding = require(localBinary); + return binding.getProfiles(); +} + +try { + const profiles = await loadProfiles(); + const content = `// This file is auto-generated by scripts/generate-browser-profiles.mjs\n// Do not edit it manually.\n\nexport const BROWSER_PROFILES = ${JSON.stringify( + profiles, + null, + 2 + )} as const;\n\nexport type BrowserProfile = (typeof BROWSER_PROFILES)[number];\n`; + + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, content, 'utf8'); + await execFileAsync(resolve(repoRoot, 'node_modules/.bin/oxfmt'), ['--write', outputPath], { + cwd: repoRoot, + }); +} catch (error) { + try { + await readFile(outputPath, 'utf8'); + } catch { + throw error; + } +} diff --git a/scripts/platform-targets.mjs b/scripts/platform-targets.mjs new file mode 100644 index 0000000..30799d1 --- /dev/null +++ b/scripts/platform-targets.mjs @@ -0,0 +1,57 @@ +export const platformTargets = [ + { + target: "x86_64-apple-darwin", + packageName: "@node-wreq/darwin-x64", + binaryName: "node-wreq.darwin-x64.node", + os: ["darwin"], + cpu: ["x64"], + }, + { + target: "aarch64-apple-darwin", + packageName: "@node-wreq/darwin-arm64", + binaryName: "node-wreq.darwin-arm64.node", + os: ["darwin"], + cpu: ["arm64"], + }, + { + target: "x86_64-unknown-linux-gnu", + packageName: "@node-wreq/linux-x64-gnu", + binaryName: "node-wreq.linux-x64-gnu.node", + os: ["linux"], + cpu: ["x64"], + libc: ["glibc"], + }, + { + target: "aarch64-unknown-linux-gnu", + packageName: "@node-wreq/linux-arm64-gnu", + binaryName: "node-wreq.linux-arm64-gnu.node", + os: ["linux"], + cpu: ["arm64"], + libc: ["glibc"], + }, + { + target: "x86_64-unknown-linux-musl", + packageName: "@node-wreq/linux-x64-musl", + binaryName: "node-wreq.linux-x64-musl.node", + os: ["linux"], + cpu: ["x64"], + libc: ["musl"], + }, + { + target: "x86_64-pc-windows-msvc", + packageName: "@node-wreq/win32-x64-msvc", + binaryName: "node-wreq.win32-x64-msvc.node", + os: ["win32"], + cpu: ["x64"], + }, +]; + +export function getPlatformTargetByTriple(target) { + return platformTargets.find((entry) => entry.target === target); +} + +export function getOptionalDependencyMap(version) { + return Object.fromEntries( + platformTargets.map(({ packageName }) => [packageName, version]), + ); +} diff --git a/scripts/postbuild.mjs b/scripts/postbuild.mjs new file mode 100644 index 0000000..f8f094a --- /dev/null +++ b/scripts/postbuild.mjs @@ -0,0 +1,47 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const distDir = resolve(__dirname, "../dist"); + +const runtimeExports = [ + "fetch", + "createClient", + "getProfiles", + "BROWSER_PROFILES", + "Headers", + "Response", + "RequestError", + "HTTPError", + "TimeoutError", + "AbortError", + "WebSocket", + "CloseEvent", + "websocket", + "WebSocketError", +]; + +const esmLines = [ + "import nodeWreq from './node-wreq.js';", + "", + ...runtimeExports.map((name) => `export const ${name} = nodeWreq.${name};`), + "", + "export default nodeWreq;", + "", +]; + +const typeLines = [ + "export * from './node-wreq';", + "import nodeWreq from './node-wreq';", + "export default nodeWreq;", + "", +]; + +await mkdir(distDir, { recursive: true }); +await writeFile(resolve(distDir, "node-wreq.mjs"), esmLines.join("\n"), "utf8"); +await writeFile( + resolve(distDir, "node-wreq.d.mts"), + typeLines.join("\n"), + "utf8", +); diff --git a/scripts/prepare-main-package.mjs b/scripts/prepare-main-package.mjs new file mode 100644 index 0000000..5c6ebd9 --- /dev/null +++ b/scripts/prepare-main-package.mjs @@ -0,0 +1,55 @@ +import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { getOptionalDependencyMap } from "./platform-targets.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(__dirname, ".."); +const outDir = resolve(repoRoot, process.argv[2] ?? ".release/main-package"); + +const rootPackage = JSON.parse( + await readFile(resolve(repoRoot, "package.json"), "utf8"), +); + +await rm(outDir, { recursive: true, force: true }); + +await mkdir(outDir, { recursive: true }); + +await cp(resolve(repoRoot, "dist"), resolve(outDir, "dist"), { + recursive: true, +}); + +await rm(resolve(outDir, "dist/test"), { recursive: true, force: true }); + +await cp(resolve(repoRoot, "docs"), resolve(outDir, "docs"), { + recursive: true, +}); + +await cp(resolve(repoRoot, "README.md"), resolve(outDir, "README.md")); + +const publishPackage = { + name: rootPackage.name, + version: rootPackage.version, + description: rootPackage.description, + main: rootPackage.main, + module: rootPackage.module, + types: rootPackage.types, + exports: rootPackage.exports, + keywords: rootPackage.keywords, + author: rootPackage.author, + license: rootPackage.license, + repository: rootPackage.repository, + bugs: rootPackage.bugs, + homepage: rootPackage.homepage, + engines: rootPackage.engines, + os: rootPackage.os, + cpu: rootPackage.cpu, + optionalDependencies: getOptionalDependencyMap(rootPackage.version), + files: ["dist", "docs", "README.md"], +}; + +await writeFile( + resolve(outDir, "package.json"), + `${JSON.stringify(publishPackage, null, 2)}\n`, + "utf8", +); diff --git a/scripts/prepare-platform-package.mjs b/scripts/prepare-platform-package.mjs new file mode 100644 index 0000000..e961b69 --- /dev/null +++ b/scripts/prepare-platform-package.mjs @@ -0,0 +1,98 @@ +import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { basename, dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { getPlatformTargetByTriple } from "./platform-targets.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(__dirname, ".."); + +function parseArgs(argv) { + const args = {}; + + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[index + 1]; + + if (!key.startsWith("--")) { + continue; + } + + args[key.slice(2)] = value; + index += 1; + } + + return args; +} + +const args = parseArgs(process.argv.slice(2)); +const target = getPlatformTargetByTriple(args.target); + +if (!target) { + throw new Error(`Unknown publish target: ${args.target}`); +} + +if (!args.binary) { + throw new Error("Missing required --binary argument"); +} + +if (!args.outDir) { + throw new Error("Missing required --outDir argument"); +} + +const rootPackage = JSON.parse( + await readFile(resolve(repoRoot, "package.json"), "utf8"), +); +const outDir = resolve(repoRoot, args.outDir); +const binarySource = resolve(repoRoot, args.binary); + +await rm(outDir, { recursive: true, force: true }); +await mkdir(outDir, { recursive: true }); +await cp(binarySource, resolve(outDir, target.binaryName)); + +const packageJson = { + name: target.packageName, + version: rootPackage.version, + description: `Prebuilt native binding for ${rootPackage.name} on ${target.target}`, + license: rootPackage.license, + author: rootPackage.author, + repository: rootPackage.repository, + bugs: rootPackage.bugs, + homepage: rootPackage.homepage, + os: target.os, + cpu: target.cpu, + ...(target.libc ? { libc: target.libc } : {}), + main: `./${target.binaryName}`, + files: [target.binaryName, "README.md"], + publishConfig: { + access: "public", + }, +}; + +const mainPackageUrl = + typeof rootPackage.homepage === "string" + ? rootPackage.homepage.replace(/#readme$/, "") + : `https://www.npmjs.com/package/${rootPackage.name}`; + +const readme = `# ${target.packageName} + +This package only contains the prebuilt native platform binary. + +It is a helper package used by [\`${rootPackage.name}\`](${mainPackageUrl}). + +You should install the main package instead: + +\`\`\`bash +npm install ${rootPackage.name} +\`\`\` + +Target triple: \`${target.target}\` +Binary: \`${basename(target.binaryName)}\` +`; + +await writeFile( + resolve(outDir, "package.json"), + `${JSON.stringify(packageJson, null, 2)}\n`, + "utf8", +); + +await writeFile(resolve(outDir, "README.md"), readme, "utf8"); diff --git a/src/client.ts b/src/client/index.ts similarity index 94% rename from src/client.ts rename to src/client/index.ts index 82f2664..d6bf883 100644 --- a/src/client.ts +++ b/src/client/index.ts @@ -1,6 +1,6 @@ -import { fetch } from './fetch'; -import { Headers } from './headers'; -import { mergeHooks } from './hooks'; +import { Headers } from '../headers'; +import { mergeHooks } from '../hooks'; +import { fetch } from '../http/fetch'; import type { Client, ClientDefaults, @@ -10,8 +10,8 @@ import type { RequestInput, WebSocketInit, WreqInit, -} from './types'; -import { websocket } from './websocket'; +} from '../types'; +import { websocket } from '../websocket'; function mergeHeaders(...sources: Array): HeaderTuple[] | undefined { const merged = new Headers(); @@ -22,12 +22,14 @@ function mergeHeaders(...sources: Array): HeaderTuple[] } const headers = source instanceof Headers ? source : new Headers(source); - for (const [name, value] of headers) { + + for (const [name, value] of headers.toTuples()) { merged.set(name, value); } } const tuples = merged.toTuples(); + return tuples.length > 0 ? tuples : undefined; } @@ -134,6 +136,7 @@ class WreqClient implements Client { hooks: mergeHooks(this.defaults.hooks, init?.hooks), retry: mergeRetry(this.defaults.retry, init?.retry), }; + return fetch(input, merged); } diff --git a/src/config/emulation.ts b/src/config/emulation.ts new file mode 100644 index 0000000..036f306 --- /dev/null +++ b/src/config/emulation.ts @@ -0,0 +1,36 @@ +import type { Http1Options, Http2Options, TlsOptions, WebSocketInit, WreqInit } from '../types'; + +type EmulationInput = Pick & + Pick; + +type SerializedEmulation = { + tlsOptions?: TlsOptions; + http1Options?: Http1Options; + http2Options?: Http2Options; +}; + +function hasAnyEmulationOptions(input: EmulationInput): boolean { + return Boolean(input.tlsOptions || input.http1Options || input.http2Options); +} + +export function serializeEmulationOptions(input: EmulationInput): string | undefined { + if (!hasAnyEmulationOptions(input)) { + return undefined; + } + + const payload: SerializedEmulation = {}; + + if (input.tlsOptions) { + payload.tlsOptions = input.tlsOptions; + } + + if (input.http1Options) { + payload.http1Options = input.http1Options; + } + + if (input.http2Options) { + payload.http2Options = input.http2Options; + } + + return JSON.stringify(payload); +} diff --git a/src/config/generated/browser-profiles.ts b/src/config/generated/browser-profiles.ts new file mode 100644 index 0000000..0cecf9a --- /dev/null +++ b/src/config/generated/browser-profiles.ts @@ -0,0 +1,115 @@ +// This file is auto-generated by scripts/generate-browser-profiles.mjs +// Do not edit it manually. + +export const BROWSER_PROFILES = [ + 'chrome_100', + 'chrome_101', + 'chrome_104', + 'chrome_105', + 'chrome_106', + 'chrome_107', + 'chrome_108', + 'chrome_109', + 'chrome_110', + 'chrome_114', + 'chrome_116', + 'chrome_117', + 'chrome_118', + 'chrome_119', + 'chrome_120', + 'chrome_123', + 'chrome_124', + 'chrome_126', + 'chrome_127', + 'chrome_128', + 'chrome_129', + 'chrome_130', + 'chrome_131', + 'chrome_132', + 'chrome_133', + 'chrome_134', + 'chrome_135', + 'chrome_136', + 'chrome_137', + 'chrome_138', + 'chrome_139', + 'chrome_140', + 'chrome_141', + 'chrome_142', + 'chrome_143', + 'chrome_144', + 'chrome_145', + 'edge_101', + 'edge_122', + 'edge_127', + 'edge_131', + 'edge_134', + 'edge_135', + 'edge_136', + 'edge_137', + 'edge_138', + 'edge_139', + 'edge_140', + 'edge_141', + 'edge_142', + 'edge_143', + 'edge_144', + 'edge_145', + 'opera_116', + 'opera_117', + 'opera_118', + 'opera_119', + 'firefox_109', + 'firefox_117', + 'firefox_128', + 'firefox_133', + 'firefox_135', + 'firefox_private_135', + 'firefox_android_135', + 'firefox_136', + 'firefox_private_136', + 'firefox_139', + 'firefox_142', + 'firefox_143', + 'firefox_144', + 'firefox_145', + 'firefox_146', + 'firefox_147', + 'safari_ios_17_2', + 'safari_ios_17_4_1', + 'safari_ios_16_5', + 'safari_15_3', + 'safari_15_5', + 'safari_15_6_1', + 'safari_16', + 'safari_16_5', + 'safari_17_0', + 'safari_17_2_1', + 'safari_17_4_1', + 'safari_17_5', + 'safari_17_6', + 'safari_18', + 'safari_ipad_18', + 'safari_18_2', + 'safari_ios_18_1_1', + 'safari_18_3', + 'safari_18_3_1', + 'safari_18_5', + 'safari_26', + 'safari_26_1', + 'safari_26_2', + 'safari_ipad_26', + 'safari_ipad_26_2', + 'safari_ios_26', + 'safari_ios_26_2', + 'okhttp_3_9', + 'okhttp_3_11', + 'okhttp_3_13', + 'okhttp_3_14', + 'okhttp_4_9', + 'okhttp_4_10', + 'okhttp_4_12', + 'okhttp_5', +] as const; + +export type BrowserProfile = (typeof BROWSER_PROFILES)[number]; diff --git a/src/errors.ts b/src/errors/index.ts similarity index 90% rename from src/errors.ts rename to src/errors/index.ts index 597fe7b..a88fc68 100644 --- a/src/errors.ts +++ b/src/errors/index.ts @@ -1,10 +1,10 @@ -import type { Response } from './response'; -import type { NormalizedRequest } from './types'; +import type { Request } from '../http/request'; +import type { Response } from '../http/response'; type RequestErrorOptions = { code?: string; cause?: unknown; - request?: NormalizedRequest; + request?: Request; response?: Response; attempt?: number; }; @@ -12,7 +12,7 @@ type RequestErrorOptions = { export class RequestError extends Error { code?: string; cause?: unknown; - request?: NormalizedRequest; + request?: Request; response?: Response; attempt?: number; diff --git a/src/fetch.ts b/src/fetch.ts deleted file mode 100644 index 83fd7ee..0000000 --- a/src/fetch.ts +++ /dev/null @@ -1,724 +0,0 @@ -import { Buffer } from 'node:buffer'; -import { AbortError, HTTPError, RequestError, TimeoutError } from './errors'; -import { Headers } from './headers'; -import { - runAfterResponseHooks, - runBeforeErrorHooks, - runBeforeRedirectHooks, - runBeforeRequestHooks, - runBeforeRetryHooks, - runInitHooks, -} from './hooks'; -import { nativeRequest, normalizeMethod, validateBrowserProfile } from './native'; -import { Response } from './response'; -import type { - BodyInit, - CookieJar, - HeadersInit, - HttpMethod, - NativeRequestOptions, - NormalizedRequest, - RedirectEntry, - RequestInput, - RequestLike, - RequestStats, - ResolvedOptions, - ResolvedRetryOptions, - RetryDecisionContext, - WreqInit, -} from './types'; - -const DEFAULT_RETRY_METHODS = ['GET', 'HEAD'] as const; -const DEFAULT_RETRY_STATUS_CODES = [408, 425, 429, 500, 502, 503, 504]; -const DEFAULT_RETRY_ERROR_CODES = [ - 'ECONNABORTED', - 'ECONNRESET', - 'ECONNREFUSED', - 'EPIPE', - 'ETIMEDOUT', - 'ERR_ABORTED', - 'ERR_TIMEOUT', -]; -const REDIRECT_STATUS_CODES = new Set([300, 301, 302, 303, 307, 308]); - -function isRequestLike(value: unknown): value is RequestLike { - return typeof value === 'object' && value !== null && 'url' in value; -} - -function isResponseStatusAllowed( - status: number, - options: Pick -): boolean { - if (options.validateStatus) { - return options.validateStatus(status); - } - - if (options.throwHttpErrors) { - return status >= 200 && status < 300; - } - - return true; -} - -function appendQuery(url: URL, query: WreqInit['query']): void { - if (!query) { - return; - } - - for (const [key, value] of Object.entries(query)) { - if (value === undefined || value === null) { - continue; - } - - url.searchParams.set(key, String(value)); - } -} - -function resolveUrl(rawUrl: string | URL, baseURL?: string, query?: WreqInit['query']): string { - const url = baseURL ? new URL(String(rawUrl), baseURL) : new URL(String(rawUrl)); - appendQuery(url, query); - return url.toString(); -} - -async function normalizeBody(body: BodyInit | null | undefined): Promise { - if (body === undefined || body === null) { - return undefined; - } - - if (typeof body === 'string') { - return body; - } - - if (body instanceof URLSearchParams) { - return body.toString(); - } - - if (Buffer.isBuffer(body)) { - return body.toString('utf8'); - } - - if (ArrayBuffer.isView(body)) { - return Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString('utf8'); - } - - if (body instanceof ArrayBuffer) { - return Buffer.from(body).toString('utf8'); - } - - throw new RequestError('Unsupported body type'); -} - -async function mergeInputAndInit( - input: RequestInput, - init?: WreqInit -): Promise<{ - urlInput: string | URL; - init: WreqInit; -}> { - let urlInput: string | URL; - let mergedInit: WreqInit = { ...init }; - - if (typeof input === 'string' || input instanceof URL) { - urlInput = input; - } else if (isRequestLike(input)) { - urlInput = input.url; - const inputHeaders = input.headers as HeadersInit | undefined; - const inputBody = input.body as WreqInit['body']; - mergedInit = { - ...init, - method: init?.method ?? input.method, - headers: init?.headers ?? inputHeaders, - body: init?.body ?? inputBody, - signal: init?.signal ?? input.signal ?? undefined, - }; - - if (mergedInit.body === undefined && typeof input.arrayBuffer === 'function') { - if (input.bodyUsed) { - throw new TypeError('Request body is already used'); - } - - const arrayBuffer = await input.arrayBuffer(); - mergedInit.body = arrayBuffer; - } - } else { - throw new RequestError('URL is required'); - } - - return { urlInput, init: mergedInit }; -} - -function resolveRetryOptions(retry?: WreqInit['retry']): ResolvedRetryOptions { - const configured = typeof retry === 'number' ? { limit: retry } : retry; - - return { - limit: configured?.limit ?? 0, - methods: configured?.methods ?? [...DEFAULT_RETRY_METHODS], - statusCodes: configured?.statusCodes ?? [...DEFAULT_RETRY_STATUS_CODES], - errorCodes: configured?.errorCodes ?? [...DEFAULT_RETRY_ERROR_CODES], - backoff: configured?.backoff, - shouldRetry: configured?.shouldRetry, - }; -} - -function resolveOptions(init: WreqInit): ResolvedOptions { - return { - ...init, - headers: new Headers(init.headers), - retry: resolveRetryOptions(init.retry), - redirect: init.redirect ?? 'follow', - maxRedirects: init.maxRedirects ?? 20, - throwHttpErrors: init.throwHttpErrors ?? false, - disableDefaultHeaders: init.disableDefaultHeaders ?? false, - compress: init.compress ?? true, - }; -} - -function createRequest(urlInput: string | URL, options: ResolvedOptions): NormalizedRequest { - validateBrowserProfile(options.browser); - - return { - url: resolveUrl(urlInput, options.baseURL, options.query), - method: normalizeMethod(options.method), - headers: options.headers, - body: options.body, - }; -} - -function buildNativeRequest( - request: NormalizedRequest, - options: ResolvedOptions, - body: string | undefined -): NativeRequestOptions { - return { - url: request.url, - method: request.method, - headers: request.headers.toObject(), - body, - browser: options.browser, - proxy: options.proxy, - timeout: options.timeout, - }; -} - -function isError(value: unknown): value is Error { - return value instanceof Error; -} - -function isRequestError(value: unknown): value is RequestError { - return value instanceof RequestError; -} - -function normalizeRequestError( - error: unknown, - request: NormalizedRequest, - attempt: number, - response?: Response -): RequestError { - if (error instanceof TimeoutError || error instanceof AbortError || error instanceof HTTPError) { - error.request ??= request; - error.response ??= response; - error.attempt ??= attempt; - return error; - } - - if (isRequestError(error)) { - error.request ??= request; - error.response ??= response; - error.attempt ??= attempt; - return error; - } - - if (isError(error)) { - return new RequestError(error.message, { - cause: error, - request, - response, - attempt, - }); - } - - return new RequestError(String(error), { - cause: error, - request, - response, - attempt, - }); -} - -function inferErrorCode(error: unknown): string | undefined { - if (error instanceof RequestError && error.code) { - return error.code; - } - - if (typeof error === 'object' && error !== null && 'code' in error) { - const code = (error as { code?: unknown }).code; - return typeof code === 'string' ? code : undefined; - } - - return undefined; -} - -function throwIfAborted(signal?: AbortSignal | null): void { - if (!signal?.aborted) { - return; - } - - throw new AbortError(undefined, { cause: signal.reason }); -} - -async function sleep(delayMs: number): Promise { - if (delayMs <= 0) { - return; - } - - await new Promise((resolve) => { - setTimeout(resolve, delayMs); - }); -} - -async function loadCookiesIntoRequest( - cookieJar: CookieJar | undefined, - request: NormalizedRequest -): Promise { - if (!cookieJar || request.headers.has('cookie')) { - return; - } - - const cookies = await cookieJar.getCookies(request.url); - if (cookies.length === 0) { - return; - } - - request.headers.set( - 'cookie', - cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ') - ); -} - -async function persistResponseCookies( - cookieJar: CookieJar | undefined, - requestUrl: string, - response: Response -): Promise { - if (!cookieJar) { - return; - } - - if (response.setCookies.length > 0) { - for (const cookie of response.setCookies) { - await cookieJar.setCookie(cookie, requestUrl); - } - - return; - } - - for (const [name, value] of Object.entries(response.cookies)) { - await cookieJar.setCookie(`${name}=${value}`, requestUrl); - } -} - -async function reportStats( - callback: WreqInit['onStats'] | undefined, - stats: RequestStats -): Promise { - if (!callback) { - return; - } - - await callback(stats); -} - -async function dispatchNativeRequest( - options: NativeRequestOptions, - startTime: number -): Promise { - const nativeResponse = await nativeRequest(options).catch((error: unknown) => { - const message = String(error); - const lowered = message.toLowerCase(); - - if (lowered.includes('timed out') || lowered.includes('timeout')) { - throw new TimeoutError(message, { cause: error }); - } - - throw new RequestError(message, { cause: error }); - }); - - const responseStart = Date.now(); - - return new Response({ - ...nativeResponse, - timings: { - startTime, - responseStart, - wait: responseStart - startTime, - }, - }); -} - -function isRedirectResponse(response: Response): boolean { - return REDIRECT_STATUS_CODES.has(response.status) && response.headers.has('location'); -} - -function resolveRedirectLocation(response: Response, requestUrl: string): string { - const location = response.headers.get('location'); - if (!location) { - throw new RequestError('Redirect response is missing Location header'); - } - - try { - return new URL(location, requestUrl).toString(); - } catch (error) { - throw new RequestError(`Invalid redirect URL: ${location}`, { cause: error }); - } -} - -function stripRedirectSensitiveHeaders( - headers: Headers, - previousUrl: string, - nextUrl: string, - bodyDropped: boolean -): void { - headers.delete('host'); - headers.delete('content-length'); - - if (bodyDropped) { - headers.delete('content-type'); - } - - if (new URL(previousUrl).origin !== new URL(nextUrl).origin) { - headers.delete('authorization'); - } -} - -function rewriteRedirectMethodAndBody( - method: HttpMethod, - status: number, - body: NormalizedRequest['body'] -): { - method: HttpMethod; - body: NormalizedRequest['body']; - bodyDropped: boolean; -} { - if (status === 303) { - return { - method: method === 'HEAD' ? 'HEAD' : 'GET', - body: undefined, - bodyDropped: true, - }; - } - - if ((status === 301 || status === 302) && method === 'POST') { - return { - method: 'GET', - body: undefined, - bodyDropped: true, - }; - } - - return { - method, - body, - bodyDropped: false, - }; -} - -async function shouldRetryRequest( - context: RetryDecisionContext, - retry: ResolvedRetryOptions -): Promise { - if (context.attempt > retry.limit + 1) { - return false; - } - - if (!retry.methods.includes(context.request.method)) { - return false; - } - - if (context.response) { - if (!retry.statusCodes.includes(context.response.status)) { - return false; - } - } else { - const code = inferErrorCode(context.error); - if (!code || !retry.errorCodes.includes(code)) { - return false; - } - } - - if (!retry.shouldRetry) { - return true; - } - - return retry.shouldRetry(context); -} - -async function runRetryDelay( - context: RetryDecisionContext, - retry: ResolvedRetryOptions -): Promise { - if (!retry.backoff) { - return; - } - - const delay = await retry.backoff(context); - await sleep(delay); -} - -function finalizeResponse(response: Response, redirectChain: RedirectEntry[]): Response { - return response.setRedirectMetadata(redirectChain); -} - -export async function fetch(input: RequestInput, init?: WreqInit): Promise { - const merged = await mergeInputAndInit(input, init); - const state = (merged.init.context ? { ...merged.init.context } : {}) as Record; - - await runInitHooks(merged.init.hooks, { - input, - options: merged.init, - state, - }); - - const options = resolveOptions(merged.init); - const request = createRequest(merged.urlInput, options); - const redirectChain: RedirectEntry[] = []; - const visitedRedirectTargets = new Set([request.url]); - - let attempt = 1; - - while (true) { - const requestBody = await normalizeBody(request.body); - const startTime = Date.now(); - - try { - throwIfAborted(options.signal); - await loadCookiesIntoRequest(options.cookieJar, request); - - const shortCircuit = await runBeforeRequestHooks(options.hooks, { - request, - options, - attempt, - state, - }); - - let response = - shortCircuit ?? - (await dispatchNativeRequest(buildNativeRequest(request, options, requestBody), startTime)); - - if (shortCircuit) { - response.setTimings({ - startTime, - responseStart: startTime, - wait: 0, - endTime: startTime, - total: 0, - }); - } - - response = await runAfterResponseHooks(options.hooks, { - request, - options, - attempt, - state, - response, - }); - - await reportStats(options.onStats, { - request, - attempt, - timings: response.timings ?? { - startTime, - responseStart: startTime, - wait: 0, - }, - response, - }); - - await persistResponseCookies(options.cookieJar, request.url, response); - - if (isRedirectResponse(response)) { - if (options.redirect === 'manual') { - return finalizeResponse(response, redirectChain); - } - - if (options.redirect === 'error') { - throw new RequestError(`Redirect encountered for ${request.url}`, { - code: 'ERR_REDIRECT', - request, - response, - attempt, - }); - } - - if (redirectChain.length >= options.maxRedirects) { - throw new RequestError(`Maximum redirects exceeded: ${options.maxRedirects}`, { - code: 'ERR_TOO_MANY_REDIRECTS', - request, - response, - attempt, - }); - } - - const nextUrl = resolveRedirectLocation(response, request.url); - if (visitedRedirectTargets.has(nextUrl)) { - throw new RequestError(`Redirect loop detected for ${nextUrl}`, { - code: 'ERR_REDIRECT_LOOP', - request, - response, - attempt, - }); - } - - const rewritten = rewriteRedirectMethodAndBody( - request.method, - response.status, - request.body - ); - const nextRequest: NormalizedRequest = { - url: nextUrl, - method: rewritten.method, - headers: request.headers, - body: rewritten.body, - }; - - stripRedirectSensitiveHeaders( - nextRequest.headers, - request.url, - nextUrl, - rewritten.bodyDropped - ); - if (options.cookieJar) { - nextRequest.headers.delete('cookie'); - } - - const redirectEntry: RedirectEntry = { - fromUrl: request.url, - status: response.status, - location: response.headers.get('location') ?? nextUrl, - toUrl: nextUrl, - method: nextRequest.method, - }; - - await runBeforeRedirectHooks(options.hooks, { - request: nextRequest, - options, - attempt, - state, - response, - redirectCount: redirectChain.length + 1, - nextUrl, - nextMethod: nextRequest.method, - redirectChain: [...redirectChain, redirectEntry], - }); - - redirectChain.push({ - ...redirectEntry, - toUrl: nextRequest.url, - method: nextRequest.method, - }); - visitedRedirectTargets.add(nextRequest.url); - - request.url = nextRequest.url; - request.method = nextRequest.method; - request.body = nextRequest.body; - - continue; - } - - const nextAttempt = attempt + 1; - const retryContext: RetryDecisionContext = { - request, - options, - attempt: nextAttempt, - state, - response, - }; - - if (await shouldRetryRequest(retryContext, options.retry)) { - const retryError = new HTTPError( - `Request failed with status ${response.status}`, - response.status, - { - request, - response, - attempt, - } - ); - - await runBeforeRetryHooks(options.hooks, { - request, - options, - attempt: nextAttempt, - state, - error: retryError, - response, - }); - - await runRetryDelay({ ...retryContext, error: retryError }, options.retry); - attempt = nextAttempt; - continue; - } - - if (!isResponseStatusAllowed(response.status, options)) { - throw new HTTPError(`Request failed with status ${response.status}`, response.status, { - request, - response, - attempt, - }); - } - - return finalizeResponse(response, redirectChain); - } catch (error: unknown) { - const normalizedError = normalizeRequestError(error, request, attempt); - const errorEndTime = Date.now(); - - await reportStats(options.onStats, { - request, - attempt, - timings: { - startTime, - responseStart: errorEndTime, - wait: errorEndTime - startTime, - endTime: errorEndTime, - total: errorEndTime - startTime, - }, - error: normalizedError, - response: normalizedError.response, - }); - - const nextAttempt = attempt + 1; - const retryContext: RetryDecisionContext = { - request, - options, - attempt: nextAttempt, - state, - error: normalizedError, - response: normalizedError.response, - }; - - if (await shouldRetryRequest(retryContext, options.retry)) { - await runBeforeRetryHooks(options.hooks, { - request, - options, - attempt: nextAttempt, - state, - error: normalizedError, - response: normalizedError.response, - }); - - await runRetryDelay(retryContext, options.retry); - attempt = nextAttempt; - continue; - } - - const finalError = await runBeforeErrorHooks(options.hooks, { - request, - options, - attempt, - state, - error: normalizedError, - }); - - throw finalError; - } - } -} diff --git a/src/generated/browser-profiles.ts b/src/generated/browser-profiles.ts new file mode 100644 index 0000000..fadd270 --- /dev/null +++ b/src/generated/browser-profiles.ts @@ -0,0 +1,90 @@ +// This file is auto-generated by scripts/generate-browser-profiles.mjs +// Do not edit it manually. + +export const BROWSER_PROFILES = [ + 'chrome_100', + 'chrome_101', + 'chrome_104', + 'chrome_105', + 'chrome_106', + 'chrome_107', + 'chrome_108', + 'chrome_109', + 'chrome_110', + 'chrome_114', + 'chrome_116', + 'chrome_117', + 'chrome_118', + 'chrome_119', + 'chrome_120', + 'chrome_123', + 'chrome_124', + 'chrome_126', + 'chrome_127', + 'chrome_128', + 'chrome_129', + 'chrome_130', + 'chrome_131', + 'chrome_132', + 'chrome_133', + 'chrome_134', + 'chrome_135', + 'chrome_136', + 'chrome_137', + 'chrome_138', + 'chrome_139', + 'chrome_140', + 'edge_101', + 'edge_122', + 'edge_127', + 'edge_131', + 'edge_134', + 'opera_116', + 'opera_117', + 'opera_118', + 'opera_119', + 'safari_ios_17_2', + 'safari_ios_17_4_1', + 'safari_ios_16_5', + 'safari_15_3', + 'safari_15_5', + 'safari_15_6_1', + 'safari_16', + 'safari_16_5', + 'safari_17_0', + 'safari_17_2_1', + 'safari_17_4_1', + 'safari_17_5', + 'safari_18', + 'safari_ipad_18', + 'safari_18_2', + 'safari_ios_18_1_1', + 'safari_18_3', + 'safari_18_3_1', + 'safari_18_5', + 'safari_26', + 'safari_ipad_26', + 'safari_ios_26', + 'firefox_109', + 'firefox_117', + 'firefox_128', + 'firefox_133', + 'firefox_135', + 'firefox_private_135', + 'firefox_android_135', + 'firefox_136', + 'firefox_private_136', + 'firefox_139', + 'firefox_142', + 'firefox_143', + 'okhttp_3_9', + 'okhttp_3_11', + 'okhttp_3_13', + 'okhttp_3_14', + 'okhttp_4_9', + 'okhttp_4_10', + 'okhttp_4_12', + 'okhttp_5', +] as const; + +export type BrowserProfile = (typeof BROWSER_PROFILES)[number]; diff --git a/src/headers.ts b/src/headers/index.ts similarity index 74% rename from src/headers.ts rename to src/headers/index.ts index 073c8f6..cbba659 100644 --- a/src/headers.ts +++ b/src/headers/index.ts @@ -1,4 +1,4 @@ -import type { HeaderTuple, HeadersInit } from './types'; +import type { HeaderTuple, HeadersInit } from '../types'; type HeaderEntry = { name: string; @@ -21,6 +21,7 @@ function isPlainObject(value: unknown): value is Record { export class Headers implements Iterable { private readonly store = new Map(); + private entriesList: HeaderTuple[] = []; constructor(init?: HeadersInit) { if (!init) { @@ -28,7 +29,7 @@ export class Headers implements Iterable { } if (init instanceof Headers) { - for (const [name, value] of init) { + for (const [name, value] of init.toTuples()) { this.append(name, value); } @@ -73,6 +74,8 @@ export class Headers implements Iterable { const entry = this.store.get(normalized.key); const stringValue = String(value); + this.entriesList.push([normalized.display, stringValue]); + if (entry) { entry.values.push(stringValue); @@ -87,11 +90,17 @@ export class Headers implements Iterable { set(name: string, value: unknown): void { const normalized = this.normalizeName(name); + const stringValue = String(value); + + this.entriesList = this.entriesList.filter( + ([entryName]) => entryName.trim().toLowerCase() !== normalized.key + ); this.store.set(normalized.key, { name: normalized.display, - values: [String(value)], + values: [stringValue], }); + this.entriesList.push([normalized.display, stringValue]); } get(name: string): string | null { @@ -111,6 +120,9 @@ export class Headers implements Iterable { const normalized = this.normalizeName(name); this.store.delete(normalized.key); + this.entriesList = this.entriesList.filter( + ([entryName]) => entryName.trim().toLowerCase() !== normalized.key + ); } toObject(): Record { @@ -124,7 +136,25 @@ export class Headers implements Iterable { } toTuples(): HeaderTuple[] { - return [...this]; + return this.entriesList.map(([name, value]) => [name, value]); + } + + toOriginalNames(): string[] { + const names: string[] = []; + const seen = new Set(); + + for (const [name] of this.entriesList) { + const normalized = name.trim().toLowerCase(); + + if (seen.has(normalized)) { + continue; + } + + seen.add(normalized); + names.push(name); + } + + return names; } *entries(): IterableIterator { diff --git a/src/hooks.ts b/src/hooks/index.ts similarity index 97% rename from src/hooks.ts rename to src/hooks/index.ts index 4d70fd8..3b24c42 100644 --- a/src/hooks.ts +++ b/src/hooks/index.ts @@ -1,4 +1,4 @@ -import type { Response } from './response'; +import type { Response } from '../http/response'; import type { AfterResponseContext, BeforeErrorContext, @@ -7,7 +7,7 @@ import type { BeforeRequestContext, Hooks, InitContext, -} from './types'; +} from '../types'; export function mergeHooks(base?: Hooks, override?: Hooks): Hooks | undefined { if (!base && !override) { @@ -70,6 +70,7 @@ export async function runBeforeErrorHooks( for (const hook of hooks?.beforeError ?? []) { const result = await hook({ ...context, error: current }); + if (result) { current = result; } diff --git a/src/http/body/bytes.ts b/src/http/body/bytes.ts new file mode 100644 index 0000000..621d3ff --- /dev/null +++ b/src/http/body/bytes.ts @@ -0,0 +1,37 @@ +import { Buffer } from 'node:buffer'; +import type { BodyInit } from '../../types'; + +export function toBodyBytes( + body: BodyInit | null | undefined, + errorMessage = 'Unsupported body type' +): Uint8Array | null { + if (body === undefined || body === null) { + return null; + } + + if (typeof body === 'string') { + return Buffer.from(body, 'utf8'); + } + + if (body instanceof URLSearchParams) { + return Buffer.from(body.toString(), 'utf8'); + } + + if (Buffer.isBuffer(body)) { + return new Uint8Array(body); + } + + if (ArrayBuffer.isView(body)) { + return new Uint8Array(body.buffer, body.byteOffset, body.byteLength); + } + + if (body instanceof ArrayBuffer) { + return new Uint8Array(body); + } + + throw new TypeError(errorMessage); +} + +export function cloneBytes(bytes: Uint8Array | null): Uint8Array | null { + return bytes ? new Uint8Array(bytes) : null; +} diff --git a/src/http/body/form-data.ts b/src/http/body/form-data.ts new file mode 100644 index 0000000..e6eb3a7 --- /dev/null +++ b/src/http/body/form-data.ts @@ -0,0 +1,79 @@ +function getMediaType(contentType: string): string { + const [mediaType = ''] = contentType.split(';', 1); + + return mediaType.trim().toLowerCase(); +} + +function getMultipartBoundary(contentType: string): string | null { + for (const parameter of contentType.split(';').slice(1)) { + const separatorIndex = parameter.indexOf('='); + + if (separatorIndex === -1) { + continue; + } + + const name = parameter.slice(0, separatorIndex).trim().toLowerCase(); + + if (name !== 'boundary') { + continue; + } + + let value = parameter.slice(separatorIndex + 1).trim(); + + if (value.startsWith('"') && value.endsWith('"')) { + value = value.slice(1, -1); + } + + return value.length > 0 ? value : null; + } + + return null; +} + +export async function parseResponseFormData( + bytes: Uint8Array, + contentType: string +): Promise { + const mediaType = getMediaType(contentType); + + if (mediaType === 'application/x-www-form-urlencoded') { + const formData = new FormData(); + const searchParams = new URLSearchParams(Buffer.from(bytes).toString('utf8')); + + for (const [name, value] of searchParams) { + formData.append(name, value); + } + + return formData; + } + + if (mediaType !== 'multipart/form-data') { + throw new TypeError(`Response content-type is not form data: ${contentType || 'unknown'}`); + } + + if (!getMultipartBoundary(contentType)) { + throw new TypeError('Missing or invalid multipart/form-data boundary in Content-Type header'); + } + + const NativeResponse = globalThis.Response; + + if (typeof NativeResponse !== 'function') { + throw new TypeError('multipart/form-data parsing requires global Response support'); + } + + try { + return await new NativeResponse(bytes, { + headers: { 'content-type': contentType }, + }).formData(); + } catch (error) { + const wrapped = new TypeError('Failed to parse multipart/form-data response body'); + + Object.defineProperty(wrapped, 'cause', { + value: error, + configurable: true, + enumerable: false, + writable: true, + }); + throw wrapped; + } +} diff --git a/src/http/fetch.ts b/src/http/fetch.ts new file mode 100644 index 0000000..d43bc07 --- /dev/null +++ b/src/http/fetch.ts @@ -0,0 +1,276 @@ +import { HTTPError, RequestError } from '../errors'; +import { + runAfterResponseHooks, + runBeforeErrorHooks, + runBeforeRedirectHooks, + runBeforeRequestHooks, + runBeforeRetryHooks, + runInitHooks, +} from '../hooks'; +import { normalizeMethod } from '../native'; +import type { RedirectEntry, RequestInput, RetryDecisionContext, WreqInit } from '../types'; +import { loadCookiesIntoRequest, persistResponseCookies } from './pipeline/cookies'; +import { dispatchNativeRequest, reportStats } from './pipeline/dispatch'; +import { isResponseStatusAllowed, normalizeRequestError, throwIfAborted } from './pipeline/errors'; +import { mergeInputAndInit } from './pipeline/input'; +import { buildNativeRequest, createRequest, resolveOptions } from './pipeline/options'; +import { + finalizeResponse, + isRedirectResponse, + resolveRedirectLocation, + rewriteRedirectMethodAndBody, + stripRedirectSensitiveHeaders, + toRedirectEntry, +} from './pipeline/redirects'; +import { runRetryDelay, shouldRetryRequest } from './pipeline/retries'; + +export async function fetch(input: RequestInput, init?: WreqInit) { + const merged = await mergeInputAndInit(input, init); + const state = (merged.init.context ? { ...merged.init.context } : {}) as Record; + + await runInitHooks(merged.init.hooks, { + input, + options: merged.init, + state, + }); + + const options = resolveOptions(merged.init); + let request = createRequest(merged.urlInput, options); + const redirectChain: RedirectEntry[] = []; + const visitedRedirectTargets = new Set([request.url]); + let attempt = 1; + + while (true) { + const startTime = Date.now(); + + try { + throwIfAborted(options.signal); + + if (options.cookieJar) { + request.headers.delete('cookie'); + } + + await loadCookiesIntoRequest(options.cookieJar, request); + + const shortCircuit = await runBeforeRequestHooks(options.hooks, { + request, + options, + attempt, + state, + }); + + let response = + shortCircuit ?? + (await dispatchNativeRequest(buildNativeRequest(request, options), startTime)); + + if (shortCircuit) { + response.setTimings({ + startTime, + responseStart: startTime, + wait: 0, + endTime: startTime, + total: 0, + }); + } + + response = await runAfterResponseHooks(options.hooks, { + request, + options, + attempt, + state, + response, + }); + + await reportStats(options.onStats, { + request, + attempt, + timings: response.wreq.timings ?? { + startTime, + responseStart: startTime, + wait: 0, + }, + response, + }); + + await persistResponseCookies(options.cookieJar, request.url, response); + + if (isRedirectResponse(response)) { + if (options.redirect === 'manual') { + return finalizeResponse(response, redirectChain); + } + + if (options.redirect === 'error') { + throw new RequestError(`Redirect encountered for ${request.url}`, { + code: 'ERR_REDIRECT', + request, + response, + attempt, + }); + } + + if (redirectChain.length >= options.maxRedirects) { + throw new RequestError(`Maximum redirects exceeded: ${options.maxRedirects}`, { + code: 'ERR_TOO_MANY_REDIRECTS', + request, + response, + attempt, + }); + } + + const nextUrl = resolveRedirectLocation(response, request.url); + + if (visitedRedirectTargets.has(nextUrl)) { + throw new RequestError(`Redirect loop detected for ${nextUrl}`, { + code: 'ERR_REDIRECT_LOOP', + request, + response, + attempt, + }); + } + + const rewritten = rewriteRedirectMethodAndBody( + normalizeMethod(request.method), + response.status, + request._cloneBodyBytes() ?? undefined + ); + + const nextRequest = request._replace({ + url: nextUrl, + method: rewritten.method, + body: rewritten.body, + }); + + stripRedirectSensitiveHeaders( + nextRequest.headers, + request.url, + nextUrl, + rewritten.bodyDropped + ); + + if (options.cookieJar) { + nextRequest.headers.delete('cookie'); + } + + const redirectEntry = toRedirectEntry(request.url, response, nextUrl, nextRequest.method); + + await runBeforeRedirectHooks(options.hooks, { + request: nextRequest, + options, + attempt, + state, + response, + redirectCount: redirectChain.length + 1, + nextUrl, + nextMethod: normalizeMethod(nextRequest.method), + redirectChain: [...redirectChain, redirectEntry], + }); + + redirectChain.push({ + ...redirectEntry, + toUrl: nextRequest.url, + method: normalizeMethod(nextRequest.method), + }); + + visitedRedirectTargets.add(nextRequest.url); + request = nextRequest; + continue; + } + + const nextAttempt = attempt + 1; + const retryContext: RetryDecisionContext = { + request, + options, + attempt: nextAttempt, + state, + response, + }; + + if (await shouldRetryRequest(retryContext, options.retry)) { + const retryError = new HTTPError( + `Request failed with status ${response.status}`, + response.status, + { + request, + response, + attempt, + } + ); + + await runBeforeRetryHooks(options.hooks, { + request, + options, + attempt: nextAttempt, + state, + error: retryError, + response, + }); + + await runRetryDelay({ ...retryContext, error: retryError }, options.retry); + attempt = nextAttempt; + continue; + } + + if (!isResponseStatusAllowed(response.status, options)) { + throw new HTTPError(`Request failed with status ${response.status}`, response.status, { + request, + response, + attempt, + }); + } + + return finalizeResponse(response, redirectChain); + } catch (error: unknown) { + const normalizedError = normalizeRequestError(error, request, attempt); + const errorEndTime = Date.now(); + + await reportStats(options.onStats, { + request, + attempt, + timings: { + startTime, + responseStart: errorEndTime, + wait: errorEndTime - startTime, + endTime: errorEndTime, + total: errorEndTime - startTime, + }, + error: normalizedError, + response: normalizedError.response, + }); + + const nextAttempt = attempt + 1; + const retryContext: RetryDecisionContext = { + request, + options, + attempt: nextAttempt, + state, + error: normalizedError, + response: normalizedError.response, + }; + + if (await shouldRetryRequest(retryContext, options.retry)) { + await runBeforeRetryHooks(options.hooks, { + request, + options, + attempt: nextAttempt, + state, + error: normalizedError, + response: normalizedError.response, + }); + + await runRetryDelay(retryContext, options.retry); + attempt = nextAttempt; + continue; + } + + const finalError = await runBeforeErrorHooks(options.hooks, { + request, + options, + attempt, + state, + error: normalizedError, + }); + + throw finalError; + } + } +} diff --git a/src/http/pipeline/cookies.ts b/src/http/pipeline/cookies.ts new file mode 100644 index 0000000..0f99fb7 --- /dev/null +++ b/src/http/pipeline/cookies.ts @@ -0,0 +1,64 @@ +import { Headers } from '../../headers'; +import type { CookieJar } from '../../types'; +import { Request } from '../request'; +import { Response } from '../response'; + +export async function loadCookiesIntoRequest( + cookieJar: CookieJar | undefined, + request: Request +): Promise { + if (!cookieJar || request.headers.has('cookie')) { + return; + } + + const cookies = await cookieJar.getCookies(request.url); + + if (cookies.length === 0) { + return; + } + + request.headers.set( + 'cookie', + cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ') + ); +} + +export async function loadCookiesIntoHeaders( + cookieJar: CookieJar | undefined, + url: string, + headers: Headers +): Promise { + if (!cookieJar || headers.has('cookie')) { + return; + } + + const cookies = await cookieJar.getCookies(url); + + if (cookies.length === 0) { + return; + } + + headers.set('cookie', cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ')); +} + +export async function persistResponseCookies( + cookieJar: CookieJar | undefined, + requestUrl: string, + response: Response +): Promise { + if (!cookieJar) { + return; + } + + if (response.wreq.setCookies.length > 0) { + for (const cookie of response.wreq.setCookies) { + await cookieJar.setCookie(cookie, requestUrl); + } + + return; + } + + for (const [name, value] of Object.entries(response.wreq.cookies)) { + await cookieJar.setCookie(`${name}=${value}`, requestUrl); + } +} diff --git a/src/http/pipeline/dispatch.ts b/src/http/pipeline/dispatch.ts new file mode 100644 index 0000000..8945989 --- /dev/null +++ b/src/http/pipeline/dispatch.ts @@ -0,0 +1,42 @@ +import { RequestError, TimeoutError } from '../../errors'; +import { nativeRequest } from '../../native'; +import type { NativeRequestOptions, RequestStats, WreqInit } from '../../types'; +import { Response } from '../response'; + +export async function reportStats( + callback: WreqInit['onStats'] | undefined, + stats: RequestStats +): Promise { + if (!callback) { + return; + } + + await callback(stats); +} + +export async function dispatchNativeRequest( + options: NativeRequestOptions, + startTime: number +): Promise { + const nativeResponse = await nativeRequest(options).catch((error: unknown) => { + const message = String(error); + const lowered = message.toLowerCase(); + + if (lowered.includes('timed out') || lowered.includes('timeout')) { + throw new TimeoutError(message, { cause: error }); + } + + throw new RequestError(message, { cause: error }); + }); + + const responseStart = Date.now(); + + return new Response({ + ...nativeResponse, + timings: { + startTime, + responseStart, + wait: responseStart - startTime, + }, + }); +} diff --git a/src/http/pipeline/errors.ts b/src/http/pipeline/errors.ts new file mode 100644 index 0000000..38ff599 --- /dev/null +++ b/src/http/pipeline/errors.ts @@ -0,0 +1,88 @@ +import { AbortError, HTTPError, RequestError, TimeoutError } from '../../errors'; +import type { ResolvedOptions } from '../../types'; +import { Request } from '../request'; +import { Response } from '../response'; + +export function isResponseStatusAllowed( + status: number, + options: Pick +): boolean { + if (options.validateStatus) { + return options.validateStatus(status); + } + + if (options.throwHttpErrors) { + return status >= 200 && status < 300; + } + + return true; +} + +function isError(value: unknown): value is Error { + return value instanceof Error; +} + +function isRequestError(value: unknown): value is RequestError { + return value instanceof RequestError; +} + +export function normalizeRequestError( + error: unknown, + request: Request, + attempt: number, + response?: Response +): RequestError { + if (error instanceof TimeoutError || error instanceof AbortError || error instanceof HTTPError) { + error.request ??= request; + error.response ??= response; + error.attempt ??= attempt; + + return error; + } + + if (isRequestError(error)) { + error.request ??= request; + error.response ??= response; + error.attempt ??= attempt; + + return error; + } + + if (isError(error)) { + return new RequestError(error.message, { + cause: error, + request, + response, + attempt, + }); + } + + return new RequestError(String(error), { + cause: error, + request, + response, + attempt, + }); +} + +export function inferErrorCode(error: unknown): string | undefined { + if (error instanceof RequestError && error.code) { + return error.code; + } + + if (typeof error === 'object' && error !== null && 'code' in error) { + const code = (error as { code?: unknown }).code; + + return typeof code === 'string' ? code : undefined; + } + + return undefined; +} + +export function throwIfAborted(signal?: AbortSignal | null): void { + if (!signal?.aborted) { + return; + } + + throw new AbortError(undefined, { cause: signal.reason }); +} diff --git a/src/http/pipeline/input.ts b/src/http/pipeline/input.ts new file mode 100644 index 0000000..fe2223b --- /dev/null +++ b/src/http/pipeline/input.ts @@ -0,0 +1,61 @@ +import { Buffer } from 'node:buffer'; +import { RequestError } from '../../errors'; +import type { RequestInput, WreqInit } from '../../types'; +import { Request } from '../request'; + +function isGlobalRequest(value: unknown): value is globalThis.Request { + return typeof globalThis.Request !== 'undefined' && value instanceof globalThis.Request; +} + +export async function mergeInputAndInit( + input: RequestInput, + init?: WreqInit +): Promise<{ + urlInput: string | URL; + init: WreqInit; +}> { + if (typeof input === 'string' || input instanceof URL || input instanceof Request) { + if (input instanceof Request && input.bodyUsed && init?.body === undefined) { + throw new TypeError('Request body is already used'); + } + + return { + urlInput: input instanceof Request ? input.url : input, + init: + input instanceof Request + ? { + ...init, + method: init?.method ?? input.method, + headers: init?.headers ?? input.headers, + signal: init?.signal ?? input.signal ?? undefined, + body: init?.body !== undefined ? init.body : (input._cloneBodyBytes() ?? undefined), + } + : { ...init }, + }; + } + + if (isGlobalRequest(input)) { + if (input.bodyUsed && init?.body === undefined) { + throw new TypeError('Request body is already used'); + } + + let body = init?.body; + + if (body === undefined && input.body !== null) { + body = Buffer.from(await input.arrayBuffer()); + } + + return { + urlInput: input.url, + init: { + ...init, + method: init?.method ?? input.method, + headers: init?.headers ?? Array.from(input.headers.entries()), + signal: init?.signal ?? input.signal ?? undefined, + body, + }, + }; + } + + throw new RequestError('URL is required'); +} diff --git a/src/http/pipeline/options.ts b/src/http/pipeline/options.ts new file mode 100644 index 0000000..0747fba --- /dev/null +++ b/src/http/pipeline/options.ts @@ -0,0 +1,101 @@ +import { serializeEmulationOptions } from '../../config/emulation'; +import { Headers } from '../../headers'; +import { normalizeMethod, validateBrowserProfile } from '../../native'; +import type { + NativeRequestOptions, + ResolvedOptions, + ResolvedRetryOptions, + WreqInit, +} from '../../types'; +import { Request } from '../request'; + +const DEFAULT_RETRY_METHODS = ['GET', 'HEAD'] as const; +const DEFAULT_RETRY_STATUS_CODES = [408, 425, 429, 500, 502, 503, 504]; +const DEFAULT_RETRY_ERROR_CODES = [ + 'ECONNABORTED', + 'ECONNRESET', + 'ECONNREFUSED', + 'EPIPE', + 'ETIMEDOUT', + 'ERR_ABORTED', + 'ERR_TIMEOUT', +]; + +function appendQuery(url: URL, query: WreqInit['query']): void { + if (!query) { + return; + } + + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) { + continue; + } + + url.searchParams.set(key, String(value)); + } +} + +function resolveUrl(rawUrl: string | URL, baseURL?: string, query?: WreqInit['query']): string { + const url = baseURL ? new URL(String(rawUrl), baseURL) : new URL(String(rawUrl)); + + appendQuery(url, query); + + return url.toString(); +} + +export function resolveRetryOptions(retry?: WreqInit['retry']): ResolvedRetryOptions { + const configured = typeof retry === 'number' ? { limit: retry } : retry; + + return { + limit: configured?.limit ?? 0, + methods: configured?.methods ?? [...DEFAULT_RETRY_METHODS], + statusCodes: configured?.statusCodes ?? [...DEFAULT_RETRY_STATUS_CODES], + errorCodes: configured?.errorCodes ?? [...DEFAULT_RETRY_ERROR_CODES], + backoff: configured?.backoff, + shouldRetry: configured?.shouldRetry, + }; +} + +export function resolveOptions(init: WreqInit): ResolvedOptions { + return { + ...init, + headers: new Headers(init.headers), + retry: resolveRetryOptions(init.retry), + redirect: init.redirect ?? 'follow', + maxRedirects: init.maxRedirects ?? 20, + throwHttpErrors: init.throwHttpErrors ?? false, + disableDefaultHeaders: init.disableDefaultHeaders ?? false, + compress: init.compress ?? true, + keepOriginalHeaderNames: init.keepOriginalHeaderNames ?? false, + }; +} + +export function createRequest(urlInput: string | URL, options: ResolvedOptions): Request { + validateBrowserProfile(options.browser); + + return new Request(resolveUrl(urlInput, options.baseURL, options.query), { + method: normalizeMethod(options.method), + headers: options.headers, + body: options.body, + signal: options.signal ?? undefined, + }); +} + +export function buildNativeRequest( + request: Request, + options: ResolvedOptions +): NativeRequestOptions { + return { + url: request.url, + method: normalizeMethod(request.method), + headers: request.headers.toTuples(), + origHeaders: options.keepOriginalHeaderNames ? request.headers.toOriginalNames() : undefined, + body: request._getBodyTextForDispatch(), + browser: options.browser, + emulationJson: serializeEmulationOptions(options), + proxy: options.proxy, + timeout: options.timeout, + disableDefaultHeaders: options.disableDefaultHeaders, + compress: options.compress, + }; +} diff --git a/src/http/pipeline/redirects.ts b/src/http/pipeline/redirects.ts new file mode 100644 index 0000000..12bc939 --- /dev/null +++ b/src/http/pipeline/redirects.ts @@ -0,0 +1,96 @@ +import { RequestError } from '../../errors'; +import { Headers } from '../../headers'; +import { normalizeMethod } from '../../native'; +import type { BodyInit, HttpMethod, RedirectEntry } from '../../types'; +import { Response } from '../response'; + +const REDIRECT_STATUS_CODES = new Set([300, 301, 302, 303, 307, 308]); + +export function isRedirectResponse(response: Response): boolean { + return REDIRECT_STATUS_CODES.has(response.status) && response.headers.has('location'); +} + +export function resolveRedirectLocation(response: Response, requestUrl: string): string { + const location = response.headers.get('location'); + + if (!location) { + throw new RequestError('Redirect response is missing Location header'); + } + + try { + return new URL(location, requestUrl).toString(); + } catch (error) { + throw new RequestError(`Invalid redirect URL: ${location}`, { + cause: error, + }); + } +} + +export function stripRedirectSensitiveHeaders( + headers: Headers, + previousUrl: string, + nextUrl: string, + bodyDropped: boolean +): void { + headers.delete('host'); + headers.delete('content-length'); + + if (bodyDropped) { + headers.delete('content-type'); + } + + if (new URL(previousUrl).origin !== new URL(nextUrl).origin) { + headers.delete('authorization'); + } +} + +export function rewriteRedirectMethodAndBody( + method: HttpMethod, + status: number, + body: BodyInit | null | undefined +): { + method: HttpMethod; + body: BodyInit | null | undefined; + bodyDropped: boolean; +} { + if (status === 303) { + return { + method: method === 'HEAD' ? 'HEAD' : 'GET', + body: undefined, + bodyDropped: true, + }; + } + + if ((status === 301 || status === 302) && method === 'POST') { + return { + method: 'GET', + body: undefined, + bodyDropped: true, + }; + } + + return { + method, + body, + bodyDropped: false, + }; +} + +export function finalizeResponse(response: Response, redirectChain: RedirectEntry[]): Response { + return response.setRedirectMetadata(redirectChain); +} + +export function toRedirectEntry( + requestUrl: string, + response: Response, + nextUrl: string, + nextMethod: string +): RedirectEntry { + return { + fromUrl: requestUrl, + status: response.status, + location: response.headers.get('location') ?? nextUrl, + toUrl: nextUrl, + method: normalizeMethod(nextMethod), + }; +} diff --git a/src/http/pipeline/retries.ts b/src/http/pipeline/retries.ts new file mode 100644 index 0000000..feb9c2b --- /dev/null +++ b/src/http/pipeline/retries.ts @@ -0,0 +1,57 @@ +import { normalizeMethod } from '../../native'; +import type { ResolvedRetryOptions, RetryDecisionContext } from '../../types'; +import { inferErrorCode } from './errors'; + +async function sleep(delayMs: number): Promise { + if (delayMs <= 0) { + return; + } + + await new Promise((resolve) => { + setTimeout(resolve, delayMs); + }); +} + +export async function shouldRetryRequest( + context: RetryDecisionContext, + retry: ResolvedRetryOptions +): Promise { + if (context.attempt > retry.limit + 1) { + return false; + } + + if (!retry.methods.includes(normalizeMethod(context.request.method))) { + return false; + } + + if (context.response) { + if (!retry.statusCodes.includes(context.response.status)) { + return false; + } + } else { + const code = inferErrorCode(context.error); + + if (!code || !retry.errorCodes.includes(code)) { + return false; + } + } + + if (!retry.shouldRetry) { + return true; + } + + return retry.shouldRetry(context); +} + +export async function runRetryDelay( + context: RetryDecisionContext, + retry: ResolvedRetryOptions +): Promise { + if (!retry.backoff) { + return; + } + + const delay = await retry.backoff(context); + + await sleep(delay); +} diff --git a/src/http/request.ts b/src/http/request.ts new file mode 100644 index 0000000..d30b291 --- /dev/null +++ b/src/http/request.ts @@ -0,0 +1,162 @@ +import { Blob, Buffer } from 'node:buffer'; +import { ReadableStream } from 'node:stream/web'; +import { Headers } from '../headers'; +import type { BodyInit, HeadersInit, WreqInit } from '../types'; +import { cloneBytes, toBodyBytes } from './body/bytes'; + +export class Request { + readonly url: string; + readonly method: string; + readonly headers: Headers; + readonly signal: AbortSignal | null; + #bodyBytes: Uint8Array | null; + #bodyUsed = false; + #stream: ReadableStream | null = null; + + constructor(input: string | URL | Request, init: WreqInit = {}) { + if (input instanceof Request) { + if (input.bodyUsed) { + throw new TypeError('Request body is already used'); + } + + this.url = String(init.baseURL ? new URL(input.url, init.baseURL) : input.url); + this.method = (init.method ?? input.method).toUpperCase(); + this.headers = new Headers(init.headers ?? input.headers); + this.signal = init.signal ?? input.signal ?? null; + this.#bodyBytes = + init.body !== undefined + ? toBodyBytes(init.body, 'Unsupported request body type') + : cloneBytes(input.#bodyBytes); + + return; + } + + this.url = String(init.baseURL ? new URL(String(input), init.baseURL) : input); + this.method = (init.method ?? 'GET').toUpperCase(); + this.headers = new Headers(init.headers); + this.signal = init.signal ?? null; + this.#bodyBytes = toBodyBytes(init.body, 'Unsupported request body type'); + } + + get body(): ReadableStream | null { + if (this.#bodyUsed || this.#bodyBytes === null) { + return null; + } + + this.#bodyUsed = true; + this.#stream ??= new ReadableStream({ + start: (controller) => { + controller.enqueue(cloneBytes(this.#bodyBytes)!); + controller.close(); + }, + }); + + return this.#stream; + } + + get bodyUsed(): boolean { + return this.#bodyUsed; + } + + clone(): Request { + if (this.#bodyUsed) { + throw new TypeError('Request body is already used'); + } + + const cloned = new Request(this.url, { + method: this.method, + headers: this.headers, + signal: this.signal ?? undefined, + }); + + cloned.#bodyBytes = cloneBytes(this.#bodyBytes); + + return cloned; + } + + async text(): Promise { + return Buffer.from(this.#consumeBytes()).toString('utf8'); + } + + async json(): Promise { + return JSON.parse(await this.text()) as T; + } + + async arrayBuffer(): Promise { + return Uint8Array.from(this.#consumeBytes()).buffer; + } + + async blob(): Promise { + return new Blob([this.#consumeBytes()]); + } + + async formData(): Promise { + const contentType = this.headers.get('content-type')?.toLowerCase() ?? ''; + + if (!contentType.includes('application/x-www-form-urlencoded')) { + throw new TypeError(`Request content-type is not form data: ${contentType || 'unknown'}`); + } + + const formData = new FormData(); + const searchParams = new URLSearchParams(await this.text()); + + for (const [name, value] of searchParams) { + formData.append(name, value); + } + + return formData; + } + + _cloneBodyBytes(): Uint8Array | null { + return cloneBytes(this.#bodyBytes); + } + + _getBodyTextForDispatch(): string | undefined { + if (this.#bodyBytes === null) { + return undefined; + } + + return Buffer.from(this.#bodyBytes).toString('utf8'); + } + + _markBodyUsed(): void { + if (this.#bodyBytes !== null) { + this.#bodyUsed = true; + } + } + + _replace(input: { + url?: string; + method?: string; + headers?: HeadersInit; + body?: BodyInit | null; + }): Request { + const hasBodyOverride = Object.prototype.hasOwnProperty.call(input, 'body'); + const next = new Request(input.url ?? this.url, { + method: input.method ?? this.method, + headers: input.headers ?? this.headers, + signal: this.signal ?? undefined, + body: hasBodyOverride ? input.body : undefined, + }); + + if (!hasBodyOverride) { + next.#bodyBytes = cloneBytes(this.#bodyBytes); + } + + return next; + } + + #consumeBytes(): Uint8Array { + if (this.#bodyUsed) { + throw new TypeError('Request body is already used'); + } + + this.#bodyUsed = true; + + return cloneBytes(this.#bodyBytes) ?? new Uint8Array(); + } +} + +export function isWreqRequest(value: unknown): value is Request { + return value instanceof Request; +} diff --git a/src/http/response-meta.ts b/src/http/response-meta.ts new file mode 100644 index 0000000..468a9d9 --- /dev/null +++ b/src/http/response-meta.ts @@ -0,0 +1,41 @@ +import { Readable } from 'node:stream'; +import type { RequestTimings, RedirectEntry, WreqResponseMeta } from '../types'; +import type { Response } from './response'; + +export class ResponseMeta implements WreqResponseMeta { + constructor(private readonly response: Response) {} + + get cookies(): Record { + return { ...this.response._cookies }; + } + + get setCookies(): string[] { + return [...this.response._setCookies]; + } + + get timings(): RequestTimings | undefined { + return this.response._timings ? { ...this.response._timings } : undefined; + } + + get redirectChain(): RedirectEntry[] { + return [...this.response._redirectChain]; + } + + get contentLength(): number | undefined { + const value = this.response.headers.get('content-length'); + + if (!value) { + return undefined; + } + + const parsed = Number(value); + + return Number.isFinite(parsed) ? parsed : undefined; + } + + readable(): Readable { + const body = this.response.clone().body; + + return body ? Readable.fromWeb(body) : Readable.from([]); + } +} diff --git a/src/http/response.ts b/src/http/response.ts new file mode 100644 index 0000000..55fe68f --- /dev/null +++ b/src/http/response.ts @@ -0,0 +1,339 @@ +import { Blob, Buffer } from 'node:buffer'; +import { STATUS_CODES } from 'node:http'; +import { ReadableStream } from 'node:stream/web'; +import { Headers } from '../headers'; +import { nativeCancelBody, nativeReadBodyChunk } from '../native'; +import type { + BodyInit, + HeadersInit, + NativeResponse, + RedirectEntry, + RequestTimings, + WreqResponseMeta, +} from '../types'; +import { cloneBytes, toBodyBytes } from './body/bytes'; +import { parseResponseFormData } from './body/form-data'; +import { ResponseMeta } from './response-meta'; + +type ResponseInitWithUrl = ResponseInit & { + url?: string; +}; + +function toHeadersInit(headers: ResponseInit['headers'] | undefined): HeadersInit | undefined { + if (headers === undefined) { + return undefined; + } + + if (headers instanceof Headers) { + return new Headers(headers); + } + + if (typeof globalThis.Headers !== 'undefined' && headers instanceof globalThis.Headers) { + return new Headers(Array.from(headers.entries())); + } + + return headers as unknown as HeadersInit; +} + +function isNativeResponse(value: unknown): value is NativeResponse { + return ( + typeof value === 'object' && + value !== null && + 'status' in value && + 'headers' in value && + 'url' in value + ); +} + +export class Response { + readonly status: number; + readonly statusText: string; + readonly ok: boolean; + readonly url: string; + readonly headers: Headers; + readonly type = 'basic' as const; + readonly wreq: WreqResponseMeta; + _cookies: Record; + _setCookies: string[]; + _timings?: RequestTimings; + _redirectChain: RedirectEntry[]; + redirected: boolean; + #payloadBytes: Uint8Array | null; + #bodyHandle: number | null; + #bodyUsed: boolean; + #streamSource: ReadableStream | null; + #stream: ReadableStream | null; + #orphanedStreamReaders: ReadableStreamDefaultReader[]; + + constructor(body?: BodyInit | NativeResponse | null, init: ResponseInitWithUrl = {}) { + if (isNativeResponse(body)) { + this.status = body.status; + this.statusText = body.statusText ?? STATUS_CODES[body.status] ?? ''; + this.url = body.url; + this.headers = new Headers(body.headers); + this._cookies = { ...body.cookies }; + this._setCookies = [...(body.setCookies ?? [])]; + this._timings = body.timings ? { ...body.timings } : undefined; + this._redirectChain = []; + this.redirected = false; + this.#payloadBytes = body.body !== undefined ? Buffer.from(body.body, 'utf8') : null; + this.#bodyHandle = body.bodyHandle ?? null; + this.#stream = null; + } else { + this.status = init.status ?? 200; + this.statusText = init.statusText ?? STATUS_CODES[this.status] ?? ''; + this.url = init.url ?? ''; + this.headers = new Headers(toHeadersInit(init.headers)); + this._cookies = {}; + this._setCookies = []; + this._timings = undefined; + this._redirectChain = []; + this.redirected = false; + this.#payloadBytes = toBodyBytes(body ?? null, 'Unsupported response body type'); + this.#bodyHandle = null; + this.#stream = null; + } + + this.ok = this.status >= 200 && this.status < 300; + this.#bodyUsed = false; + this.#streamSource = null; + this.wreq = new ResponseMeta(this); + this.#orphanedStreamReaders = []; + } + + get bodyUsed(): boolean { + return this.#bodyUsed; + } + + setRedirectMetadata(chain: RedirectEntry[]): this { + this.redirected = chain.length > 0; + this._redirectChain = [...chain]; + + return this; + } + + setTimings(timings: RequestTimings): this { + this._timings = { ...timings }; + + return this; + } + + get body(): ReadableStream | null { + return this.#ensureStream(); + } + + async text(): Promise { + return Buffer.from(await this.#consumeBytes()).toString('utf8'); + } + + async json(): Promise { + return JSON.parse(await this.text()) as T; + } + + async arrayBuffer(): Promise { + return Uint8Array.from(await this.#consumeBytes()).buffer; + } + + async blob(): Promise { + return new Blob([await this.#consumeBytes()]); + } + + async formData(): Promise { + const contentType = this.headers.get('content-type') ?? ''; + + return parseResponseFormData(await this.#consumeBytes(), contentType); + } + + clone(): Response { + if (this.#isBodyUnusable()) { + throw new TypeError('Response.clone: Body has already been consumed.'); + } + + const cloned = new Response(null, { + status: this.status, + statusText: this.statusText, + headers: this.headers.toObject(), + url: this.url, + }); + + cloned._cookies = { ...this._cookies }; + cloned._setCookies = [...this._setCookies]; + cloned._timings = this._timings ? { ...this._timings } : undefined; + cloned._redirectChain = [...this._redirectChain]; + cloned.redirected = this.redirected; + + const source = this.#ensureStreamSource(); + + if (source) { + const previousStream = this.#stream; + const [left, right] = source.tee(); + + this.#streamSource = left; + this.#stream = null; + cloned.#streamSource = right; + if (previousStream) { + this.#orphanedStreamReaders.push(previousStream.getReader()); + } + + return cloned; + } + + if (this.#payloadBytes !== null) { + cloned.#payloadBytes = cloneBytes(this.#payloadBytes); + + return cloned; + } + + return cloned; + } + + #isBodyUnusable(): boolean { + return this.#bodyUsed || this.#stream?.locked === true; + } + + #ensureStreamSource(): ReadableStream | null { + if (this.#streamSource) { + return this.#streamSource; + } + + if (this.#payloadBytes !== null) { + const bytes = cloneBytes(this.#payloadBytes)!; + let emitted = false; + + this.#streamSource = new ReadableStream({ + pull: (controller) => { + if (!emitted) { + emitted = true; + controller.enqueue(bytes); + } + + controller.close(); + }, + }); + + return this.#streamSource; + } + + if (this.#bodyHandle === null) { + return null; + } + + const handle = this.#bodyHandle; + + this.#bodyHandle = null; + this.#streamSource = new ReadableStream({ + pull: async (controller) => { + const result = await nativeReadBodyChunk(handle); + + if (result.chunk.length > 0) { + controller.enqueue(new Uint8Array(result.chunk)); + } + + if (result.done) { + controller.close(); + } + }, + cancel: async () => { + nativeCancelBody(handle); + }, + }); + + return this.#streamSource; + } + + #markBodyComplete(): void { + if (!this._timings || this._timings.endTime !== undefined) { + return; + } + + const endTime = Date.now(); + + this._timings = { + ...this._timings, + endTime, + total: endTime - this._timings.startTime, + }; + } + + #ensureStream(): ReadableStream | null { + if (this.#stream) { + return this.#stream; + } + + const source = this.#ensureStreamSource(); + + if (!source) { + return null; + } + + let sourceReader: ReadableStreamDefaultReader | null = null; + + this.#stream = new ReadableStream( + { + pull: async (controller) => { + this.#bodyUsed = true; + sourceReader ??= source.getReader(); + const result = await sourceReader.read(); + + if (result.done) { + this.#markBodyComplete(); + controller.close(); + + return; + } + + controller.enqueue(result.value); + }, + cancel: async () => { + this.#bodyUsed = true; + if (sourceReader) { + await sourceReader.cancel(); + } else { + await source.cancel(); + } + + this.#markBodyComplete(); + }, + }, + { highWaterMark: 0 } + ); + + return this.#stream; + } + + async #consumeBytes(): Promise { + if (this.#isBodyUnusable()) { + throw new TypeError('Body is unusable: Body has already been read'); + } + + const stream = this.#ensureStream(); + + if (stream) { + this.#bodyUsed = true; + const reader = stream.getReader(); + + this.#orphanedStreamReaders.push(reader); + const chunks: Uint8Array[] = []; + + while (true) { + const result = await reader.read(); + + if (result.done) { + break; + } + + chunks.push(result.value); + } + + const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + + this.#payloadBytes = new Uint8Array(merged); + + return new Uint8Array(this.#payloadBytes); + } + + const bytes = this.#payloadBytes; + + return bytes === null ? new Uint8Array() : new Uint8Array(bytes); + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..3705d94 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,141 @@ +import { createClient } from './client'; +import { BROWSER_PROFILES } from './config/generated/browser-profiles'; +import { AbortError, HTTPError, RequestError, TimeoutError, WebSocketError } from './errors'; +import { Headers } from './headers'; +import { fetch } from './http/fetch'; +import { Request } from './http/request'; +import { Response } from './http/response'; +import { getProfiles } from './native'; +import type { + AfterResponseContext, + AlpnProtocol, + AlpsProtocol, + BeforeErrorContext, + BeforeRedirectContext, + BeforeRetryContext, + BeforeRequestContext, + BodyInit, + BrowserProfile, + Client, + ClientDefaults, + CookieJar, + CookieJarCookie, + HeaderTuple, + HeadersInit, + HookState, + Hooks, + Http1Options, + Http2ExperimentalSetting, + Http2Options, + Http2Priority, + Http2PseudoHeaderId, + Http2SettingId, + Http2StreamDependency, + HttpMethod, + InitContext, + NativeRequestOptions, + NativeResponse, + NativeWebSocketConnectOptions, + NativeWebSocketConnection, + NativeWebSocketReadResult, + RedirectEntry, + RedirectMode, + RequestInput, + RequestStats, + RequestTimings, + ResolvedOptions, + ResolvedRetryOptions, + RetryDecisionContext, + RetryOptions, + TlsOptions, + TlsVersion, + WebSocketBinaryType, + WebSocketInit, + WreqInit, +} from './types'; +import { CloseEvent, WebSocket, websocket } from './websocket'; + +export { + fetch, + createClient, + Headers, + Request, + Response, + RequestError, + HTTPError, + TimeoutError, + AbortError, + BROWSER_PROFILES, + WebSocket, + CloseEvent, + websocket, + WebSocketError, +}; + +export { getProfiles }; + +export type { + AfterResponseContext, + AlpnProtocol, + AlpsProtocol, + BeforeErrorContext, + BeforeRedirectContext, + BeforeRetryContext, + BeforeRequestContext, + BodyInit, + BrowserProfile, + Client, + ClientDefaults, + CookieJar, + CookieJarCookie, + HeaderTuple, + HeadersInit, + HookState, + Hooks, + Http1Options, + Http2ExperimentalSetting, + Http2Options, + Http2Priority, + Http2PseudoHeaderId, + Http2SettingId, + Http2StreamDependency, + HttpMethod, + InitContext, + NativeRequestOptions, + NativeResponse, + NativeWebSocketConnectOptions, + NativeWebSocketConnection, + NativeWebSocketReadResult, + RedirectEntry, + RedirectMode, + RequestInput, + RequestStats, + RequestTimings, + ResolvedOptions, + ResolvedRetryOptions, + RetryDecisionContext, + RetryOptions, + TlsOptions, + TlsVersion, + WebSocketBinaryType, + WebSocketInit, + WreqInit, +}; + +export default { + fetch, + createClient, + getProfiles, + BROWSER_PROFILES, + Headers, + Request, + Response, + RequestError, + HTTPError, + TimeoutError, + AbortError, + WebSocket, + CloseEvent, + websocket, + WebSocketError, +}; diff --git a/src/native.ts b/src/native.ts deleted file mode 100644 index c374315..0000000 --- a/src/native.ts +++ /dev/null @@ -1,157 +0,0 @@ -import type { - BrowserProfile, - HttpMethod, - NativeResponse, - NativeRequestOptions, - NativeWebSocketConnectOptions, - NativeWebSocketConnection, - NativeWebSocketReadResult, -} from './types'; - -type NativeBinding = { - request: (options: NativeRequestOptions) => Promise; - websocketConnect: (options: NativeWebSocketConnectOptions) => Promise; - websocketRead: (handle: number) => Promise; - websocketSendText: (handle: number, text: string) => Promise; - websocketSendBinary: (handle: number, data: Buffer) => Promise; - websocketClose: (handle: number, code?: number, reason?: string) => Promise; - readBodyChunk: ( - handle: number, - size?: number - ) => Promise<{ - chunk: Buffer; - done: boolean; - }>; - readBodyAll: (handle: number) => Promise; - cancelBody: (handle: number) => boolean; - getProfiles: () => string[]; -}; - -let nativeBinding: NativeBinding | undefined; - -function loadNativeBinding(): NativeBinding { - const platform = process.platform; - const arch = process.arch; - - const platformArchMap: Record> = { - darwin: { - x64: 'darwin-x64', - arm64: 'darwin-arm64', - }, - linux: { - x64: 'linux-x64-gnu', - }, - win32: { - x64: 'win32-x64-msvc', - }, - }; - - const platformArch = platformArchMap[platform]?.[arch]; - if (!platformArch) { - throw new Error( - `Unsupported platform: ${platform}-${arch}. ` + - `Supported platforms: darwin-x64, darwin-arm64, linux-x64, win32-x64` - ); - } - - const binaryName = `node-wreq.${platformArch}.node`; - - try { - return require(`../rust/${binaryName}`) as NativeBinding; - } catch { - try { - return require('../rust/node-wreq.node') as NativeBinding; - } catch { - throw new Error( - `Failed to load native module for ${platform}-${arch}. ` + - `Tried: ../rust/${binaryName} and ../rust/node-wreq.node. ` + - `Make sure the package is installed correctly and the native module is built for your platform.` - ); - } - } -} - -function getBinding(): NativeBinding { - nativeBinding ??= loadNativeBinding(); - return nativeBinding; -} - -let cachedProfiles: BrowserProfile[] | undefined; - -export function getProfiles(): BrowserProfile[] { - cachedProfiles ??= getBinding().getProfiles() as BrowserProfile[]; - return cachedProfiles; -} - -export async function nativeRequest(options: NativeRequestOptions): Promise { - return getBinding().request(options); -} - -export async function nativeReadBodyChunk( - handle: number, - size?: number -): Promise<{ - chunk: Uint8Array; - done: boolean; -}> { - return getBinding().readBodyChunk(handle, size); -} - -export async function nativeReadBodyAll(handle: number): Promise { - return getBinding().readBodyAll(handle); -} - -export function nativeCancelBody(handle: number): boolean { - return getBinding().cancelBody(handle); -} - -export async function nativeWebSocketConnect( - options: NativeWebSocketConnectOptions -): Promise { - return getBinding().websocketConnect(options); -} - -export async function nativeWebSocketRead(handle: number): Promise { - return getBinding().websocketRead(handle); -} - -export async function nativeWebSocketSendText(handle: number, text: string): Promise { - return getBinding().websocketSendText(handle, text); -} - -export async function nativeWebSocketSendBinary(handle: number, data: Uint8Array): Promise { - return getBinding().websocketSendBinary(handle, Buffer.from(data)); -} - -export async function nativeWebSocketClose( - handle: number, - code?: number, - reason?: string -): Promise { - return getBinding().websocketClose(handle, code, reason); -} - -export function validateBrowserProfile(browser?: BrowserProfile): void { - if (!browser) { - return; - } - - if (!getProfiles().includes(browser)) { - throw new Error(`Invalid browser profile: ${browser}`); - } -} - -export function normalizeMethod(method?: string): HttpMethod { - const normalized = (method ?? 'GET').toUpperCase(); - switch (normalized) { - case 'GET': - case 'POST': - case 'PUT': - case 'DELETE': - case 'PATCH': - case 'HEAD': - return normalized; - default: - throw new Error(`Unsupported HTTP method: ${method}`); - } -} diff --git a/src/native/binding.ts b/src/native/binding.ts new file mode 100644 index 0000000..037c42b --- /dev/null +++ b/src/native/binding.ts @@ -0,0 +1,142 @@ +import { execSync } from 'node:child_process'; +import type { + NativeRequestOptions, + NativeResponse, + NativeWebSocketConnectOptions, + NativeWebSocketConnection, + NativeWebSocketReadResult, +} from '../types'; + +export type NativeBinding = { + request: (options: NativeRequestOptions) => Promise; + websocketConnect: (options: NativeWebSocketConnectOptions) => Promise; + websocketRead: (handle: number) => Promise; + websocketSendText: (handle: number, text: string) => Promise; + websocketSendBinary: (handle: number, data: Buffer) => Promise; + websocketClose: (handle: number, code?: number, reason?: string) => Promise; + readBodyChunk: ( + handle: number, + size?: number + ) => Promise<{ + chunk: Buffer; + done: boolean; + }>; + readBodyAll: (handle: number) => Promise; + cancelBody: (handle: number) => boolean; + getProfiles: () => string[]; +}; + +type NativeTarget = { + binaryName: string; + packageName: string; +}; + +let nativeBinding: NativeBinding | undefined; + +function isMuslRuntime(): boolean { + if (process.platform !== 'linux') { + return false; + } + + if (typeof process.report?.getReport === 'function') { + const report = process.report.getReport() as { + header?: { + glibcVersionRuntime?: string; + }; + }; + + if (report.header?.glibcVersionRuntime) { + return false; + } + } + + try { + const output = execSync('ldd --version', { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + + return output.toLowerCase().includes('musl'); + } catch { + return true; + } +} + +function loadNativeBinding(): NativeBinding { + const platform = process.platform; + const arch = process.arch; + const libc = platform === 'linux' ? (isMuslRuntime() ? 'musl' : 'gnu') : undefined; + const platformArchMap: Record> = { + darwin: { + x64: { + binaryName: 'node-wreq.darwin-x64.node', + packageName: '@node-wreq/darwin-x64', + }, + arm64: { + binaryName: 'node-wreq.darwin-arm64.node', + packageName: '@node-wreq/darwin-arm64', + }, + }, + linux: { + x64: + libc === 'musl' + ? { + binaryName: 'node-wreq.linux-x64-musl.node', + packageName: '@node-wreq/linux-x64-musl', + } + : { + binaryName: 'node-wreq.linux-x64-gnu.node', + packageName: '@node-wreq/linux-x64-gnu', + }, + arm64: { + binaryName: 'node-wreq.linux-arm64-gnu.node', + packageName: '@node-wreq/linux-arm64-gnu', + }, + }, + win32: { + x64: { + binaryName: 'node-wreq.win32-x64-msvc.node', + packageName: '@node-wreq/win32-x64-msvc', + }, + }, + }; + + const target = platformArchMap[platform]?.[arch]; + + if (!target) { + throw new Error( + `Unsupported platform: ${platform}-${arch}. ` + + `Supported platforms: darwin-x64, darwin-arm64, linux-x64-gnu, linux-x64-musl, linux-arm64-gnu, win32-x64-msvc` + ); + } + + const attempted: string[] = [target.packageName]; + + try { + return require(target.packageName) as NativeBinding; + } catch { + attempted.push(`../rust/${target.binaryName}`); + } + + try { + return require(`../rust/${target.binaryName}`) as NativeBinding; + } catch { + attempted.push('../rust/node-wreq.node'); + } + + try { + return require('../rust/node-wreq.node') as NativeBinding; + } catch { + throw new Error( + `Failed to load native module for ${platform}-${arch}. ` + + `Tried: ${attempted.join(', ')}. ` + + `Make sure the matching @node-wreq platform package is installed or build the local native module.` + ); + } +} + +export function getBinding(): NativeBinding { + nativeBinding ??= loadNativeBinding(); + + return nativeBinding; +} diff --git a/src/native/index.ts b/src/native/index.ts new file mode 100644 index 0000000..7a9be2a --- /dev/null +++ b/src/native/index.ts @@ -0,0 +1,10 @@ +export { normalizeMethod } from './methods'; +export { getProfiles, validateBrowserProfile } from './profiles'; +export { nativeCancelBody, nativeReadBodyAll, nativeReadBodyChunk, nativeRequest } from './request'; +export { + nativeWebSocketClose, + nativeWebSocketConnect, + nativeWebSocketRead, + nativeWebSocketSendBinary, + nativeWebSocketSendText, +} from './websocket'; diff --git a/src/native/methods.ts b/src/native/methods.ts new file mode 100644 index 0000000..5da6bb1 --- /dev/null +++ b/src/native/methods.ts @@ -0,0 +1,17 @@ +import type { HttpMethod } from '../types'; + +export function normalizeMethod(method?: string): HttpMethod { + const normalized = (method ?? 'GET').toUpperCase(); + + switch (normalized) { + case 'GET': + case 'POST': + case 'PUT': + case 'DELETE': + case 'PATCH': + case 'HEAD': + return normalized; + default: + throw new Error(`Unsupported HTTP method: ${method}`); + } +} diff --git a/src/native/profiles.ts b/src/native/profiles.ts new file mode 100644 index 0000000..5995228 --- /dev/null +++ b/src/native/profiles.ts @@ -0,0 +1,20 @@ +import type { BrowserProfile } from '../types'; +import { getBinding } from './binding'; + +let cachedProfiles: BrowserProfile[] | undefined; + +export function getProfiles(): BrowserProfile[] { + cachedProfiles ??= getBinding().getProfiles() as BrowserProfile[]; + + return cachedProfiles; +} + +export function validateBrowserProfile(browser?: BrowserProfile): void { + if (!browser) { + return; + } + + if (!getProfiles().includes(browser)) { + throw new Error(`Invalid browser profile: ${browser}`); + } +} diff --git a/src/native/request.ts b/src/native/request.ts new file mode 100644 index 0000000..b0c4b4d --- /dev/null +++ b/src/native/request.ts @@ -0,0 +1,24 @@ +import type { NativeRequestOptions, NativeResponse } from '../types'; +import { getBinding } from './binding'; + +export async function nativeRequest(options: NativeRequestOptions): Promise { + return getBinding().request(options); +} + +export async function nativeReadBodyChunk( + handle: number, + size?: number +): Promise<{ + chunk: Uint8Array; + done: boolean; +}> { + return getBinding().readBodyChunk(handle, size); +} + +export async function nativeReadBodyAll(handle: number): Promise { + return getBinding().readBodyAll(handle); +} + +export function nativeCancelBody(handle: number): boolean { + return getBinding().cancelBody(handle); +} diff --git a/src/native/websocket.ts b/src/native/websocket.ts new file mode 100644 index 0000000..c769c54 --- /dev/null +++ b/src/native/websocket.ts @@ -0,0 +1,33 @@ +import { Buffer } from 'node:buffer'; +import type { + NativeWebSocketConnectOptions, + NativeWebSocketConnection, + NativeWebSocketReadResult, +} from '../types'; +import { getBinding } from './binding'; + +export async function nativeWebSocketConnect( + options: NativeWebSocketConnectOptions +): Promise { + return getBinding().websocketConnect(options); +} + +export async function nativeWebSocketRead(handle: number): Promise { + return getBinding().websocketRead(handle); +} + +export async function nativeWebSocketSendText(handle: number, text: string): Promise { + return getBinding().websocketSendText(handle, text); +} + +export async function nativeWebSocketSendBinary(handle: number, data: Uint8Array): Promise { + return getBinding().websocketSendBinary(handle, Buffer.from(data)); +} + +export async function nativeWebSocketClose( + handle: number, + code?: number, + reason?: string +): Promise { + return getBinding().websocketClose(handle, code, reason); +} diff --git a/src/node-wreq.ts b/src/node-wreq.ts index 74781bb..04c9c98 100644 --- a/src/node-wreq.ts +++ b/src/node-wreq.ts @@ -1,87 +1,2 @@ -import { createClient } from './client'; -import { AbortError, HTTPError, RequestError, TimeoutError, WebSocketError } from './errors'; -import { fetch } from './fetch'; -import { Headers } from './headers'; -import { getProfiles } from './native'; -import { Response } from './response'; -import type { - AfterResponseContext, - BeforeErrorContext, - BeforeRedirectContext, - BeforeRetryContext, - BeforeRequestContext, - BrowserProfile, - ClientDefaults, - CookieJar, - CookieJarCookie, - Hooks, - HttpMethod, - InitContext, - RedirectEntry, - RedirectMode, - RequestStats, - RequestTimings, - RequestInput, - RetryOptions, - WebSocketBinaryType, - WebSocketInit, - WreqInit, -} from './types'; -import { CloseEvent, WebSocket, websocket } from './websocket'; - -export { - fetch, - createClient, - Headers, - Response, - RequestError, - HTTPError, - TimeoutError, - AbortError, - WebSocket, - CloseEvent, - websocket, - WebSocketError, -}; - -export { getProfiles }; - -export type { - AfterResponseContext, - BeforeErrorContext, - BeforeRedirectContext, - BeforeRetryContext, - BeforeRequestContext, - BrowserProfile, - ClientDefaults, - CookieJar, - CookieJarCookie, - Hooks, - HttpMethod, - InitContext, - RedirectEntry, - RedirectMode, - RequestStats, - RequestTimings, - RequestInput, - RetryOptions, - WebSocketBinaryType, - WebSocketInit, - WreqInit, -}; - -export default { - fetch, - createClient, - getProfiles, - Headers, - Response, - RequestError, - HTTPError, - TimeoutError, - AbortError, - WebSocket, - CloseEvent, - websocket, - WebSocketError, -}; +export * from './index'; +export { default } from './index'; diff --git a/src/ws.d.ts b/src/platform/ws.d.ts similarity index 100% rename from src/ws.d.ts rename to src/platform/ws.d.ts diff --git a/src/response.ts b/src/response.ts deleted file mode 100644 index 7836cc0..0000000 --- a/src/response.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { Blob, Buffer } from 'node:buffer'; -import { ReadableStream } from 'node:stream/web'; -import { Headers } from './headers'; -import { nativeCancelBody, nativeReadBodyAll, nativeReadBodyChunk } from './native'; -import type { NativeResponse, RedirectEntry, RequestTimings } from './types'; - -export class Response { - readonly status: number; - readonly ok: boolean; - readonly url: string; - readonly headers: Headers; - readonly cookies: Record; - readonly setCookies: string[]; - timings?: RequestTimings; - redirected: boolean; - redirectChain: RedirectEntry[]; - bodyUsed: boolean; - private payload: string | null; - private payloadBytes: Uint8Array | null; - private bodyHandle: number | null; - private stream: ReadableStream | null; - - constructor(nativeResponse: NativeResponse) { - this.status = nativeResponse.status; - this.ok = this.status >= 200 && this.status < 300; - this.url = nativeResponse.url; - this.headers = new Headers(nativeResponse.headers); - this.cookies = { ...nativeResponse.cookies }; - this.setCookies = [...(nativeResponse.setCookies ?? [])]; - this.timings = nativeResponse.timings ? { ...nativeResponse.timings } : undefined; - this.redirected = false; - this.redirectChain = []; - this.payload = nativeResponse.body ?? null; - this.payloadBytes = nativeResponse.body ? Buffer.from(nativeResponse.body, 'utf8') : null; - this.bodyHandle = nativeResponse.bodyHandle ?? null; - this.bodyUsed = false; - this.stream = null; - } - - setRedirectMetadata(chain: RedirectEntry[]): this { - this.redirected = chain.length > 0; - this.redirectChain = [...chain]; - return this; - } - - setTimings(timings: RequestTimings): this { - this.timings = { ...timings }; - return this; - } - - private markBodyComplete(): void { - if (!this.timings || this.timings.endTime !== undefined) { - return; - } - - const endTime = Date.now(); - this.timings = { - ...this.timings, - endTime, - total: endTime - this.timings.startTime, - }; - } - - get body(): ReadableStream | null { - if (this.bodyUsed) { - return null; - } - - this.bodyUsed = true; - - if (this.payloadBytes) { - this.stream ??= new ReadableStream({ - start: (controller) => { - controller.enqueue(new Uint8Array(this.payloadBytes!)); - controller.close(); - }, - }); - - return this.stream; - } - - if (this.bodyHandle === null) { - return null; - } - - const handle = this.bodyHandle; - this.stream = new ReadableStream({ - pull: async (controller) => { - const result = await nativeReadBodyChunk(handle); - if (result.chunk.length > 0) { - controller.enqueue(new Uint8Array(result.chunk)); - } - - if (result.done) { - this.bodyHandle = null; - this.markBodyComplete(); - controller.close(); - } - }, - cancel: async () => { - this.bodyHandle = null; - nativeCancelBody(handle); - this.markBodyComplete(); - }, - }); - - return this.stream; - } - - get bodyText(): string { - if (this.payload === null) { - throw new TypeError('Response body text is not available synchronously'); - } - - return this.payload; - } - - private async consumeBytes(): Promise { - if (this.bodyUsed) { - throw new TypeError('Response body is already used'); - } - - this.bodyUsed = true; - - if (this.payloadBytes) { - this.markBodyComplete(); - return new Uint8Array(this.payloadBytes); - } - - if (this.bodyHandle === null) { - this.markBodyComplete(); - return new Uint8Array(); - } - - const bytes = await nativeReadBodyAll(this.bodyHandle); - this.bodyHandle = null; - this.payloadBytes = new Uint8Array(bytes); - this.payload = Buffer.from(this.payloadBytes).toString('utf8'); - this.markBodyComplete(); - return new Uint8Array(this.payloadBytes); - } - - private getContentType(): string { - return this.headers.get('content-type')?.toLowerCase() ?? ''; - } - - async text(): Promise { - return Buffer.from(await this.consumeBytes()).toString('utf8'); - } - - async json(): Promise { - return JSON.parse(await this.text()) as T; - } - - async arrayBuffer(): Promise { - const buffer = Buffer.from(await this.consumeBytes()); - - return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); - } - - async blob(): Promise { - return new Blob([await this.consumeBytes()]); - } - - async formData(): Promise { - const contentType = this.getContentType(); - - if (contentType.includes('application/x-www-form-urlencoded')) { - const formData = new FormData(); - const searchParams = new URLSearchParams(await this.text()); - - for (const [name, value] of searchParams) { - formData.append(name, value); - } - - return formData; - } - - if (contentType.includes('multipart/form-data')) { - throw new TypeError('multipart/form-data parsing is not implemented yet'); - } - - throw new TypeError(`Response content-type is not form data: ${contentType || 'unknown'}`); - } - - clone(): Response { - if (this.bodyUsed) { - throw new TypeError('Response body is already used'); - } - - if (!this.payloadBytes || this.payload === null) { - throw new TypeError('Cannot clone a native streaming response before it is buffered'); - } - - const cloned = new Response({ - status: this.status, - headers: this.headers.toObject(), - body: this.payload, - cookies: this.cookies, - setCookies: this.setCookies, - timings: this.timings, - url: this.url, - }); - - return cloned.setRedirectMetadata(this.redirectChain); - } -} diff --git a/src/test/cookies-redirects.spec.ts b/src/test/cookies-redirects.spec.ts new file mode 100644 index 0000000..6ad6c2c --- /dev/null +++ b/src/test/cookies-redirects.spec.ts @@ -0,0 +1,183 @@ +import assert from 'node:assert'; +import { describe, test } from 'node:test'; +import { fetch } from '../node-wreq'; +import { setupLocalTestServer } from './helpers/local-server'; + +describe('cookies and redirects', () => { + const { getBaseUrl } = setupLocalTestServer(); + + test('should integrate cookieJar across requests', async () => { + const jarStore = new Map(); + const cookieJar = { + getCookies: () => + [...jarStore.entries()].map(([name, value]) => ({ + name, + value, + })), + setCookie: (cookie: string) => { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + + jarStore.set(name, value); + }, + }; + + const setResponse = await fetch(`${getBaseUrl()}/cookies/set`, { + cookieJar, + }); + + assert.strictEqual(setResponse.status, 200, 'cookie source request should succeed'); + assert.strictEqual( + jarStore.get('session'), + 'abc123', + 'cookieJar should persist response cookies' + ); + + const echoResponse = await fetch(`${getBaseUrl()}/cookies/echo`, { + cookieJar, + }); + + const body = await echoResponse.json<{ cookie: string }>(); + + assert.ok( + body.cookie.includes('session=abc123'), + 'stored cookies should be sent on the next request' + ); + }); + + test('should persist multiple raw set-cookie headers through cookieJar', async () => { + const rawCookies: string[] = []; + const cookieJar = { + getCookies: () => + rawCookies.map((cookie) => { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + + return { name, value }; + }), + setCookie: (cookie: string) => { + rawCookies.push(cookie); + }, + }; + + const response = await fetch(`${getBaseUrl()}/cookies/set-multiple`, { + cookieJar, + }); + + assert.strictEqual(response.status, 200, 'multiple set-cookie response should succeed'); + assert.deepStrictEqual( + rawCookies, + ['session=abc123; Path=/', 'csrf=token123; Path=/'], + 'cookieJar should receive each raw Set-Cookie value separately' + ); + + const echoResponse = await fetch(`${getBaseUrl()}/cookies/echo`, { + cookieJar, + }); + + const body = await echoResponse.json<{ cookie: string }>(); + + assert.ok(body.cookie.includes('session=abc123'), 'first cookie should be replayed'); + assert.ok(body.cookie.includes('csrf=token123'), 'second cookie should be replayed'); + }); + + test('should follow redirects manually and run beforeRedirect hooks', async () => { + const cookieJar = { + store: new Map(), + getCookies(this: { store: Map }) { + return [...this.store.entries()].map(([name, value]) => ({ + name, + value, + })); + }, + setCookie(this: { store: Map }, cookie: string) { + const [pair] = cookie.split(';'); + const [name, value = ''] = pair.split('='); + + this.store.set(name, value); + }, + }; + + const response = await fetch(`${getBaseUrl()}/redirect/start`, { + cookieJar, + hooks: { + beforeRedirect: [ + ({ request, redirectCount, nextUrl }) => { + assert.strictEqual(redirectCount, 1); + assert.strictEqual(nextUrl, `${getBaseUrl()}/redirect/final`); + request.headers.set('X-Redirect-Hook', 'active'); + }, + ], + }, + }); + + assert.strictEqual(response.status, 200, 'redirect chain should resolve to final response'); + assert.strictEqual(response.redirected, true, 'final response should be marked redirected'); + assert.strictEqual(response.wreq.redirectChain.length, 1, 'redirect chain should be recorded'); + + const body = await response.json<{ + method: string; + cookie: string; + hookHeader: string; + }>(); + + assert.strictEqual(body.method, 'GET', 'GET redirect should preserve method'); + assert.ok( + body.cookie.includes('redirect_session=1'), + 'intermediate set-cookie should affect the next redirect hop' + ); + assert.strictEqual( + body.hookHeader, + 'active', + 'beforeRedirect should be able to mutate next request' + ); + }); + + test('should rewrite POST to GET on 302 redirects', async () => { + const response = await fetch(`${getBaseUrl()}/redirect/post-start`, { + method: 'POST', + body: 'hello', + }); + + const body = await response.json<{ method: string }>(); + + assert.strictEqual(body.method, 'GET', '302 redirect from POST should be rewritten to GET'); + }); + + test('should support manual redirect mode', async () => { + const response = await fetch(`${getBaseUrl()}/redirect/start`, { + redirect: 'manual', + }); + + assert.strictEqual( + response.status, + 302, + 'manual redirect mode should return the redirect response' + ); + assert.strictEqual( + response.headers.get('location'), + '/redirect/final', + 'manual redirect mode should expose Location' + ); + assert.strictEqual( + response.redirected, + false, + 'manual redirect response should not be marked redirected' + ); + }); + + test('should support redirect error mode', async () => { + await assert.rejects( + async () => { + await fetch(`${getBaseUrl()}/redirect/start`, { + redirect: 'error', + }); + }, + (error: unknown) => + error instanceof Error && + error.message.includes('Redirect encountered') && + 'code' in (error as object), + 'redirect error mode should throw on first redirect response' + ); + }); +}); diff --git a/src/test/helpers/local-server.ts b/src/test/helpers/local-server.ts new file mode 100644 index 0000000..2738b3c --- /dev/null +++ b/src/test/helpers/local-server.ts @@ -0,0 +1,234 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; +import { after, before } from 'node:test'; +import { WebSocketServer, type WebSocket as WsPeer } from 'ws'; + +export function onceEvent(target: EventTarget, type: string): Promise { + return new Promise((resolve) => { + const listener = (event: Event) => { + resolve(event as T); + }; + + target.addEventListener(type, listener, { once: true }); + }); +} + +export function setupLocalTestServer() { + let localBaseUrl = ''; + let localServer: Server | undefined; + let wsServer: WebSocketServer | undefined; + const retryAttempts = new Map(); + + function readCookieHeader(request: IncomingMessage): string { + const cookie = request.headers.cookie; + + if (Array.isArray(cookie)) { + return cookie.join('; '); + } + + return cookie ?? ''; + } + + function sendJson( + response: ServerResponse, + status: number, + body: unknown, + headers?: Record + ) { + response.writeHead(status, { + 'content-type': 'application/json', + ...headers, + }); + response.end(JSON.stringify(body)); + } + + before(async () => { + wsServer = new WebSocketServer({ + noServer: true, + perMessageDeflate: true, + handleProtocols(protocols: Set) { + if (protocols.has('chat')) { + return 'chat'; + } + + return false; + }, + }); + + wsServer.on('connection', (socket: WsPeer, request: IncomingMessage) => { + const cookie = readCookieHeader(request); + + socket.send( + JSON.stringify({ + kind: 'connected', + cookie, + protocol: socket.protocol, + rawHeaders: request.rawHeaders, + }) + ); + + socket.on('message', (data: Buffer, isBinary: boolean) => { + if (!isBinary && data.toString() === 'close-me') { + socket.close(1000, 'done'); + + return; + } + + socket.send(data, { binary: isBinary }); + }); + }); + + localServer = createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + + if (url.pathname === '/retry') { + const key = url.searchParams.get('key') ?? 'default'; + const failCount = Number(url.searchParams.get('failCount') ?? '0'); + const count = (retryAttempts.get(key) ?? 0) + 1; + + retryAttempts.set(key, count); + + if (count <= failCount) { + sendJson(response, 503, { attempt: count, retried: false }); + + return; + } + + sendJson(response, 200, { attempt: count, retried: count > 1 }); + + return; + } + + if (url.pathname === '/timings/delay') { + setTimeout(() => { + sendJson(response, 200, { delayed: true }); + }, 50); + + return; + } + + if (url.pathname === '/cookies/set') { + sendJson( + response, + 200, + { stored: true }, + { + 'set-cookie': 'session=abc123', + } + ); + + return; + } + + if (url.pathname === '/cookies/set-multiple') { + sendJson( + response, + 200, + { stored: true }, + { + 'set-cookie': ['session=abc123; Path=/', 'csrf=token123; Path=/'], + } + ); + + return; + } + + if (url.pathname === '/cookies/echo') { + sendJson(response, 200, { cookie: readCookieHeader(request) }); + + return; + } + + if (url.pathname === '/headers/raw') { + sendJson(response, 200, { + rawHeaders: request.rawHeaders, + headers: request.headers, + }); + + return; + } + + if (url.pathname === '/redirect/start') { + response.writeHead(302, { + location: '/redirect/final', + 'set-cookie': 'redirect_session=1; Path=/', + }); + response.end(); + + return; + } + + if (url.pathname === '/redirect/post-start') { + response.writeHead(302, { + location: '/redirect/final', + }); + response.end(); + + return; + } + + if (url.pathname === '/redirect/final') { + sendJson(response, 200, { + method: request.method, + cookie: readCookieHeader(request), + hookHeader: request.headers['x-redirect-hook'] ?? '', + }); + + return; + } + + sendJson(response, 404, { path: url.pathname }); + }); + + localServer.on('upgrade', (request, socket, head) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + + if (url.pathname !== '/ws') { + socket.destroy(); + + return; + } + + wsServer?.handleUpgrade(request, socket, head, (websocketSocket: WsPeer) => { + wsServer?.emit('connection', websocketSocket, request); + }); + }); + + await new Promise((resolve) => { + localServer?.listen(0, '127.0.0.1', () => { + const address = localServer?.address(); + + if (!address || typeof address === 'string') { + throw new Error('Failed to bind local test server'); + } + + localBaseUrl = `http://127.0.0.1:${address.port}`; + resolve(); + }); + }); + }); + + after(async () => { + await new Promise((resolve) => { + wsServer?.close(() => resolve()); + }); + + await new Promise((resolve, reject) => { + localServer?.close((error) => { + if (error) { + reject(error); + + return; + } + + resolve(); + }); + }); + }); + + return { + getBaseUrl() { + return localBaseUrl; + }, + retryAttempts, + }; +} diff --git a/src/test/hooks-retries.spec.ts b/src/test/hooks-retries.spec.ts new file mode 100644 index 0000000..2d906c5 --- /dev/null +++ b/src/test/hooks-retries.spec.ts @@ -0,0 +1,143 @@ +import assert from 'node:assert'; +import { describe, test } from 'node:test'; +import { Response as WreqResponse, fetch } from '../node-wreq'; +import { setupLocalTestServer } from './helpers/local-server'; + +describe('hooks and retries', () => { + const { getBaseUrl, retryAttempts } = setupLocalTestServer(); + + test('should run init and beforeRequest hooks', async () => { + const response = await fetch('https://httpbin.org/anything', { + browser: 'chrome_137', + timeout: 15000, + hooks: { + init: [ + ({ options }) => { + options.query = { from: 'init-hook' }; + }, + ], + beforeRequest: [ + ({ request }) => { + request.headers.set('X-Hook-Header', 'active'); + }, + ], + }, + }); + + const body = await response.json<{ + args: Record; + headers: Record; + }>(); + + assert.strictEqual(body.args.from, 'init-hook', 'Init hook should mutate query'); + assert.strictEqual( + body.headers['X-Hook-Header'], + 'active', + 'beforeRequest should mutate headers' + ); + }); + + test('should allow afterResponse to replace the response', async () => { + const response = await fetch('https://httpbin.org/status/201', { + browser: 'chrome_137', + timeout: 15000, + hooks: { + afterResponse: [ + () => + new WreqResponse(JSON.stringify({ replaced: true }), { + status: 299, + headers: { 'content-type': 'application/json' }, + url: 'https://local/replaced', + }), + ], + }, + }); + + assert.strictEqual(response.status, 299, 'afterResponse should replace the response'); + const body = await response.json<{ replaced: boolean }>(); + + assert.strictEqual(body.replaced, true, 'Replaced response body should be returned'); + }); + + test('should allow beforeError to rewrite thrown errors', async () => { + await assert.rejects( + async () => { + await fetch('https://httpbin.org/status/418', { + browser: 'chrome_137', + timeout: 15000, + throwHttpErrors: true, + hooks: { + beforeError: [ + ({ error }) => { + error.message = `hooked: ${error.message}`; + + return error; + }, + ], + }, + }); + }, + (error: unknown) => + error instanceof Error && error.message.includes('hooked: Request failed with status 418'), + 'beforeError should be able to rewrite the final error' + ); + }); + + test('should run beforeRetry hooks and retry retriable responses', async () => { + retryAttempts.set('status-retry', 0); + const hookAttempts: number[] = []; + const response = await fetch(`${getBaseUrl()}/retry?key=status-retry&failCount=2`, { + retry: { + limit: 2, + statusCodes: [503], + backoff: () => 0, + }, + hooks: { + beforeRetry: [ + ({ attempt, response: retryResponse }) => { + hookAttempts.push(attempt); + assert.strictEqual(retryResponse?.status, 503); + }, + ], + }, + }); + + assert.strictEqual(response.status, 200, 'request should eventually succeed'); + assert.deepStrictEqual( + hookAttempts, + [2, 3], + 'beforeRetry should run before each retry attempt' + ); + + const body = await response.json<{ attempt: number; retried: boolean }>(); + + assert.strictEqual(body.attempt, 3, 'third attempt should be the successful response'); + assert.strictEqual(body.retried, true, 'server should observe retries'); + }); + + test('should expose response timings and onStats callback data', async () => { + let capturedStats: + | { + attempt: number; + wait: number; + status?: number; + } + | undefined; + + const response = await fetch(`${getBaseUrl()}/timings/delay`, { + onStats: ({ attempt, timings, response: statsResponse }) => { + capturedStats = { + attempt, + wait: timings.wait, + status: statsResponse?.status, + }; + }, + }); + + assert.ok(response.wreq.timings, 'response should expose timings'); + assert.ok((response.wreq.timings?.wait ?? 0) >= 20, 'timings should record server wait time'); + assert.strictEqual(capturedStats?.attempt, 1); + assert.strictEqual(capturedStats?.status, 200); + assert.ok((capturedStats?.wait ?? 0) >= 20, 'onStats should receive request timings'); + }); +}); diff --git a/src/test/http-client.spec.ts b/src/test/http-client.spec.ts new file mode 100644 index 0000000..99ffa8e --- /dev/null +++ b/src/test/http-client.spec.ts @@ -0,0 +1,166 @@ +import assert from 'node:assert'; +import { describe, test } from 'node:test'; +import { Request as WreqRequest, createClient, fetch, getProfiles } from '../node-wreq'; +import { setupLocalTestServer } from './helpers/local-server'; + +describe('http client', () => { + const { getBaseUrl } = setupLocalTestServer(); + + test('should return available browser profiles', () => { + const profiles = getProfiles(); + + assert.ok(Array.isArray(profiles), 'Profiles should be an array'); + assert.ok(profiles.length > 0, 'Should have at least one profile'); + assert.ok( + profiles.includes('chrome_137') || + profiles.includes('firefox_139') || + profiles.includes('safari_18'), + 'Should include standard browser profiles' + ); + }); + + test('should make a simple GET request', async () => { + const response = await fetch('https://httpbin.org/get', { + browser: 'chrome_131', + timeout: 15000, + }); + + assert.ok(response.status >= 200 && response.status < 300, 'Should return successful status'); + assert.ok(Object.keys(response.headers).length > 0, 'Should have response headers'); + + const body = await response.json<{ headers: Record }>(); + + assert.ok(body.headers['User-Agent'], 'Should have User-Agent header'); + }); + + test('should work with different browser profiles', async () => { + const testUrl = 'https://httpbin.org/user-agent'; + const browsers = ['chrome_137', 'firefox_139', 'safari_18']; + + for (const browser of browsers) { + const response = await fetch(testUrl, { + browser: browser as any, + timeout: 30000, + }); + + assert.ok(response.status === 200, `${browser} should return status 200`); + + const data = await response.json>(); + + assert.ok(data['user-agent'], `${browser} should have user-agent`); + } + }); + + test('should handle timeout errors', async () => { + await assert.rejects( + async () => { + await fetch('https://httpbin.org/delay/10', { + browser: 'chrome_137', + timeout: 1000, + }); + }, + { + name: 'TimeoutError', + }, + 'Should throw an error on timeout' + ); + }); + + test('should support fetch-style requests', async () => { + const response = await fetch('https://httpbin.org/get', { + browser: 'chrome_137', + query: { source: 'fetch' }, + throwHttpErrors: true, + timeout: 15000, + }); + + assert.strictEqual(response.ok, true, 'Fetch response should expose ok'); + + const body = await response.json<{ args: Record }>(); + + assert.strictEqual(body.args.source, 'fetch', 'Should apply query params'); + }); + + test('should support createClient defaults', async () => { + const client = createClient({ + browser: 'chrome_137', + baseURL: 'https://httpbin.org', + timeout: 15000, + headers: { + 'X-Test-Client': 'node-wreq', + }, + }); + + const response = await client.get('/headers'); + const body = await response.json<{ headers: Record }>(); + + assert.strictEqual( + body.headers['X-Test-Client'], + 'node-wreq', + 'Client defaults should be merged into outgoing requests' + ); + }); + + test('should preserve ordered header tuples and original header names when requested', async () => { + const response = await fetch(`${getBaseUrl()}/headers/raw`, { + browser: 'chrome_137', + disableDefaultHeaders: true, + keepOriginalHeaderNames: true, + headers: [ + ['x-lower', 'one'], + ['X-Mixed', 'two'], + ], + }); + + const body = await response.json<{ rawHeaders: string[] }>(); + const lowerIndex = body.rawHeaders.indexOf('x-lower'); + const mixedIndex = body.rawHeaders.indexOf('X-Mixed'); + + assert.ok(lowerIndex >= 0, 'raw headers should include original lowercase name'); + assert.ok(mixedIndex >= 0, 'raw headers should include original mixed-case name'); + assert.ok(lowerIndex < mixedIndex, 'tuple header order should be preserved'); + assert.strictEqual(body.rawHeaders[lowerIndex + 1], 'one'); + assert.strictEqual(body.rawHeaders[mixedIndex + 1], 'two'); + }); + + test('should accept advanced emulation options', async () => { + const response = await fetch(`${getBaseUrl()}/headers/raw`, { + browser: 'chrome_137', + tlsOptions: { + greaseEnabled: true, + }, + http1Options: { + writev: true, + }, + http2Options: { + adaptiveWindow: false, + maxConcurrentStreams: 64, + }, + }); + + assert.strictEqual(response.status, 200); + }); + + test('should support native-like Request instances', async () => { + const request = new WreqRequest('https://httpbin.org/anything', { + method: 'POST', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify({ via: 'request' }), + }); + + assert.strictEqual(request.method, 'POST'); + assert.strictEqual(await request.clone().text(), JSON.stringify({ via: 'request' })); + + const response = await fetch(request, { + browser: 'chrome_137', + timeout: 15000, + }); + + const body = await response.json<{ method: string; data: string }>(); + + assert.strictEqual(body.method, 'POST'); + assert.strictEqual(body.data, JSON.stringify({ via: 'request' })); + }); +}); diff --git a/src/test/node-wreq.spec.ts b/src/test/node-wreq.spec.ts index 8445ee5..b9976b4 100644 --- a/src/test/node-wreq.spec.ts +++ b/src/test/node-wreq.spec.ts @@ -1,841 +1,5 @@ -import assert from 'node:assert'; -import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; -import { test, describe, before, after } from 'node:test'; -import { TextDecoder } from 'node:util'; -import { WebSocketServer, type WebSocket as WsPeer } from 'ws'; -import { - CloseEvent as WreqCloseEvent, - Response as WreqResponse, - WebSocket as WreqWebSocket, - createClient, - fetch, - getProfiles, - websocket, -} from '../node-wreq'; - -describe('node-wreq', () => { - let localBaseUrl = ''; - let localServer: Server | undefined; - let wsServer: WebSocketServer | undefined; - const retryAttempts = new Map(); - - function onceEvent(target: EventTarget, type: string): Promise { - return new Promise((resolve) => { - const listener = (event: Event) => { - resolve(event as T); - }; - - target.addEventListener(type, listener, { once: true }); - }); - } - - function readCookieHeader(request: IncomingMessage): string { - const cookie = request.headers.cookie; - if (Array.isArray(cookie)) { - return cookie.join('; '); - } - - return cookie ?? ''; - } - - function sendJson( - response: ServerResponse, - status: number, - body: unknown, - headers?: Record - ) { - response.writeHead(status, { - 'content-type': 'application/json', - ...headers, - }); - response.end(JSON.stringify(body)); - } - - before(() => { - console.log('🚀 node-wreq - Test Suite\n'); - }); - - before(async () => { - wsServer = new WebSocketServer({ - noServer: true, - perMessageDeflate: true, - handleProtocols(protocols: Set) { - if (protocols.has('chat')) { - return 'chat'; - } - - return false; - }, - }); - - wsServer.on('connection', (socket: WsPeer, request: IncomingMessage) => { - const cookie = readCookieHeader(request); - socket.send(JSON.stringify({ kind: 'connected', cookie, protocol: socket.protocol })); - - socket.on('message', (data: Buffer, isBinary: boolean) => { - if (!isBinary && data.toString() === 'close-me') { - socket.close(1000, 'done'); - return; - } - - socket.send(data, { binary: isBinary }); - }); - }); - - localServer = createServer((request, response) => { - const url = new URL(request.url ?? '/', 'http://127.0.0.1'); - - if (url.pathname === '/retry') { - const key = url.searchParams.get('key') ?? 'default'; - const failCount = Number(url.searchParams.get('failCount') ?? '0'); - const count = (retryAttempts.get(key) ?? 0) + 1; - retryAttempts.set(key, count); - - if (count <= failCount) { - sendJson(response, 503, { attempt: count, retried: false }); - return; - } - - sendJson(response, 200, { attempt: count, retried: count > 1 }); - return; - } - - if (url.pathname === '/timings/delay') { - setTimeout(() => { - sendJson(response, 200, { delayed: true }); - }, 50); - return; - } - - if (url.pathname === '/cookies/set') { - sendJson( - response, - 200, - { stored: true }, - { - 'set-cookie': 'session=abc123', - } - ); - return; - } - - if (url.pathname === '/cookies/set-multiple') { - sendJson( - response, - 200, - { stored: true }, - { - 'set-cookie': ['session=abc123; Path=/', 'csrf=token123; Path=/'], - } - ); - return; - } - - if (url.pathname === '/cookies/echo') { - sendJson(response, 200, { cookie: readCookieHeader(request) }); - return; - } - - if (url.pathname === '/redirect/start') { - response.writeHead(302, { - location: '/redirect/final', - 'set-cookie': 'redirect_session=1; Path=/', - }); - response.end(); - return; - } - - if (url.pathname === '/redirect/post-start') { - response.writeHead(302, { - location: '/redirect/final', - }); - response.end(); - return; - } - - if (url.pathname === '/redirect/final') { - sendJson(response, 200, { - method: request.method, - cookie: readCookieHeader(request), - hookHeader: request.headers['x-redirect-hook'] ?? '', - }); - return; - } - - sendJson(response, 404, { path: url.pathname }); - }); - - localServer.on('upgrade', (request, socket, head) => { - const url = new URL(request.url ?? '/', 'http://127.0.0.1'); - if (url.pathname !== '/ws') { - socket.destroy(); - return; - } - - wsServer?.handleUpgrade(request, socket, head, (websocketSocket: WsPeer) => { - wsServer?.emit('connection', websocketSocket, request); - }); - }); - - await new Promise((resolve) => { - localServer?.listen(0, '127.0.0.1', () => { - const address = localServer?.address(); - if (!address || typeof address === 'string') { - throw new Error('Failed to bind local test server'); - } - - localBaseUrl = `http://127.0.0.1:${address.port}`; - resolve(); - }); - }); - }); - - after(async () => { - await new Promise((resolve) => { - wsServer?.close(() => resolve()); - }); - - await new Promise((resolve, reject) => { - localServer?.close((error) => { - if (error) { - reject(error); - return; - } - - resolve(); - }); - }); - }); - - test('should return available browser profiles', () => { - const profiles = getProfiles(); - - assert.ok(Array.isArray(profiles), 'Profiles should be an array'); - assert.ok(profiles.length > 0, 'Should have at least one profile'); - assert.ok( - profiles.includes('chrome_137') || - profiles.includes('firefox_139') || - profiles.includes('safari_18'), - 'Should include standard browser profiles' - ); - - console.log(' Available profiles:', profiles.join(', ')); - }); - - test('should make a simple GET request', async () => { - const response = await fetch('https://httpbin.org/get', { - browser: 'chrome_131', - timeout: 15000, - }); - - assert.ok(response.status >= 200 && response.status < 300, 'Should return successful status'); - assert.ok(Object.keys(response.headers).length > 0, 'Should have response headers'); - - const body = await response.json<{ headers: Record }>(); - assert.ok(body.headers['User-Agent'], 'Should have User-Agent header'); - - console.log(' Status:', response.status); - console.log(' User-Agent:', body.headers['User-Agent']); - }); - - test('should work with different browser profiles', async () => { - const testUrl = 'https://httpbin.org/user-agent'; - const browsers = ['chrome_137', 'firefox_139', 'safari_18']; - - for (const browser of browsers) { - const response = await fetch(testUrl, { - browser: browser as any, - timeout: 30000, - }); - - assert.ok(response.status === 200, `${browser} should return status 200`); - - const data = await response.json>(); - assert.ok(data['user-agent'], `${browser} should have user-agent`); - - console.log(` ${browser}:`, data['user-agent'].substring(0, 50) + '...'); - } - }); - - test('should handle timeout errors', async () => { - await assert.rejects( - async () => { - await fetch('https://httpbin.org/delay/10', { - browser: 'chrome_137', - timeout: 1000, // 1 second timeout for 10 second delay - }); - }, - { - name: 'TimeoutError', - }, - 'Should throw an error on timeout' - ); - }); - - test('should support fetch-style requests', async () => { - const response = await fetch('https://httpbin.org/get', { - browser: 'chrome_137', - query: { source: 'fetch' }, - throwHttpErrors: true, - timeout: 15000, - }); - - assert.strictEqual(response.ok, true, 'Fetch response should expose ok'); - const body = await response.json<{ args: Record }>(); - assert.strictEqual(body.args.source, 'fetch', 'Should apply query params'); - }); - - test('should support createClient defaults', async () => { - const client = createClient({ - browser: 'chrome_137', - baseURL: 'https://httpbin.org', - timeout: 15000, - headers: { - 'X-Test-Client': 'node-wreq', - }, - }); - - const response = await client.get('/headers'); - const body = await response.json<{ headers: Record }>(); - - assert.strictEqual( - body.headers['X-Test-Client'], - 'node-wreq', - 'Client defaults should be merged into outgoing requests' - ); - }); - - test('should run init and beforeRequest hooks', async () => { - const response = await fetch('https://httpbin.org/anything', { - browser: 'chrome_137', - timeout: 15000, - hooks: { - init: [ - ({ options }) => { - options.query = { from: 'init-hook' }; - }, - ], - beforeRequest: [ - ({ request }) => { - request.headers.set('X-Hook-Header', 'active'); - }, - ], - }, - }); - - const body = await response.json<{ - args: Record; - headers: Record; - }>(); - - assert.strictEqual(body.args.from, 'init-hook', 'Init hook should mutate query'); - assert.strictEqual( - body.headers['X-Hook-Header'], - 'active', - 'beforeRequest should mutate headers' - ); - }); - - test('should allow afterResponse to replace the response', async () => { - const response = await fetch('https://httpbin.org/status/201', { - browser: 'chrome_137', - timeout: 15000, - hooks: { - afterResponse: [ - () => - new WreqResponse({ - status: 299, - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ replaced: true }), - cookies: {}, - url: 'https://local/replaced', - }), - ], - }, - }); - - assert.strictEqual(response.status, 299, 'afterResponse should replace the response'); - const body = await response.json<{ replaced: boolean }>(); - assert.strictEqual(body.replaced, true, 'Replaced response body should be returned'); - }); - - test('should expose fetch-style response body lifecycle', async () => { - const response = new WreqResponse({ - status: 200, - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ streamed: true }), - cookies: {}, - setCookies: [], - url: 'https://local/body', - }); - - assert.strictEqual(response.bodyUsed, false, 'body should start unused'); - - const bodyStream = response.body; - assert.ok(bodyStream, 'body should expose a stream'); - assert.strictEqual(response.bodyUsed, true, 'accessing body should mark it used'); - - const reader = bodyStream?.getReader(); - const chunks: Uint8Array[] = []; - - while (reader) { - const result = await reader.read(); - if (result.done) { - break; - } - - chunks.push(result.value); - } - - const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); - assert.strictEqual( - new TextDecoder().decode(merged), - JSON.stringify({ streamed: true }), - 'body stream should contain the response payload' - ); - - await assert.rejects( - async () => { - await response.text(); - }, - (error: unknown) => error instanceof TypeError && error.message.includes('already used'), - 'consumers should reject after the body is used' - ); - }); - - test('should support cloning unused responses', async () => { - const response = new WreqResponse({ - status: 200, - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ cloned: true }), - cookies: {}, - setCookies: [], - url: 'https://local/clone', - }); - - const cloned = response.clone(); - assert.notStrictEqual(cloned, response, 'clone should return a new response instance'); - assert.strictEqual(await cloned.text(), JSON.stringify({ cloned: true })); - assert.strictEqual(await response.text(), JSON.stringify({ cloned: true })); - }); - - test('should expose native-backed response streams for fetched responses', async () => { - const response = await fetch(`${localBaseUrl}/cookies/echo`); - const stream = response.body; - - assert.ok(stream, 'fetched response should expose a body stream'); - assert.strictEqual(response.bodyUsed, true, 'reading body stream should mark it used'); - - const reader = stream?.getReader(); - const chunks: Uint8Array[] = []; - while (reader) { - const result = await reader.read(); - if (result.done) { - break; - } - - chunks.push(result.value); - } - - const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); - assert.ok( - new TextDecoder().decode(merged).includes('"cookie":""'), - 'streamed native response should contain the expected payload' - ); - }); - - test('should support formData for urlencoded responses', async () => { - const response = new WreqResponse({ - status: 200, - headers: { 'content-type': 'application/x-www-form-urlencoded; charset=utf-8' }, - body: 'alpha=1&beta=two&beta=three', - cookies: {}, - setCookies: [], - url: 'https://local/form', - }); - - const formData = await response.formData(); - assert.strictEqual(formData.get('alpha'), '1'); - assert.deepStrictEqual(formData.getAll('beta'), ['two', 'three']); - assert.strictEqual(response.bodyUsed, true, 'formData should consume the body'); - }); - - test('should reject unsupported multipart formData parsing for now', async () => { - const response = new WreqResponse({ - status: 200, - headers: { 'content-type': 'multipart/form-data; boundary=test' }, - body: '--test\r\ncontent-disposition: form-data; name="alpha"\r\n\r\n1\r\n--test--\r\n', - cookies: {}, - setCookies: [], - url: 'https://local/form', - }); - - await assert.rejects( - async () => { - await response.formData(); - }, - (error: unknown) => - error instanceof TypeError && error.message.includes('multipart/form-data parsing'), - 'multipart form parsing should fail explicitly until it is implemented' - ); - }); - - test('should allow beforeError to rewrite thrown errors', async () => { - await assert.rejects( - async () => { - await fetch('https://httpbin.org/status/418', { - browser: 'chrome_137', - timeout: 15000, - throwHttpErrors: true, - hooks: { - beforeError: [ - ({ error }) => { - error.message = `hooked: ${error.message}`; - return error; - }, - ], - }, - }); - }, - (error: unknown) => - error instanceof Error && error.message.includes('hooked: Request failed with status 418'), - 'beforeError should be able to rewrite the final error' - ); - }); - - test('should run beforeRetry hooks and retry retriable responses', async () => { - retryAttempts.set('status-retry', 0); - const hookAttempts: number[] = []; - - const response = await fetch(`${localBaseUrl}/retry?key=status-retry&failCount=2`, { - retry: { - limit: 2, - statusCodes: [503], - backoff: () => 0, - }, - hooks: { - beforeRetry: [ - ({ attempt, response: retryResponse }) => { - hookAttempts.push(attempt); - assert.strictEqual(retryResponse?.status, 503); - }, - ], - }, - }); - - assert.strictEqual(response.status, 200, 'request should eventually succeed'); - assert.deepStrictEqual( - hookAttempts, - [2, 3], - 'beforeRetry should run before each retry attempt' - ); - - const body = await response.json<{ attempt: number; retried: boolean }>(); - assert.strictEqual(body.attempt, 3, 'third attempt should be the successful response'); - assert.strictEqual(body.retried, true, 'server should observe retries'); - }); - - test('should integrate cookieJar across requests', async () => { - const jarStore = new Map(); - const cookieJar = { - getCookies: () => - [...jarStore.entries()].map(([name, value]) => ({ - name, - value, - })), - setCookie: (cookie: string) => { - const [pair] = cookie.split(';'); - const [name, value = ''] = pair.split('='); - jarStore.set(name, value); - }, - }; - - const setResponse = await fetch(`${localBaseUrl}/cookies/set`, { - cookieJar, - }); - assert.strictEqual(setResponse.status, 200, 'cookie source request should succeed'); - assert.strictEqual( - jarStore.get('session'), - 'abc123', - 'cookieJar should persist response cookies' - ); - - const echoResponse = await fetch(`${localBaseUrl}/cookies/echo`, { - cookieJar, - }); - const body = await echoResponse.json<{ cookie: string }>(); - - assert.ok( - body.cookie.includes('session=abc123'), - 'stored cookies should be sent on the next request' - ); - }); - - test('should persist multiple raw set-cookie headers through cookieJar', async () => { - const rawCookies: string[] = []; - const cookieJar = { - getCookies: () => - rawCookies.map((cookie) => { - const [pair] = cookie.split(';'); - const [name, value = ''] = pair.split('='); - return { name, value }; - }), - setCookie: (cookie: string) => { - rawCookies.push(cookie); - }, - }; - - const response = await fetch(`${localBaseUrl}/cookies/set-multiple`, { - cookieJar, - }); - - assert.strictEqual(response.status, 200, 'multiple set-cookie response should succeed'); - assert.deepStrictEqual( - rawCookies, - ['session=abc123; Path=/', 'csrf=token123; Path=/'], - 'cookieJar should receive each raw Set-Cookie value separately' - ); - - const echoResponse = await fetch(`${localBaseUrl}/cookies/echo`, { - cookieJar, - }); - const body = await echoResponse.json<{ cookie: string }>(); - - assert.ok(body.cookie.includes('session=abc123'), 'first cookie should be replayed'); - assert.ok(body.cookie.includes('csrf=token123'), 'second cookie should be replayed'); - }); - - test('should follow redirects manually and run beforeRedirect hooks', async () => { - const cookieJar = { - store: new Map(), - getCookies(this: { store: Map }) { - return [...this.store.entries()].map(([name, value]) => ({ name, value })); - }, - setCookie(this: { store: Map }, cookie: string) { - const [pair] = cookie.split(';'); - const [name, value = ''] = pair.split('='); - this.store.set(name, value); - }, - }; - - const response = await fetch(`${localBaseUrl}/redirect/start`, { - cookieJar, - hooks: { - beforeRedirect: [ - ({ request, redirectCount, nextUrl }) => { - assert.strictEqual(redirectCount, 1); - assert.strictEqual(nextUrl, `${localBaseUrl}/redirect/final`); - request.headers.set('X-Redirect-Hook', 'active'); - }, - ], - }, - }); - - assert.strictEqual(response.status, 200, 'redirect chain should resolve to final response'); - assert.strictEqual(response.redirected, true, 'final response should be marked redirected'); - assert.strictEqual(response.redirectChain.length, 1, 'redirect chain should be recorded'); - - const body = await response.json<{ method: string; cookie: string; hookHeader: string }>(); - assert.strictEqual(body.method, 'GET', 'GET redirect should preserve method'); - assert.ok( - body.cookie.includes('redirect_session=1'), - 'intermediate set-cookie should affect the next redirect hop' - ); - assert.strictEqual( - body.hookHeader, - 'active', - 'beforeRedirect should be able to mutate next request' - ); - }); - - test('should rewrite POST to GET on 302 redirects', async () => { - const response = await fetch(`${localBaseUrl}/redirect/post-start`, { - method: 'POST', - body: 'hello', - }); - - const body = await response.json<{ method: string }>(); - assert.strictEqual(body.method, 'GET', '302 redirect from POST should be rewritten to GET'); - }); - - test('should support manual redirect mode', async () => { - const response = await fetch(`${localBaseUrl}/redirect/start`, { - redirect: 'manual', - }); - - assert.strictEqual( - response.status, - 302, - 'manual redirect mode should return the redirect response' - ); - assert.strictEqual( - response.headers.get('location'), - '/redirect/final', - 'manual redirect mode should expose Location' - ); - assert.strictEqual( - response.redirected, - false, - 'manual redirect response should not be marked redirected' - ); - }); - - test('should support redirect error mode', async () => { - await assert.rejects( - async () => { - await fetch(`${localBaseUrl}/redirect/start`, { - redirect: 'error', - }); - }, - (error: unknown) => - error instanceof Error && - error.message.includes('Redirect encountered') && - 'code' in (error as object), - 'redirect error mode should throw on first redirect response' - ); - }); - - test('should expose a WHATWG-like websocket helper and lifecycle', async () => { - const socket = await websocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { - protocols: 'chat', - }); - - assert.strictEqual(socket.readyState, WreqWebSocket.OPEN); - assert.strictEqual(socket.protocol, 'chat'); - - const connectedEvent = await onceEvent(socket, 'message'); - const connected = JSON.parse(String(connectedEvent.data)) as { - kind: string; - cookie: string; - protocol: string; - }; - assert.strictEqual(connected.kind, 'connected'); - assert.strictEqual(connected.protocol, 'chat'); - - const replyPromise = onceEvent(socket, 'message'); - socket.send('hello'); - const replyEvent = await replyPromise; - assert.strictEqual(replyEvent.data, 'hello'); - - const closePromise = onceEvent(socket, 'close'); - socket.close(1000, 'done'); - const closeEvent = await closePromise; - - assert.strictEqual(closeEvent.code, 1000); - assert.strictEqual(closeEvent.reason, 'done'); - assert.strictEqual(closeEvent.wasClean, true); - assert.strictEqual(socket.readyState, WreqWebSocket.CLOSED); - }); - - test('should support binary messages and arraybuffer binaryType', async () => { - const socket = new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { - binaryType: 'arraybuffer', - }); - - await onceEvent(socket, 'open'); - await onceEvent(socket, 'message'); - - const replyPromise = onceEvent(socket, 'message'); - socket.send(new Uint8Array([1, 2, 3])); - const replyEvent = await replyPromise; - - assert.ok(replyEvent.data instanceof ArrayBuffer); - assert.deepStrictEqual([...new Uint8Array(replyEvent.data)], [1, 2, 3]); - - const closePromise = onceEvent(socket, 'close'); - socket.close(1000, 'done'); - await closePromise; - }); - - test('should send cookieJar cookies during websocket handshake', async () => { - const cookieJar = { - getCookies: () => [{ name: 'session', value: 'ws123' }], - setCookie: () => {}, - }; - - const socket = await websocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { - cookieJar, - }); - - const connectedEvent = await onceEvent(socket, 'message'); - const connected = JSON.parse(String(connectedEvent.data)) as { cookie: string }; - assert.ok( - connected.cookie.includes('session=ws123'), - 'cookieJar cookies should be sent during the websocket handshake' - ); - - const closePromise = onceEvent(socket, 'close'); - socket.close(1000, 'done'); - await closePromise; - }); - - test('should reject websocket URLs with fragments', () => { - assert.throws( - () => { - new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws#fragment'); - }, - (error: unknown) => error instanceof DOMException && error.name === 'SyntaxError', - 'fragment websocket URLs should be rejected' - ); - }); - - test('should reject forbidden websocket headers and duplicate protocols', () => { - assert.throws( - () => { - new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { - headers: { - Upgrade: 'websocket', - }, - }); - }, - (error: unknown) => error instanceof DOMException && error.name === 'SyntaxError', - 'forbidden managed websocket headers should be rejected' - ); - - assert.throws( - () => { - new WreqWebSocket(localBaseUrl.replace('http://', 'ws://') + '/ws', { - protocols: ['chat', 'chat'], - }); - }, - (error: unknown) => - error instanceof SyntaxError && error.message.includes('Duplicate WebSocket subprotocol'), - 'duplicate websocket subprotocols should be rejected' - ); - }); - - test('should expose negotiated websocket extensions as a string', async () => { - const socket = await websocket(localBaseUrl.replace('http://', 'ws://') + '/ws'); - assert.strictEqual(typeof socket.extensions, 'string'); - - const closePromise = onceEvent(socket, 'close'); - socket.close(1000, 'done'); - await closePromise; - }); - - test('should expose response timings and onStats callback data', async () => { - let capturedStats: - | { - attempt: number; - wait: number; - status?: number; - } - | undefined; - - const response = await fetch(`${localBaseUrl}/timings/delay`, { - onStats: ({ attempt, timings, response: statsResponse }) => { - capturedStats = { - attempt, - wait: timings.wait, - status: statsResponse?.status, - }; - }, - }); - - assert.ok(response.timings, 'response should expose timings'); - assert.ok((response.timings?.wait ?? 0) >= 20, 'timings should record server wait time'); - assert.strictEqual(capturedStats?.attempt, 1); - assert.strictEqual(capturedStats?.status, 200); - assert.ok((capturedStats?.wait ?? 0) >= 20, 'onStats should receive request timings'); - }); -}); +import './cookies-redirects.spec'; +import './hooks-retries.spec'; +import './http-client.spec'; +import './response.spec'; +import './websocket.spec'; diff --git a/src/test/response.spec.ts b/src/test/response.spec.ts new file mode 100644 index 0000000..9cccc29 --- /dev/null +++ b/src/test/response.spec.ts @@ -0,0 +1,298 @@ +import assert from 'node:assert'; +import { Buffer } from 'node:buffer'; +import { describe, test } from 'node:test'; +import { TextDecoder } from 'node:util'; +import { Response as WreqResponse, fetch } from '../node-wreq'; +import { setupLocalTestServer } from './helpers/local-server'; + +describe('response behavior', () => { + const { getBaseUrl } = setupLocalTestServer(); + + test('should expose fetch-style response body lifecycle', async () => { + const response = new WreqResponse(JSON.stringify({ streamed: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + url: 'https://local/body', + }); + + assert.strictEqual(response.bodyUsed, false, 'body should start unused'); + + const bodyStream = response.body; + + assert.ok(bodyStream, 'body should expose a stream'); + assert.strictEqual(response.bodyUsed, false, 'accessing body should not mark it used'); + + const reader = bodyStream?.getReader(); + const chunks: Uint8Array[] = []; + + while (reader) { + const result = await reader.read(); + + if (result.done) { + break; + } + + assert.strictEqual(response.bodyUsed, true, 'reading the stream should mark it used'); + chunks.push(result.value); + } + + const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + + assert.strictEqual( + new TextDecoder().decode(merged), + JSON.stringify({ streamed: true }), + 'body stream should contain the response payload' + ); + + await assert.rejects( + async () => { + await response.text(); + }, + (error: unknown) => error instanceof TypeError && error.message.includes('already been read'), + 'consumers should reject after the body is used' + ); + }); + + test('should support cloning buffered responses before they are read', async () => { + const response = new WreqResponse(JSON.stringify({ cloned: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + url: 'https://local/clone', + }); + + const initialBody = response.body; + + assert.ok(initialBody, 'response should expose a body stream before cloning'); + assert.strictEqual(response.bodyUsed, false, 'getting the body should not disturb it'); + + const cloned = response.clone(); + + assert.notStrictEqual(cloned, response, 'clone should return a new response instance'); + assert.notStrictEqual( + response.body, + initialBody, + 'clone should replace the original body branch' + ); + assert.throws( + () => initialBody?.getReader(), + (error: unknown) => error instanceof TypeError && error.message.includes('locked'), + 'the pre-clone body stream should no longer be readable after tee()' + ); + + const [left, right] = await Promise.all([response.text(), cloned.text()]); + + assert.strictEqual(left, JSON.stringify({ cloned: true })); + assert.strictEqual(right, JSON.stringify({ cloned: true })); + }); + + test('should reject clone and convenience readers while the body stream is locked', async () => { + const response = new WreqResponse('locked', { + status: 200, + url: 'https://local/locked', + }); + + const bodyStream = response.body; + + assert.ok(bodyStream, 'response should expose a body stream'); + + const reader = bodyStream.getReader(); + + assert.strictEqual(response.bodyUsed, false, 'locking the stream should not mark it used'); + + assert.throws( + () => response.clone(), + (error: unknown) => error instanceof TypeError && error.message.includes('consumed'), + 'clone should reject while the body stream is locked' + ); + + await assert.rejects( + async () => response.text(), + (error: unknown) => error instanceof TypeError && error.message.includes('already been read'), + 'convenience readers should reject while the body stream is locked' + ); + + reader.releaseLock(); + + const cloned = response.clone(); + + assert.strictEqual(await cloned.text(), 'locked'); + assert.strictEqual(await response.text(), 'locked'); + }); + + test('should reject cloning once the body stream has been disturbed', async () => { + const response = new WreqResponse('already used', { + status: 200, + url: 'https://local/already-used', + }); + + const reader = response.body?.getReader(); + + assert.ok(reader, 'response should expose a reader'); + + const firstChunk = await reader.read(); + + assert.strictEqual(firstChunk.done, false, 'stream should yield data'); + assert.strictEqual(response.bodyUsed, true, 'reading from the stream should disturb the body'); + + assert.throws( + () => response.clone(), + (error: unknown) => error instanceof TypeError && error.message.includes('consumed'), + 'clone should reject after the body has been disturbed' + ); + }); + + test('should expose native-backed response streams for fetched responses', async () => { + const response = await fetch(`${getBaseUrl()}/cookies/echo`); + const stream = response.body; + + assert.ok(stream, 'fetched response should expose a body stream'); + assert.strictEqual( + response.bodyUsed, + false, + 'accessing the fetched body should not mark it used' + ); + + const reader = stream?.getReader(); + const chunks: Uint8Array[] = []; + + while (reader) { + const result = await reader.read(); + + if (result.done) { + break; + } + + assert.strictEqual( + response.bodyUsed, + true, + 'reading the fetched body stream should mark it used' + ); + chunks.push(result.value); + } + + const merged = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))); + + assert.ok( + new TextDecoder().decode(merged).includes('"cookie":""'), + 'streamed native response should contain the expected payload' + ); + }); + + test('should support parallel clone consumption for native-backed streamed responses', async () => { + const response = await fetch(`${getBaseUrl()}/cookies/echo`); + const cloned = response.clone(); + const [left, right] = await Promise.all([response.text(), cloned.text()]); + + assert.strictEqual(left, right, 'both tee branches should observe the same payload'); + assert.ok( + left.includes('"cookie":""'), + 'the tee payload should match the fetched response body' + ); + assert.strictEqual( + response.bodyUsed, + true, + 'original response should be marked used after reading' + ); + assert.strictEqual( + cloned.bodyUsed, + true, + 'cloned response should be marked used after reading' + ); + }); + + test('should support formData for urlencoded responses', async () => { + const response = new WreqResponse('alpha=1&beta=two&beta=three', { + status: 200, + headers: { 'content-type': 'application/x-www-form-urlencoded; charset=utf-8' }, + url: 'https://local/form', + }); + + const formData = await response.formData(); + + assert.strictEqual(formData.get('alpha'), '1'); + assert.deepStrictEqual(formData.getAll('beta'), ['two', 'three']); + assert.strictEqual(response.bodyUsed, true, 'formData should consume the body'); + }); + + test('should support multipart formData responses with repeated fields and files', async () => { + const response = new WreqResponse( + '--XyZ123\r\n' + + 'Content-Disposition: form-data; name="alpha"\r\n' + + '\r\n' + + '1\r\n' + + '--XyZ123\r\n' + + 'Content-Disposition: form-data; name="beta"\r\n' + + '\r\n' + + 'two\r\n' + + '--XyZ123\r\n' + + 'Content-Disposition: form-data; name="beta"\r\n' + + '\r\n' + + 'three\r\n' + + '--XyZ123\r\n' + + 'Content-Disposition: form-data; name="upload"; filename="hello.txt"\r\n' + + 'Content-Type: text/plain\r\n' + + '\r\n' + + 'hello world\r\n' + + '--XyZ123--\r\n', + { + status: 200, + headers: { 'content-type': 'multipart/form-data; boundary="XyZ123"' }, + url: 'https://local/form', + } + ); + + const formData = await response.formData(); + const upload = formData.get('upload'); + + assert.strictEqual(formData.get('alpha'), '1'); + assert.deepStrictEqual(formData.getAll('beta'), ['two', 'three']); + assert.ok(upload instanceof Blob, 'file part should be represented as a Blob/File'); + assert.strictEqual((upload as File).name, 'hello.txt'); + assert.strictEqual(upload.type, 'text/plain'); + assert.strictEqual(await upload.text(), 'hello world'); + assert.strictEqual(response.bodyUsed, true, 'multipart formData should consume the body'); + }); + + test('should reject multipart formData responses without a boundary', async () => { + const response = new WreqResponse('--test--\r\n', { + status: 200, + headers: { 'content-type': 'multipart/form-data' }, + url: 'https://local/form', + }); + + await assert.rejects( + async () => response.formData(), + (error: unknown) => + error instanceof TypeError && + error.message === 'Missing or invalid multipart/form-data boundary in Content-Type header', + 'multipart form parsing should fail when the boundary is missing' + ); + + assert.strictEqual( + response.bodyUsed, + true, + 'multipart parse errors should still consume the body' + ); + }); + + test('should reject malformed multipart formData responses with a clear error', async () => { + const response = new WreqResponse( + '--test\r\nContent-Disposition: form-data; name="alpha"\r\n\r\n1\r\n', + { + status: 200, + headers: { 'content-type': 'multipart/form-data; boundary=test' }, + url: 'https://local/form', + } + ); + + await assert.rejects( + async () => response.formData(), + (error: unknown) => + error instanceof TypeError && + error.message === 'Failed to parse multipart/form-data response body', + 'malformed multipart parsing should fail explicitly' + ); + + assert.strictEqual(response.bodyUsed, true, 'multipart parse failures should consume the body'); + }); +}); diff --git a/src/test/websocket.spec.ts b/src/test/websocket.spec.ts new file mode 100644 index 0000000..eb44835 --- /dev/null +++ b/src/test/websocket.spec.ts @@ -0,0 +1,177 @@ +import assert from 'node:assert'; +import { Buffer } from 'node:buffer'; +import { describe, test } from 'node:test'; +import { CloseEvent as WreqCloseEvent, WebSocket as WreqWebSocket, websocket } from '../node-wreq'; +import { onceEvent, setupLocalTestServer } from './helpers/local-server'; + +describe('websocket', () => { + const { getBaseUrl } = setupLocalTestServer(); + + test('should expose a WHATWG-like websocket helper and lifecycle', async () => { + const socket = await websocket(getBaseUrl().replace('http://', 'ws://') + '/ws', { + protocols: 'chat', + }); + + assert.strictEqual(socket.readyState, WreqWebSocket.OPEN); + assert.strictEqual(socket.protocol, 'chat'); + + const connectedEvent = await onceEvent(socket, 'message'); + const connected = JSON.parse(String(connectedEvent.data)) as { + kind: string; + cookie: string; + protocol: string; + }; + + assert.strictEqual(connected.kind, 'connected'); + assert.strictEqual(connected.protocol, 'chat'); + + const replyPromise = onceEvent(socket, 'message'); + + socket.send('hello'); + const replyEvent = await replyPromise; + + assert.strictEqual(replyEvent.data, 'hello'); + + const closePromise = onceEvent(socket, 'close'); + + socket.close(1000, 'done'); + const closeEvent = await closePromise; + + assert.strictEqual(closeEvent.code, 1000); + assert.strictEqual(closeEvent.reason, 'done'); + assert.strictEqual(closeEvent.wasClean, true); + assert.strictEqual(socket.readyState, WreqWebSocket.CLOSED); + }); + + test('should support binary messages and arraybuffer binaryType', async () => { + const socket = new WreqWebSocket(getBaseUrl().replace('http://', 'ws://') + '/ws', { + binaryType: 'arraybuffer', + }); + + await onceEvent(socket, 'open'); + await onceEvent(socket, 'message'); + + const replyPromise = onceEvent(socket, 'message'); + + socket.send(new Uint8Array([1, 2, 3])); + const replyEvent = await replyPromise; + + assert.ok(replyEvent.data instanceof ArrayBuffer); + assert.deepStrictEqual([...new Uint8Array(replyEvent.data)], [1, 2, 3]); + + const closePromise = onceEvent(socket, 'close'); + + socket.close(1000, 'done'); + await closePromise; + }); + + test('should send cookieJar cookies during websocket handshake', async () => { + const cookieJar = { + getCookies: () => [{ name: 'session', value: 'ws123' }], + setCookie: () => {}, + }; + + const socket = await websocket(getBaseUrl().replace('http://', 'ws://') + '/ws', { + cookieJar, + }); + + const connectedEvent = await onceEvent(socket, 'message'); + const connected = JSON.parse(String(connectedEvent.data)) as { cookie: string }; + + assert.ok( + connected.cookie.includes('session=ws123'), + 'cookieJar cookies should be sent during the websocket handshake' + ); + + const closePromise = onceEvent(socket, 'close'); + + socket.close(1000, 'done'); + await closePromise; + }); + + test('should reject websocket URLs with fragments', () => { + assert.throws( + () => { + new WreqWebSocket(getBaseUrl().replace('http://', 'ws://') + '/ws#fragment'); + }, + (error: unknown) => error instanceof DOMException && error.name === 'SyntaxError', + 'fragment websocket URLs should be rejected' + ); + }); + + test('should reject forbidden websocket headers and duplicate protocols', () => { + assert.throws( + () => { + new WreqWebSocket(getBaseUrl().replace('http://', 'ws://') + '/ws', { + headers: { + Upgrade: 'websocket', + }, + }); + }, + (error: unknown) => error instanceof DOMException && error.name === 'SyntaxError', + 'forbidden managed websocket headers should be rejected' + ); + + assert.throws( + () => { + new WreqWebSocket(getBaseUrl().replace('http://', 'ws://') + '/ws', { + protocols: ['chat', 'chat'], + }); + }, + (error: unknown) => + error instanceof SyntaxError && error.message.includes('Duplicate WebSocket subprotocol'), + 'duplicate websocket subprotocols should be rejected' + ); + }); + + test('should expose negotiated websocket extensions as a string', async () => { + const socket = await websocket(getBaseUrl().replace('http://', 'ws://') + '/ws'); + + assert.strictEqual(typeof socket.extensions, 'string'); + + const closePromise = onceEvent(socket, 'close'); + + socket.close(1000, 'done'); + await closePromise; + }); + + test('should preserve handshake header names and expose bufferedAmount', async () => { + const socket = await websocket(getBaseUrl().replace('http://', 'ws://') + '/ws', { + browser: 'chrome_137', + disableDefaultHeaders: true, + keepOriginalHeaderNames: true, + headers: [ + ['x-lower', 'one'], + ['X-Mixed', 'two'], + ], + }); + + const connectedEvent = await onceEvent(socket, 'message'); + const payload = JSON.parse(String(connectedEvent.data)) as { rawHeaders: string[] }; + const lowerIndex = payload.rawHeaders.indexOf('x-lower'); + const mixedIndex = payload.rawHeaders.indexOf('X-Mixed'); + + assert.ok(lowerIndex >= 0, 'handshake should preserve lowercase header name'); + assert.ok(mixedIndex >= 0, 'handshake should preserve mixed-case header name'); + assert.ok(lowerIndex < mixedIndex, 'handshake tuple order should be preserved'); + + const largePayload = 'x'.repeat(256 * 1024); + + socket.send(largePayload); + assert.ok( + socket.bufferedAmount >= Buffer.byteLength(largePayload), + 'bufferedAmount should reflect queued outgoing bytes' + ); + + await onceEvent(socket, 'message'); + await new Promise((resolve) => { + setTimeout(resolve, 25); + }); + assert.strictEqual(socket.bufferedAmount, 0, 'bufferedAmount should drain after send'); + + const closePromise = onceEvent(socket, 'close'); + + socket.close(1000, 'done'); + await closePromise; + }); +}); diff --git a/src/types.ts b/src/types.ts index 7c5e1b8..a3d3b4a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,359 +1 @@ -/** - * Browser profile names supported - */ -export type BrowserProfile = - | 'chrome_100' - | 'chrome_101' - | 'chrome_104' - | 'chrome_105' - | 'chrome_106' - | 'chrome_107' - | 'chrome_108' - | 'chrome_109' - | 'chrome_110' - | 'chrome_114' - | 'chrome_116' - | 'chrome_117' - | 'chrome_118' - | 'chrome_119' - | 'chrome_120' - | 'chrome_123' - | 'chrome_124' - | 'chrome_126' - | 'chrome_127' - | 'chrome_128' - | 'chrome_129' - | 'chrome_130' - | 'chrome_131' - | 'chrome_132' - | 'chrome_133' - | 'chrome_134' - | 'chrome_135' - | 'chrome_136' - | 'chrome_137' - | 'edge_101' - | 'edge_122' - | 'edge_127' - | 'edge_131' - | 'edge_134' - | 'safari_ios_17_2' - | 'safari_ios_17_4_1' - | 'safari_ios_16_5' - | 'safari_15_3' - | 'safari_15_5' - | 'safari_15_6_1' - | 'safari_16' - | 'safari_16_5' - | 'safari_17_0' - | 'safari_17_2_1' - | 'safari_17_4_1' - | 'safari_17_5' - | 'safari_18' - | 'safari_ipad_18' - | 'safari_18_2' - | 'safari_ios_18_1_1' - | 'safari_18_3' - | 'safari_18_3_1' - | 'safari_18_5' - | 'firefox_109' - | 'firefox_117' - | 'firefox_128' - | 'firefox_133' - | 'firefox_135' - | 'firefox_private_135' - | 'firefox_android_135' - | 'firefox_136' - | 'firefox_private_136' - | 'firefox_139' - | 'opera_116' - | 'opera_117' - | 'opera_118' - | 'opera_119' - | 'okhttp_3_9' - | 'okhttp_3_11' - | 'okhttp_3_13' - | 'okhttp_3_14' - | 'okhttp_4_9' - | 'okhttp_4_10' - | 'okhttp_4_12' - | 'okhttp_5'; - -export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD'; - -export type HeaderTuple = [string, string]; - -export type HeadersInit = - | Record - | HeaderTuple[] - | Iterable; - -export type BodyInit = string | URLSearchParams | Buffer | ArrayBuffer | ArrayBufferView; - -export interface RequestLike { - url: string; - method?: string; - headers?: HeadersInit; - body?: unknown; - signal?: AbortSignal | null; - bodyUsed?: boolean; - arrayBuffer?: () => Promise; -} - -export type RequestInput = string | URL | RequestLike; - -export interface HookState { - [key: string]: unknown; -} - -export interface CookieJarCookie { - name: string; - value: string; -} - -export interface CookieJar { - getCookies(url: string): Promise | CookieJarCookie[]; - setCookie(cookie: string, url: string): Promise | void; -} - -export interface RetryDecisionContext { - request: NormalizedRequest; - options: ResolvedOptions; - attempt: number; - state: HookState; - error?: unknown; - response?: import('./response').Response; -} - -export interface RetryOptions { - limit?: number; - methods?: HttpMethod[]; - statusCodes?: number[]; - errorCodes?: string[]; - backoff?: (ctx: RetryDecisionContext) => number | Promise; - shouldRetry?: (ctx: RetryDecisionContext) => boolean | Promise; -} - -export type RedirectMode = 'follow' | 'manual' | 'error'; - -export interface RedirectEntry { - fromUrl: string; - status: number; - location: string; - toUrl: string; - method: HttpMethod; -} - -export type WebSocketBinaryType = 'blob' | 'arraybuffer'; - -export interface RequestTimings { - startTime: number; - responseStart: number; - wait: number; - endTime?: number; - total?: number; -} - -export interface RequestStats { - request: NormalizedRequest; - attempt: number; - timings: RequestTimings; - response?: import('./response').Response; - error?: Error; -} - -export interface WebSocketInit { - headers?: HeadersInit; - baseURL?: string; - query?: Record; - browser?: BrowserProfile; - proxy?: string; - timeout?: number; - cookieJar?: CookieJar; - protocols?: string | string[]; - binaryType?: WebSocketBinaryType; -} - -export interface WreqInit { - method?: string; - headers?: HeadersInit; - body?: BodyInit | null; - signal?: AbortSignal | null; - baseURL?: string; - query?: Record; - browser?: BrowserProfile; - proxy?: string; - timeout?: number; - retry?: number | RetryOptions; - redirect?: RedirectMode; - maxRedirects?: number; - cookieJar?: CookieJar; - throwHttpErrors?: boolean; - validateStatus?: (status: number) => boolean; - disableDefaultHeaders?: boolean; - compress?: boolean; - onStats?: (stats: RequestStats) => void | Promise; - context?: Record; - hooks?: Hooks; -} - -export interface NormalizedRequest { - url: string; - method: HttpMethod; - headers: import('./headers').Headers; - body?: BodyInit | null; -} - -export interface ResolvedRetryOptions { - limit: number; - methods: HttpMethod[]; - statusCodes: number[]; - errorCodes: string[]; - backoff?: RetryOptions['backoff']; - shouldRetry?: RetryOptions['shouldRetry']; -} - -export interface ResolvedOptions extends Omit< - WreqInit, - | 'headers' - | 'retry' - | 'throwHttpErrors' - | 'disableDefaultHeaders' - | 'compress' - | 'redirect' - | 'maxRedirects' -> { - headers: import('./headers').Headers; - retry: ResolvedRetryOptions; - throwHttpErrors: boolean; - disableDefaultHeaders: boolean; - compress: boolean; - redirect: RedirectMode; - maxRedirects: number; -} - -export interface InitContext { - input: RequestInput; - options: WreqInit; - state: HookState; -} - -export interface BaseHookContext { - request: NormalizedRequest; - options: ResolvedOptions; - attempt: number; - state: HookState; -} - -export interface BeforeRequestContext extends BaseHookContext {} - -export interface AfterResponseContext extends BaseHookContext { - response: import('./response').Response; -} - -export interface BeforeRetryContext extends BaseHookContext { - error: unknown; - response?: import('./response').Response; -} - -export interface BeforeErrorContext extends BaseHookContext { - error: Error; -} - -export interface BeforeRedirectContext extends BaseHookContext { - response: import('./response').Response; - redirectCount: number; - nextUrl: string; - nextMethod: HttpMethod; - redirectChain: RedirectEntry[]; -} - -export interface Hooks { - init?: InitHook[]; - beforeRequest?: BeforeRequestHook[]; - afterResponse?: AfterResponseHook[]; - beforeRetry?: BeforeRetryHook[]; - beforeError?: BeforeErrorHook[]; - beforeRedirect?: BeforeRedirectHook[]; -} - -export type InitHook = (ctx: InitContext) => void | Promise; -export type BeforeRequestHook = ( - ctx: BeforeRequestContext -) => void | import('./response').Response | Promise; -export type AfterResponseHook = ( - ctx: AfterResponseContext -) => void | import('./response').Response | Promise; -export type BeforeRetryHook = (ctx: BeforeRetryContext) => void | Promise; -export type BeforeErrorHook = (ctx: BeforeErrorContext) => Error | void | Promise; -export type BeforeRedirectHook = (ctx: BeforeRedirectContext) => void | Promise; - -export interface NativeRequestOptions { - url: string; - method: HttpMethod; - headers: Record; - body?: string; - browser?: BrowserProfile; - proxy?: string; - timeout?: number; -} - -export interface NativeResponse { - status: number; - headers: Record; - body?: string; - bodyHandle?: number; - cookies: Record; - setCookies?: string[]; - timings?: RequestTimings; - url: string; -} - -export interface NativeWebSocketConnectOptions { - url: string; - headers: Record; - browser?: BrowserProfile; - proxy?: string; - timeout?: number; - protocols: string[]; -} - -export interface NativeWebSocketConnection { - handle: number; - url: string; - protocol: string | null; - extensions: string | null; -} - -export type NativeWebSocketReadResult = - | { - type: 'text'; - data: string; - } - | { - type: 'binary'; - data: Uint8Array; - } - | { - type: 'close'; - code: number; - reason: string; - wasClean: boolean; - }; - -export interface ClientDefaults extends Omit { - headers?: HeadersInit; - hooks?: Hooks; -} - -export interface Client { - readonly defaults: ClientDefaults; - fetch(input: RequestInput, init?: WreqInit): Promise; - websocket(input: string | URL, init?: WebSocketInit): Promise; - get(input: RequestInput, init?: Omit): Promise; - post( - input: RequestInput, - body?: BodyInit | null, - init?: Omit - ): Promise; - extend(defaults: ClientDefaults): Client; -} +export * from './types/index'; diff --git a/src/types/client.ts b/src/types/client.ts new file mode 100644 index 0000000..389b016 --- /dev/null +++ b/src/types/client.ts @@ -0,0 +1,27 @@ +import type { WebSocket } from '../websocket'; +import type { Hooks } from './hooks'; +import type { RequestInput, WreqInit } from './http'; +import type { HeadersInit } from './shared'; +import type { BodyInit } from './shared'; +import type { WebSocketInit } from './websocket'; + +export interface ClientDefaults extends Omit { + headers?: HeadersInit; + hooks?: Hooks; +} + +export interface Client { + readonly defaults: ClientDefaults; + fetch(input: RequestInput, init?: WreqInit): Promise; + websocket(input: string | URL, init?: WebSocketInit): Promise; + get( + input: RequestInput, + init?: Omit + ): Promise; + post( + input: RequestInput, + body?: BodyInit | null, + init?: Omit + ): Promise; + extend(defaults: ClientDefaults): Client; +} diff --git a/src/types/hooks.ts b/src/types/hooks.ts new file mode 100644 index 0000000..f1708e1 --- /dev/null +++ b/src/types/hooks.ts @@ -0,0 +1,64 @@ +import type { Request as WreqRequest } from '../http/request'; +import type { Response } from '../http/response'; +import type { RedirectEntry, RequestInput, ResolvedOptions, WreqInit } from './http'; +import type { HttpMethod } from './shared'; + +export interface HookState { + [key: string]: unknown; +} + +export interface InitContext { + input: RequestInput; + options: WreqInit; + state: HookState; +} + +export interface BaseHookContext { + request: WreqRequest; + options: ResolvedOptions; + attempt: number; + state: HookState; +} + +export interface BeforeRequestContext extends BaseHookContext {} + +export interface AfterResponseContext extends BaseHookContext { + response: Response; +} + +export interface BeforeRetryContext extends BaseHookContext { + error: unknown; + response?: Response; +} + +export interface BeforeErrorContext extends BaseHookContext { + error: Error; +} + +export interface BeforeRedirectContext extends BaseHookContext { + response: Response; + redirectCount: number; + nextUrl: string; + nextMethod: HttpMethod; + redirectChain: RedirectEntry[]; +} + +export interface Hooks { + init?: InitHook[]; + beforeRequest?: BeforeRequestHook[]; + afterResponse?: AfterResponseHook[]; + beforeRetry?: BeforeRetryHook[]; + beforeError?: BeforeErrorHook[]; + beforeRedirect?: BeforeRedirectHook[]; +} + +export type InitHook = (ctx: InitContext) => void | Promise; +export type BeforeRequestHook = ( + ctx: BeforeRequestContext +) => void | Response | Promise; +export type AfterResponseHook = ( + ctx: AfterResponseContext +) => void | Response | Promise; +export type BeforeRetryHook = (ctx: BeforeRetryContext) => void | Promise; +export type BeforeErrorHook = (ctx: BeforeErrorContext) => Error | void | Promise; +export type BeforeRedirectHook = (ctx: BeforeRedirectContext) => void | Promise; diff --git a/src/types/http.ts b/src/types/http.ts new file mode 100644 index 0000000..e08efb8 --- /dev/null +++ b/src/types/http.ts @@ -0,0 +1,119 @@ +import type { Headers } from '../headers'; +import type { Request as WreqRequest } from '../http/request'; +import type { Response } from '../http/response'; +import type { Hooks, HookState } from './hooks'; +import type { + BodyInit, + BrowserProfile, + CookieJar, + HeadersInit, + Http1Options, + Http2Options, + HttpMethod, + RequestTimings, + TlsOptions, +} from './shared'; + +export type RequestInput = string | URL | WreqRequest | globalThis.Request; + +export interface RetryDecisionContext { + request: WreqRequest; + options: ResolvedOptions; + attempt: number; + state: HookState; + error?: unknown; + response?: Response; +} + +export interface RetryOptions { + limit?: number; + methods?: HttpMethod[]; + statusCodes?: number[]; + errorCodes?: string[]; + backoff?: (ctx: RetryDecisionContext) => number | Promise; + shouldRetry?: (ctx: RetryDecisionContext) => boolean | Promise; +} + +export type RedirectMode = 'follow' | 'manual' | 'error'; + +export interface RedirectEntry { + fromUrl: string; + status: number; + location: string; + toUrl: string; + method: HttpMethod; +} + +export interface RequestStats { + request: WreqRequest; + attempt: number; + timings: RequestTimings; + response?: Response; + error?: Error; +} + +export interface WreqInit { + method?: string; + headers?: HeadersInit; + keepOriginalHeaderNames?: boolean; + body?: BodyInit | null; + signal?: AbortSignal | null; + baseURL?: string; + query?: Record; + browser?: BrowserProfile; + proxy?: string; + timeout?: number; + retry?: number | RetryOptions; + redirect?: RedirectMode; + maxRedirects?: number; + cookieJar?: CookieJar; + throwHttpErrors?: boolean; + validateStatus?: (status: number) => boolean; + disableDefaultHeaders?: boolean; + compress?: boolean; + tlsOptions?: TlsOptions; + http1Options?: Http1Options; + http2Options?: Http2Options; + onStats?: (stats: RequestStats) => void | Promise; + context?: Record; + hooks?: Hooks; +} + +export interface ResolvedRetryOptions { + limit: number; + methods: HttpMethod[]; + statusCodes: number[]; + errorCodes: string[]; + backoff?: RetryOptions['backoff']; + shouldRetry?: RetryOptions['shouldRetry']; +} + +export interface ResolvedOptions extends Omit< + WreqInit, + | 'headers' + | 'retry' + | 'throwHttpErrors' + | 'disableDefaultHeaders' + | 'compress' + | 'keepOriginalHeaderNames' + | 'redirect' + | 'maxRedirects' +> { + headers: Headers; + retry: ResolvedRetryOptions; + throwHttpErrors: boolean; + disableDefaultHeaders: boolean; + compress: boolean; + keepOriginalHeaderNames: boolean; + redirect: RedirectMode; + maxRedirects: number; +} + +export interface WreqResponseMeta { + readonly cookies: Record; + readonly setCookies: string[]; + readonly timings?: RequestTimings; + readonly redirectChain: RedirectEntry[]; + readonly contentLength?: number; + readable(): import('node:stream').Readable; +} diff --git a/src/types/index.ts b/src/types/index.ts new file mode 100644 index 0000000..2b00960 --- /dev/null +++ b/src/types/index.ts @@ -0,0 +1,8 @@ +export type { BrowserProfile } from '../config/generated/browser-profiles'; + +export * from './client'; +export * from './hooks'; +export * from './http'; +export * from './native'; +export * from './shared'; +export * from './websocket'; diff --git a/src/types/native.ts b/src/types/native.ts new file mode 100644 index 0000000..8f6f7e8 --- /dev/null +++ b/src/types/native.ts @@ -0,0 +1,27 @@ +import type { BrowserProfile, HeaderTuple, HttpMethod, RequestTimings } from './shared'; + +export interface NativeRequestOptions { + url: string; + method: HttpMethod; + headers: HeaderTuple[]; + origHeaders?: string[]; + body?: string; + browser?: BrowserProfile; + emulationJson?: string; + proxy?: string; + timeout?: number; + disableDefaultHeaders?: boolean; + compress?: boolean; +} + +export interface NativeResponse { + status: number; + statusText?: string; + headers: Record; + body?: string; + bodyHandle?: number; + cookies: Record; + setCookies?: string[]; + timings?: RequestTimings; + url: string; +} diff --git a/src/types/shared.ts b/src/types/shared.ts new file mode 100644 index 0000000..53d04fc --- /dev/null +++ b/src/types/shared.ts @@ -0,0 +1,126 @@ +export type { BrowserProfile } from '../config/generated/browser-profiles'; + +export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD'; + +export type HeaderTuple = [string, string]; + +export type HeadersInit = + | Record + | HeaderTuple[] + | Iterable; + +export type BodyInit = string | URLSearchParams | Buffer | ArrayBuffer | ArrayBufferView; + +export type AlpnProtocol = 'HTTP1' | 'HTTP2' | 'HTTP3'; +export type AlpsProtocol = 'HTTP1' | 'HTTP2' | 'HTTP3'; +export type TlsVersion = '1.0' | '1.1' | '1.2' | '1.3' | 'TLS1.0' | 'TLS1.1' | 'TLS1.2' | 'TLS1.3'; +export type Http2PseudoHeaderId = 'Method' | 'Scheme' | 'Authority' | 'Path' | 'Protocol'; +export type Http2SettingId = + | 'HeaderTableSize' + | 'EnablePush' + | 'MaxConcurrentStreams' + | 'InitialWindowSize' + | 'MaxFrameSize' + | 'MaxHeaderListSize' + | 'EnableConnectProtocol' + | 'NoRfc7540Priorities'; + +export interface Http2StreamDependency { + dependencyId: number; + weight: number; + exclusive?: boolean; +} + +export interface Http2Priority { + streamId: number; + dependency: Http2StreamDependency; +} + +export interface Http2ExperimentalSetting { + id: number; + value: number; +} + +export interface TlsOptions { + alpnProtocols?: AlpnProtocol[]; + alpsProtocols?: AlpsProtocol[]; + alpsUseNewCodepoint?: boolean; + sessionTicket?: boolean; + minTlsVersion?: TlsVersion; + maxTlsVersion?: TlsVersion; + preSharedKey?: boolean; + enableEchGrease?: boolean; + permuteExtensions?: boolean; + greaseEnabled?: boolean; + enableOcspStapling?: boolean; + enableSignedCertTimestamps?: boolean; + recordSizeLimit?: number; + pskSkipSessionTicket?: boolean; + keySharesLimit?: number; + pskDheKe?: boolean; + renegotiation?: boolean; + delegatedCredentials?: string; + curvesList?: string; + cipherList?: string; + sigalgsList?: string; + certificateCompressionAlgorithms?: Array<'zlib' | 'brotli' | 'zstd'>; + extensionPermutation?: number[]; + aesHwOverride?: boolean; + preserveTls13CipherList?: boolean; + randomAesHwOverride?: boolean; +} + +export interface Http1Options { + http09Responses?: boolean; + writev?: boolean; + maxHeaders?: number; + readBufExactSize?: number; + maxBufSize?: number; + ignoreInvalidHeadersInResponses?: boolean; + allowSpacesAfterHeaderNameInResponses?: boolean; + allowObsoleteMultilineHeadersInResponses?: boolean; +} + +export interface Http2Options { + adaptiveWindow?: boolean; + initialStreamId?: number; + initialConnectionWindowSize?: number; + initialWindowSize?: number; + initialMaxSendStreams?: number; + maxFrameSize?: number; + keepAliveInterval?: number; + keepAliveTimeout?: number; + keepAliveWhileIdle?: boolean; + maxConcurrentResetStreams?: number; + maxSendBufferSize?: number; + maxConcurrentStreams?: number; + maxHeaderListSize?: number; + maxPendingAcceptResetStreams?: number; + enablePush?: boolean; + headerTableSize?: number; + enableConnectProtocol?: boolean; + noRfc7540Priorities?: boolean; + settingsOrder?: Http2SettingId[]; + headersPseudoOrder?: Http2PseudoHeaderId[]; + headersStreamDependency?: Http2StreamDependency; + priorities?: Http2Priority[]; + experimentalSettings?: Http2ExperimentalSetting[]; +} + +export interface CookieJarCookie { + name: string; + value: string; +} + +export interface CookieJar { + getCookies(url: string): Promise | CookieJarCookie[]; + setCookie(cookie: string, url: string): Promise | void; +} + +export interface RequestTimings { + startTime: number; + responseStart: number; + wait: number; + endTime?: number; + total?: number; +} diff --git a/src/types/websocket.ts b/src/types/websocket.ts new file mode 100644 index 0000000..e2685a3 --- /dev/null +++ b/src/types/websocket.ts @@ -0,0 +1,63 @@ +import type { + BrowserProfile, + CookieJar, + HeadersInit, + Http1Options, + Http2Options, + TlsOptions, +} from './shared'; +import type { HeaderTuple } from './shared'; + +export type WebSocketBinaryType = 'blob' | 'arraybuffer'; + +export interface WebSocketInit { + headers?: HeadersInit; + keepOriginalHeaderNames?: boolean; + baseURL?: string; + query?: Record; + browser?: BrowserProfile; + proxy?: string; + timeout?: number; + cookieJar?: CookieJar; + disableDefaultHeaders?: boolean; + tlsOptions?: TlsOptions; + http1Options?: Http1Options; + http2Options?: Http2Options; + protocols?: string | string[]; + binaryType?: WebSocketBinaryType; +} + +export interface NativeWebSocketConnectOptions { + url: string; + headers: HeaderTuple[]; + origHeaders?: string[]; + browser?: BrowserProfile; + emulationJson?: string; + proxy?: string; + timeout?: number; + disableDefaultHeaders?: boolean; + protocols: string[]; +} + +export interface NativeWebSocketConnection { + handle: number; + url: string; + protocol: string | null; + extensions: string | null; +} + +export type NativeWebSocketReadResult = + | { + type: 'text'; + data: string; + } + | { + type: 'binary'; + data: Uint8Array; + } + | { + type: 'close'; + code: number; + reason: string; + wasClean: boolean; + }; diff --git a/src/websocket/close-event.ts b/src/websocket/close-event.ts new file mode 100644 index 0000000..d3ae3e3 --- /dev/null +++ b/src/websocket/close-event.ts @@ -0,0 +1,12 @@ +export class CloseEvent extends Event { + readonly code: number; + readonly reason: string; + readonly wasClean: boolean; + + constructor(type: string, init?: { code?: number; reason?: string; wasClean?: boolean }) { + super(type); + this.code = init?.code ?? 1005; + this.reason = init?.reason ?? ''; + this.wasClean = init?.wasClean ?? false; + } +} diff --git a/src/websocket.ts b/src/websocket/index.ts similarity index 61% rename from src/websocket.ts rename to src/websocket/index.ts index efaddaf..dd2370d 100644 --- a/src/websocket.ts +++ b/src/websocket/index.ts @@ -1,6 +1,6 @@ -import { Buffer } from 'node:buffer'; -import { WebSocketError } from './errors'; -import { Headers } from './headers'; +import { serializeEmulationOptions } from '../config/emulation'; +import { WebSocketError } from '../errors'; +import { loadCookiesIntoHeaders } from '../http/pipeline/cookies'; import { nativeWebSocketClose, nativeWebSocketConnect, @@ -8,205 +8,26 @@ import { nativeWebSocketSendBinary, nativeWebSocketSendText, validateBrowserProfile, -} from './native'; -import type { - CookieJar, - HeadersInit, - NativeWebSocketReadResult, - WebSocketBinaryType, - WebSocketInit, -} from './types'; - -const SUBPROTOCOL_PATTERN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; -const DEFAULT_TIMEOUT = 30_000; -const FORBIDDEN_WEBSOCKET_HEADERS = new Set([ - 'connection', - 'sec-websocket-extensions', - 'sec-websocket-key', - 'sec-websocket-protocol', - 'sec-websocket-version', - 'upgrade', -]); - -function appendQuery(url: URL, query: WebSocketInit['query']): void { - if (!query) { - return; - } - - for (const [key, value] of Object.entries(query)) { - if (value === undefined || value === null) { - continue; - } - - url.searchParams.set(key, String(value)); - } -} - -function resolveWebSocketUrl(rawUrl: string | URL, init?: WebSocketInit): string { - const url = init?.baseURL ? new URL(String(rawUrl), init.baseURL) : new URL(String(rawUrl)); - appendQuery(url, init?.query); - - if (url.protocol !== 'ws:' && url.protocol !== 'wss:') { - throw new WebSocketError(`Invalid WebSocket URL protocol: ${url.protocol}`); - } - - if (url.hash) { - throw new DOMException('WebSocket URL must not include a fragment', 'SyntaxError'); - } - - return url.toString(); -} - -function normalizeHeaders(headers?: HeadersInit): Headers { - const normalized = headers instanceof Headers ? new Headers(headers) : new Headers(headers); - - for (const [name] of normalized) { - if (FORBIDDEN_WEBSOCKET_HEADERS.has(name.toLowerCase())) { - throw new DOMException(`Forbidden WebSocket header: ${name}`, 'SyntaxError'); - } - } - - return normalized; -} - -function normalizeProtocols(protocols?: string | string[]): string[] { - if (!protocols) { - return []; - } - - const values = Array.isArray(protocols) ? protocols : [protocols]; - const seen = new Set(); - - for (const value of values) { - if (!SUBPROTOCOL_PATTERN.test(value)) { - throw new SyntaxError(`Invalid WebSocket subprotocol: ${value}`); - } - - if (seen.has(value)) { - throw new SyntaxError(`Duplicate WebSocket subprotocol: ${value}`); - } - - seen.add(value); - } - - return values; -} - -async function loadCookiesIntoHeaders( - cookieJar: CookieJar | undefined, - url: string, - headers: Headers -) { - if (!cookieJar || headers.has('cookie')) { - return; - } - - const cookies = await cookieJar.getCookies(url); - if (cookies.length === 0) { - return; - } - - headers.set('cookie', cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ')); -} - -function validateCloseCode(code: number): void { - if (code === 1000) { - return; - } - - if (code >= 3000 && code <= 4999) { - return; - } - - throw new DOMException(`Invalid WebSocket close code: ${code}`, 'InvalidAccessError'); -} - -function validateCloseReason(reason: string): void { - if (Buffer.byteLength(reason) > 123) { - throw new DOMException('WebSocket close reason must be 123 bytes or fewer', 'SyntaxError'); - } -} - -async function normalizeSendData(data: string | Blob | ArrayBuffer | ArrayBufferView): Promise< - | { - type: 'text'; - data: string; - } - | { - type: 'binary'; - data: Uint8Array; - } -> { - if (typeof data === 'string') { - return { - type: 'text', - data, - }; - } - - if (data instanceof Blob) { - return { - type: 'binary', - data: new Uint8Array(await data.arrayBuffer()), - }; - } - - if (ArrayBuffer.isView(data)) { - return { - type: 'binary', - data: new Uint8Array(data.buffer, data.byteOffset, data.byteLength), - }; - } - - if (data instanceof ArrayBuffer) { - return { - type: 'binary', - data: new Uint8Array(data), - }; - } - - throw new TypeError('Unsupported WebSocket message type'); -} - -function toMessageEventData( - result: NativeWebSocketReadResult, - binaryType: WebSocketBinaryType -): unknown { - switch (result.type) { - case 'text': { - return result.data; - } - case 'binary': { - if (binaryType === 'arraybuffer') { - const bytes = result.data; - return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); - } +} from '../native'; +import type { WebSocketBinaryType, WebSocketInit } from '../types'; +import { CloseEvent } from './close-event'; +import { getSendByteLength, normalizeSendData, toMessageEventData } from './send-data'; +import { + normalizeHeaders, + normalizeProtocols, + resolveWebSocketUrl, + validateCloseCode, + validateCloseReason, +} from './validation'; - return new Blob([result.data]); - } - case 'close': { - throw new TypeError('Close frames cannot be converted to message events'); - } - } -} +const DEFAULT_TIMEOUT = 30_000; type OpenHandler = ((event: Event) => void) | null; type MessageHandler = ((event: MessageEvent) => void) | null; type CloseHandler = ((event: CloseEvent) => void) | null; type ErrorHandler = ((event: Event) => void) | null; -export class CloseEvent extends Event { - readonly code: number; - readonly reason: string; - readonly wasClean: boolean; - - constructor(type: string, init?: { code?: number; reason?: string; wasClean?: boolean }) { - super(type); - this.code = init?.code ?? 1005; - this.reason = init?.reason ?? ''; - this.wasClean = init?.wasClean ?? false; - } -} +export { CloseEvent }; export class WebSocket extends EventTarget { static readonly CONNECTING = 0; @@ -228,6 +49,7 @@ export class WebSocket extends EventTarget { #handle?: number; #protocol = ''; #binaryType: WebSocketBinaryType; + #bufferedAmount = 0; #sendQueue = Promise.resolve(); #settled = false; #onopen: OpenHandler = null; @@ -242,6 +64,7 @@ export class WebSocket extends EventTarget { validateBrowserProfile(init.browser); const headers = normalizeHeaders(init.headers); const protocols = normalizeProtocols(init.protocols); + if (protocols.length > 0 && headers.has('sec-websocket-protocol')) { throw new DOMException( 'Do not provide sec-websocket-protocol header when protocols option is used', @@ -279,7 +102,7 @@ export class WebSocket extends EventTarget { } get bufferedAmount(): number { - return 0; + return this.#bufferedAmount; } get onopen(): OpenHandler { @@ -323,15 +146,20 @@ export class WebSocket extends EventTarget { throw new DOMException('WebSocket is not open', 'InvalidStateError'); } + const queuedBytes = getSendByteLength(data); + + this.#bufferedAmount += queuedBytes; this.#sendQueue = this.#sendQueue .then(async () => { const normalized = await normalizeSendData(data); + if (this.#readyState !== WebSocket.OPEN || this.#handle === undefined) { throw new DOMException('WebSocket is not open', 'InvalidStateError'); } if (normalized.type === 'text') { await nativeWebSocketSendText(this.#handle, normalized.data); + return; } @@ -339,6 +167,9 @@ export class WebSocket extends EventTarget { }) .catch((error: unknown) => { this.#handleError(error); + }) + .finally(() => { + this.#bufferedAmount = Math.max(0, this.#bufferedAmount - queuedBytes); }); } @@ -353,17 +184,14 @@ export class WebSocket extends EventTarget { return; } - if (this.#readyState === WebSocket.CONNECTING) { - this.#readyState = WebSocket.CLOSING; - } else { - this.#readyState = WebSocket.CLOSING; - } + this.#readyState = WebSocket.CLOSING; if (this.#handle === undefined) { return; } const handle = this.#handle; + this.#handle = undefined; void nativeWebSocketClose(handle, code, reason) @@ -384,16 +212,23 @@ export class WebSocket extends EventTarget { }); } - async #connect(init: WebSocketInit, headers: Headers, protocols: string[]): Promise { + async #connect( + init: WebSocketInit, + headers: import('../headers').Headers, + protocols: string[] + ): Promise { await loadCookiesIntoHeaders(init.cookieJar, this.url, headers); try { const connection = await nativeWebSocketConnect({ url: this.url, - headers: headers.toObject(), + headers: headers.toTuples(), + origHeaders: init.keepOriginalHeaderNames ? headers.toOriginalNames() : undefined, browser: init.browser, + emulationJson: serializeEmulationOptions(init), proxy: init.proxy, timeout: init.timeout ?? DEFAULT_TIMEOUT, + disableDefaultHeaders: init.disableDefaultHeaders ?? false, protocols, }); @@ -422,9 +257,11 @@ export class WebSocket extends EventTarget { while (this.#readyState === WebSocket.OPEN && this.#handle !== undefined) { try { const result = await nativeWebSocketRead(this.#handle); + if (result.type === 'close') { this.#handle = undefined; this.#finalizeClose(result); + return; } @@ -441,6 +278,7 @@ export class WebSocket extends EventTarget { reason: '', wasClean: false, }); + return; } } @@ -462,6 +300,7 @@ export class WebSocket extends EventTarget { #handleError(error: unknown): void { const event = new Event('error'); + Object.defineProperty(event, 'error', { configurable: true, enumerable: false, @@ -497,6 +336,8 @@ export class WebSocket extends EventTarget { export async function websocket(url: string | URL, init?: WebSocketInit): Promise { const socket = new WebSocket(url, init); + await socket.opened; + return socket; } diff --git a/src/websocket/send-data.ts b/src/websocket/send-data.ts new file mode 100644 index 0000000..f390ff7 --- /dev/null +++ b/src/websocket/send-data.ts @@ -0,0 +1,88 @@ +import { Buffer } from 'node:buffer'; +import type { NativeWebSocketReadResult, WebSocketBinaryType } from '../types'; + +export function getSendByteLength(data: string | Blob | ArrayBuffer | ArrayBufferView): number { + if (typeof data === 'string') { + return Buffer.byteLength(data); + } + + if (data instanceof Blob) { + return data.size; + } + + if (ArrayBuffer.isView(data)) { + return data.byteLength; + } + + if (data instanceof ArrayBuffer) { + return data.byteLength; + } + + return 0; +} + +export async function normalizeSendData( + data: string | Blob | ArrayBuffer | ArrayBufferView +): Promise< + | { + type: 'text'; + data: string; + } + | { + type: 'binary'; + data: Uint8Array; + } +> { + if (typeof data === 'string') { + return { + type: 'text', + data, + }; + } + + if (data instanceof Blob) { + return { + type: 'binary', + data: new Uint8Array(await data.arrayBuffer()), + }; + } + + if (ArrayBuffer.isView(data)) { + return { + type: 'binary', + data: new Uint8Array(data.buffer, data.byteOffset, data.byteLength), + }; + } + + if (data instanceof ArrayBuffer) { + return { + type: 'binary', + data: new Uint8Array(data), + }; + } + + throw new TypeError('Unsupported WebSocket message type'); +} + +export function toMessageEventData( + result: NativeWebSocketReadResult, + binaryType: WebSocketBinaryType +): unknown { + switch (result.type) { + case 'text': { + return result.data; + } + case 'binary': { + if (binaryType === 'arraybuffer') { + const bytes = result.data; + + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + } + + return new Blob([result.data]); + } + case 'close': { + throw new TypeError('Close frames cannot be converted to message events'); + } + } +} diff --git a/src/websocket/validation.ts b/src/websocket/validation.ts new file mode 100644 index 0000000..d87843c --- /dev/null +++ b/src/websocket/validation.ts @@ -0,0 +1,97 @@ +import { Buffer } from 'node:buffer'; +import { WebSocketError } from '../errors'; +import { Headers } from '../headers'; +import type { HeadersInit, WebSocketInit } from '../types'; + +const SUBPROTOCOL_PATTERN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; +const FORBIDDEN_WEBSOCKET_HEADERS = new Set([ + 'connection', + 'sec-websocket-extensions', + 'sec-websocket-key', + 'sec-websocket-protocol', + 'sec-websocket-version', + 'upgrade', +]); + +function appendQuery(url: URL, query: WebSocketInit['query']): void { + if (!query) { + return; + } + + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) { + continue; + } + + url.searchParams.set(key, String(value)); + } +} + +export function resolveWebSocketUrl(rawUrl: string | URL, init?: WebSocketInit): string { + const url = init?.baseURL ? new URL(String(rawUrl), init.baseURL) : new URL(String(rawUrl)); + + appendQuery(url, init?.query); + + if (url.protocol !== 'ws:' && url.protocol !== 'wss:') { + throw new WebSocketError(`Invalid WebSocket URL protocol: ${url.protocol}`); + } + + if (url.hash) { + throw new DOMException('WebSocket URL must not include a fragment', 'SyntaxError'); + } + + return url.toString(); +} + +export function normalizeHeaders(headers?: HeadersInit): Headers { + const normalized = headers instanceof Headers ? new Headers(headers) : new Headers(headers); + + for (const [name] of normalized) { + if (FORBIDDEN_WEBSOCKET_HEADERS.has(name.toLowerCase())) { + throw new DOMException(`Forbidden WebSocket header: ${name}`, 'SyntaxError'); + } + } + + return normalized; +} + +export function normalizeProtocols(protocols?: string | string[]): string[] { + if (!protocols) { + return []; + } + + const values = Array.isArray(protocols) ? protocols : [protocols]; + const seen = new Set(); + + for (const value of values) { + if (!SUBPROTOCOL_PATTERN.test(value)) { + throw new SyntaxError(`Invalid WebSocket subprotocol: ${value}`); + } + + if (seen.has(value)) { + throw new SyntaxError(`Duplicate WebSocket subprotocol: ${value}`); + } + + seen.add(value); + } + + return values; +} + +export function validateCloseCode(code: number): void { + if (code === 1000) { + return; + } + + if (code >= 3000 && code <= 4999) { + return; + } + + throw new DOMException(`Invalid WebSocket close code: ${code}`, 'InvalidAccessError'); +} + +export function validateCloseReason(reason: string): void { + if (Buffer.byteLength(reason) > 123) { + throw new DOMException('WebSocket close reason must be 123 bytes or fewer', 'SyntaxError'); + } +} From 29ac32afbb24a142210bbf4ed75a04194dbf009d Mon Sep 17 00:00:00 2001 From: StopMakingThatBigFace Date: Sun, 12 Apr 2026 06:59:34 +0300 Subject: [PATCH 04/15] Update README with anchor links and formatting --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 3354d6f..a04fabe 100644 --- a/README.md +++ b/README.md @@ -12,19 +12,19 @@ npm install node-wreq ## Contents -⚡ **[Quick Start](#quick-start)**\ -🌐 **[Fetch](#fetch)**\ -🧩 **[Client](#client)** — shared defaults, reusable config.\ -🎭 **[Browser Profiles](#browser-profiles)**\ -🪝 **[Hooks](#hooks)** — request lifecycle, dynamic auth, retries, etc.\ -🍪 **[Cookies and Sessions](#cookie-jar)**\ -🔁 **[Redirects and Retries](#redirects-and-retries)**\ -📊 **[Observability](#observability)**\ -🚨 **[Error Handling](#errors)**\ -🔌 **[WebSockets](#websockets)**\ -🧪 **[Networking / Transport Knobs](#networking--transport-knobs)** — TLS, HTTP/1, HTTP/2 options; header ordering. +#### ⚡   **[Quick Start](#quick-start)** +#### 🌐   **[Fetch](#fetch)** +#### 🧩   **[Client](#client)** — shared defaults, reusable config. +#### 🎭   **[Browser Profiles](#browser-profiles)** +#### 🪝   **[Hooks](#hooks)** — request lifecycle, dynamic auth, retries, etc. +#### 🍪   **[Cookies and Sessions](#cookies)** +#### 🔁   **[Redirects and Retries](#redirects-and-retries)** +#### 📊   **[Observability](#observability)** +#### 🚨   **[Error Handling](#errors)** +#### 🔌   **[WebSockets](#websockets)** +#### 🧪   **[Networking / Transport Knobs](#networking)** — TLS, HTTP/1, HTTP/2 options; header ordering. -## Quick Start +## Quick Start [↑](#contents) ```ts import { fetch } from 'node-wreq'; @@ -59,7 +59,7 @@ console.log(response.status); console.log(await response.json()); ``` -## Fetch +## Fetch [↑](#contents) ### Simple GET @@ -148,7 +148,7 @@ const readable = response.wreq.readable(); readable.pipe(process.stdout); ``` -## Client +## Client [↑](#contents) Use `createClient(...)` when requests share defaults: @@ -214,7 +214,7 @@ await base.get('/health'); await admin.get('/admin/stats'); ``` -## Browser Profiles +## Browser Profiles [↑](#contents) Inspect the available profiles at runtime: @@ -235,7 +235,7 @@ Typical profiles include browser families like: - Opera - OkHttp -## Hooks +## Hooks [↑](#contents) Hooks are the request pipeline. @@ -347,7 +347,7 @@ Rule of thumb: - use hooks for dynamic behavior - use client defaults for static behavior -## Cookie Jar +## Cookies and Sessions [↑](#contents) `node-wreq` does not force a built-in cookie store. @@ -437,7 +437,7 @@ console.log(response.wreq.setCookies); console.log(response.wreq.cookies); ``` -## Redirects and Retries +## Redirects and Retries [↑](#contents) Both are opt-in controls on top of the normal request pipeline. @@ -552,7 +552,7 @@ Current timings are wrapper-level timings that are still useful in practice: - response available - total time when body consumption is known -## Errors +## Errors [↑](#contents) Main error classes: @@ -583,7 +583,7 @@ try { } ``` -## WebSockets +## WebSockets [↑](#contents) You can use either: @@ -646,7 +646,7 @@ Notes: - cookies from `cookieJar` are sent during handshake - duplicate subprotocols are rejected -## Networking / Transport Knobs +## Networking / Transport Knobs [↑](#contents) This is the "transport nerd" section. From 3f4e028bf0b4a5dcb3bbb3735c94d58ecb59055d Mon Sep 17 00:00:00 2001 From: StopMakingThatBigFace Date: Sun, 12 Apr 2026 07:01:53 +0300 Subject: [PATCH 05/15] Fix section headers in README for clarity --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a04fabe..8a19985 100644 --- a/README.md +++ b/README.md @@ -513,7 +513,7 @@ Defaults: - default status codes include `408`, `425`, `429`, `500`, `502`, `503`, `504` - default error codes include `ECONNRESET`, `ECONNREFUSED`, `ETIMEDOUT`, `ERR_TIMEOUT` -## Observability +## Observability [↑](#contents) Two main surfaces: @@ -552,7 +552,7 @@ Current timings are wrapper-level timings that are still useful in practice: - response available - total time when body consumption is known -## Errors [↑](#contents) +## Error Handling [↑](#contents) Main error classes: @@ -646,7 +646,7 @@ Notes: - cookies from `cookieJar` are sent during handshake - duplicate subprotocols are rejected -## Networking / Transport Knobs [↑](#contents) +## Networking / Transport Knobs [↑](#contents) This is the "transport nerd" section. From fc3e6ec7a9735ef95ecfb6029c231708d8ff0208 Mon Sep 17 00:00:00 2001 From: StopMakingThatBigFace Date: Sun, 12 Apr 2026 07:19:56 +0300 Subject: [PATCH 06/15] Refactor README section titles and formatting Updated README formatting and section titles to lowercase. --- README.md | 96 +++++++++++++++++++++++++++---------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 8a19985..73613f9 100644 --- a/README.md +++ b/README.md @@ -4,27 +4,27 @@ This package helps if you need low-level control over the network layer — TLS configuration, transport fingerprinting, browser impersonation, or fine-grained HTTP/WebSocket behavior that standard Node.js clients don't expose. -## Install +## install ```bash npm install node-wreq ``` -## Contents +## contents -#### ⚡   **[Quick Start](#quick-start)** -#### 🌐   **[Fetch](#fetch)** -#### 🧩   **[Client](#client)** — shared defaults, reusable config. -#### 🎭   **[Browser Profiles](#browser-profiles)** -#### 🪝   **[Hooks](#hooks)** — request lifecycle, dynamic auth, retries, etc. -#### 🍪   **[Cookies and Sessions](#cookies)** -#### 🔁   **[Redirects and Retries](#redirects-and-retries)** -#### 📊   **[Observability](#observability)** -#### 🚨   **[Error Handling](#errors)** -#### 🔌   **[WebSockets](#websockets)** -#### 🧪   **[Networking / Transport Knobs](#networking)** — TLS, HTTP/1, HTTP/2 options; header ordering. +#### ⚡   **[quick start](#quick-start)** +#### 🌐   **[fetch](#fetch)** +#### 🧩   **[client](#client)** — shared defaults, reusable config. +#### 🎭   **[browser profiles](#browser-profiles)** +#### 🪝   **[hooks](#hooks)** — request lifecycle, dynamic auth, retries, etc. +#### 🍪   **[cookies and sessions](#cookies)** +#### 🔁   **[redirects and retries](#redirects-and-retries)** +#### 📊   **[observability](#observability)** +#### 🚨   **[error handling](#errors)** +#### 🔌   **[websockets](#websockets)** +#### 🧪   **[networking / transport knobs](#networking)** — TLS, HTTP/1, HTTP/2 options; header ordering. -## Quick Start [↑](#contents) +## ⚡ quick start   ·   [↑](#contents) ```ts import { fetch } from 'node-wreq'; @@ -59,9 +59,9 @@ console.log(response.status); console.log(await response.json()); ``` -## Fetch [↑](#contents) +## 🌐 fetch   ·   [↑](#contents) -### Simple GET +### simple GET ```ts import { fetch } from 'node-wreq'; @@ -102,7 +102,7 @@ const response = await fetch('https://api.example.com/items', { console.log(await response.json()); ``` -### Build a `Request` first +### build a `Request` first ```ts import { Request, fetch } from 'node-wreq'; @@ -122,7 +122,7 @@ const response = await fetch(request, { console.log(await response.json()); ``` -### Read extra metadata +### read extra metadata `fetch()` returns a fetch-style `Response`, plus extra metadata under `response.wreq`. @@ -148,7 +148,7 @@ const readable = response.wreq.readable(); readable.pipe(process.stdout); ``` -## Client [↑](#contents) +## 🧩 client   ·   [↑](#contents) Use `createClient(...)` when requests share defaults: @@ -161,7 +161,7 @@ Use `createClient(...)` when requests share defaults: - retry policy - cookie jar -### Shared defaults +### shared defaults ```ts import { createClient } from 'node-wreq'; @@ -196,7 +196,7 @@ const created = await client.post( console.log(created.status); ``` -### Extend a client +### extend a client ```ts const base = createClient({ @@ -214,7 +214,7 @@ await base.get('/health'); await admin.get('/admin/stats'); ``` -## Browser Profiles [↑](#contents) +## 🎭 browser profiles   ·   [↑](#contents) Inspect the available profiles at runtime: @@ -235,7 +235,7 @@ Typical profiles include browser families like: - Opera - OkHttp -## Hooks [↑](#contents) +## 🪝 hooks   ·   [↑](#contents) Hooks are the request pipeline. @@ -248,7 +248,7 @@ Available phases: - `beforeError` - `beforeRedirect` -### Common pattern: auth, tracing, proxy rotation +### common pattern: auth, tracing, proxy rotation ```ts import { createClient } from 'node-wreq'; @@ -301,7 +301,7 @@ const client = createClient({ }); ``` -### Replace a response in `afterResponse` +### replace a response in `afterResponse` ```ts import { Response, fetch } from 'node-wreq'; @@ -327,7 +327,7 @@ const response = await fetch('https://example.com/account', { console.log(await response.json()); ``` -### Mutate redirect hops +### mutate redirect hops ```ts await fetch('https://example.com/login', { @@ -347,7 +347,7 @@ Rule of thumb: - use hooks for dynamic behavior - use client defaults for static behavior -## Cookies and Sessions [↑](#contents) +## 🍪 cookies and sessions   ·   [↑](#contents) `node-wreq` does not force a built-in cookie store. @@ -364,7 +364,7 @@ That jar can be: - DB-backed - anything else that matches the interface -### Tiny in-memory jar +### tiny in-memory jar ```ts import { fetch, websocket } from 'node-wreq'; @@ -426,7 +426,7 @@ await client.fetch('https://example.com/login'); await client.fetch('https://example.com/profile'); ``` -### Inspect cookies on a response +### inspect cookies on a response ```ts import { fetch } from 'node-wreq'; @@ -437,11 +437,11 @@ console.log(response.wreq.setCookies); console.log(response.wreq.cookies); ``` -## Redirects and Retries [↑](#contents) +## 🔁 redirects and retries   ·   [↑](#contents) Both are opt-in controls on top of the normal request pipeline. -### Manual redirects +### manual redirects ```ts const response = await fetch('https://example.com/login', { @@ -467,7 +467,7 @@ Useful redirect facts: - `307` / `308` preserve method and body - `authorization` is stripped on cross-origin redirect -### Simple retries +### simple retries ```ts const response = await fetch('https://example.com', { @@ -475,7 +475,7 @@ const response = await fetch('https://example.com', { }); ``` -### Explicit retry policy +### explicit retry policy ```ts const response = await fetch('https://example.com', { @@ -487,7 +487,7 @@ const response = await fetch('https://example.com', { }); ``` -### Custom retry decision +### custom retry decision ```ts import { TimeoutError, fetch } from 'node-wreq'; @@ -513,14 +513,14 @@ Defaults: - default status codes include `408`, `425`, `429`, `500`, `502`, `503`, `504` - default error codes include `ECONNRESET`, `ECONNREFUSED`, `ETIMEDOUT`, `ERR_TIMEOUT` -## Observability [↑](#contents) +## 📊 observability   ·   [↑](#contents) Two main surfaces: - `response.wreq.timings` - `onStats(stats)` -### Per-request stats callback +### per-request stats callback ```ts await fetch('https://example.com', { @@ -536,7 +536,7 @@ await fetch('https://example.com', { }); ``` -### Read timings from the final response +### read timings from the final response ```ts const response = await fetch('https://example.com', { @@ -552,7 +552,7 @@ Current timings are wrapper-level timings that are still useful in practice: - response available - total time when body consumption is known -## Error Handling [↑](#contents) +## 🚨 error handling   ·   [↑](#contents) Main error classes: @@ -583,14 +583,14 @@ try { } ``` -## WebSockets [↑](#contents) +## 🔌 websockets   ·   [↑](#contents) You can use either: - `await websocket(url, init?)` - `new WebSocket(url, init?)` -### Simple helper +### simple helper ```ts import { websocket } from 'node-wreq'; @@ -628,7 +628,7 @@ socket.send(new Uint8Array([1, 2, 3])); socket.close(1000, 'done'); ``` -### WebSocket from a client +### websocket from a client Useful when you want shared defaults like browser, proxy, or cookies: @@ -646,13 +646,13 @@ Notes: - cookies from `cookieJar` are sent during handshake - duplicate subprotocols are rejected -## Networking / Transport Knobs [↑](#contents) +## 🧪 networking / transport knobs   ·   [↑](#contents) This is the "transport nerd" section. Everything else here is for debugging request shape, fingerprint-sensitive targets, or testing transport hypotheses. -### Browser profile + proxy + timeout +### browser profile + proxy + timeout ```ts const response = await fetch('https://httpbin.org/anything', { @@ -662,7 +662,7 @@ const response = await fetch('https://httpbin.org/anything', { }); ``` -### Disable default browser-like headers +### disable default browser-like headers By default, `node-wreq` may apply profile-appropriate default headers. @@ -678,7 +678,7 @@ await fetch('https://example.com', { }); ``` -### Exact header order +### exact header order Use tuples when header order matters: @@ -691,7 +691,7 @@ await fetch('https://example.com', { }); ``` -### Exact original header names on the wire +### exact original header names on the wire Use this only if you really need exact casing / spelling preservation: @@ -706,7 +706,7 @@ await fetch('https://example.com', { }); ``` -### Lower-level transport tuning +### lower-level transport tuning If a browser preset gets you close but not all the way there: @@ -732,7 +732,7 @@ Use these only when: - you are comparing transport behavior - you want to debug fingerprint mismatches -### Compression +### compression Compression is enabled by default. From 32c62a1aff56c9817367199f64205c1685825e63 Mon Sep 17 00:00:00 2001 From: StopMakingThatBigFace Date: Sun, 12 Apr 2026 16:19:31 +0300 Subject: [PATCH 07/15] Enhance README with additional details and tips Expanded README with detailed explanations on usage and features. --- README.md | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 73613f9..b3a21e1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,32 @@ # node-wreq -`node-wreq` is a Node HTTP client with a Rust transport underneath backed by [`wreq`](https://github.com/0x676e67/wreq). - -This package helps if you need low-level control over the network layer — TLS configuration, transport fingerprinting, browser impersonation, or fine-grained HTTP/WebSocket behavior that standard Node.js clients don't expose. +[![NPM Version](https://img.shields.io/npm/v/node-wreq)](https://npmjs.com/package/node-wreq) +![ESM](https://img.shields.io/badge/ESM-supported-brightgreen) +![Platforms](https://img.shields.io/badge/platforms-linux%20%7C%20macos%20%7C%20windows-lightgrey) + +`node-wreq` is a thin Node.js wrapper around 0x676e67's [`wreq`](https://github.com/0x676e67/wreq) — +a Rust HTTP client exposing its full power to JavaScript. + +Use it when you need low-level control over the network layer: TLS configuration, +transport fingerprinting, browser impersonation, or fine-grained HTTP/WebSocket +behavior that standard Node.js clients simply don't expose. + +> [!TIP] +> ### why does this exist? +> +> Node.js ships with a built-in `https` module, and the ecosystem offers popular clients like `axios`, `got`, and `node-fetch` — but all of them are built on top of OpenSSL via Node's `tls` module, which exposes **no control over low-level TLS handshake parameters**. This makes it fundamentally impossible to emulate real browser network behavior from pure JavaScript. +> +> - **HTTP/1 over TLS** +> +> Node.js HTTP clients normalize headers to lowercase internally, which is compliant with HTTP/2 semantics but breaks compatibility with some **WAFs** that enforce case-sensitive header validation on **HTTP/1** requests. This wrapper preserves header case exactly as specified, preventing requests from being silently rejected. +> +> - **HTTP/2 over TLS** +> +> Fingerprints like **JA3**, **JA4**, and **Akamai HTTP/2** are derived from the specifics of the TLS handshake and HTTP/2 SETTINGS frames — cipher suite ordering, TLS extensions, ALPN values, HPACK header compression parameters, and more. Node.js exposes none of these through its `tls` or `http2` APIs. You simply cannot spoof them from JS land, no matter the library. This package solves that at the native layer, giving you fine-grained control over TLS and HTTP/2 extensions to precisely match real browser behavior. +> +> - **Device Emulation** +> +> Because TLS and HTTP/2 fingerprints evolve slowly relative to browser release cycles, a single fingerprint profile often covers many browser versions. **100+ pre-built browser device profiles** are bundled, so you don't have to figure out the right combination of settings yourself. ## install @@ -24,7 +48,7 @@ npm install node-wreq #### 🔌   **[websockets](#websockets)** #### 🧪   **[networking / transport knobs](#networking)** — TLS, HTTP/1, HTTP/2 options; header ordering. -## ⚡ quick start   ·   [↑](#contents) +## ⚡ quick start ```ts import { fetch } from 'node-wreq'; From d20dade73970c230771e5774a1a39b6fe3bb43d2 Mon Sep 17 00:00:00 2001 From: StopMakingThatBigFace Date: Sun, 12 Apr 2026 16:25:41 +0300 Subject: [PATCH 08/15] Enhance TLS and HTTP/2 fingerprinting section in README Expanded the explanation of TLS and HTTP/2 fingerprinting and its relevance to bot protection services. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index b3a21e1..565ee9c 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,11 @@ behavior that standard Node.js clients simply don't expose. > > Because TLS and HTTP/2 fingerprints evolve slowly relative to browser release cycles, a single fingerprint profile often covers many browser versions. **100+ pre-built browser device profiles** are bundled, so you don't have to figure out the right combination of settings yourself. +TLS and HTTP/2 fingerprinting is actively used by major bot protection and WAF providers — including **Cloudflare** Bot Management, **AWS WAF** (Bot Control + CloudFront JA3 headers), **Google Cloud Armor**, **Akamai** (which maintains its own HTTP/2 fingerprint format on top of JA3/JA4), **ServicePipe** (a Russian DDoS protection and WAF provider), and various specialized anti-bot services like **DataDome** and **PerimeterX**. Correctly emulating a browser's TLS handshake and HTTP/2 SETTINGS frames is a hard requirement to get past these layers undetected. + +> [!NOTE] +> This only covers the transport layer. It does not help bypass JavaScript-based challenges (Cloudflare Turnstile, Akamai sensor data, Kasada, etc.), CAPTCHA, or behavioral analysis — those require a different approach entirely + ## install ```bash From 04ca5d6a4afe5afc9c164c457b2210cbf56a9b33 Mon Sep 17 00:00:00 2001 From: StopMakingThatBigFace Date: Sun, 12 Apr 2026 16:35:29 +0300 Subject: [PATCH 09/15] Update README with TLS fingerprinting resources Added links to resources on TLS fingerprinting and bot protection. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 565ee9c..e5702c6 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,12 @@ behavior that standard Node.js clients simply don't expose. TLS and HTTP/2 fingerprinting is actively used by major bot protection and WAF providers — including **Cloudflare** Bot Management, **AWS WAF** (Bot Control + CloudFront JA3 headers), **Google Cloud Armor**, **Akamai** (which maintains its own HTTP/2 fingerprint format on top of JA3/JA4), **ServicePipe** (a Russian DDoS protection and WAF provider), and various specialized anti-bot services like **DataDome** and **PerimeterX**. Correctly emulating a browser's TLS handshake and HTTP/2 SETTINGS frames is a hard requirement to get past these layers undetected. + [**TLS Fingerprinting with JA3 and JA3S**](https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/) + + [**JA3/JA4 Fingerprint — Cloudflare Bot Solutions**](https://developers.cloudflare.com/bots/additional-configurations/ja3-ja4-fingerprint/) + + [**TLS Fingerprinting: How It Works & How to Bypass It**](https://www.browserless.io/blog/tls-fingerprinting-explanation-detection-and-bypassing-it-in-playwright-and-puppeteer) + > [!NOTE] > This only covers the transport layer. It does not help bypass JavaScript-based challenges (Cloudflare Turnstile, Akamai sensor data, Kasada, etc.), CAPTCHA, or behavioral analysis — those require a different approach entirely From 2616531b97d501933d348bfe60837ba57f035ace Mon Sep 17 00:00:00 2001 From: ruby Date: Sun, 12 Apr 2026 18:19:50 +0400 Subject: [PATCH 10/15] ci: add git hooks --- .githooks/pre-commit | 13 +++++++++ package.json | 1 + scripts/install-git-hooks.mjs | 22 +++++++++++++++ src/native/binding.ts | 50 ++++++++++++++++++++++++----------- 4 files changed, 70 insertions(+), 16 deletions(-) create mode 100755 .githooks/pre-commit create mode 100644 scripts/install-git-hooks.mjs diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..80ea62c --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +echo "pre-commit: running lint autofix" +npm run lint:fix + +echo "pre-commit: running formatter" +npm run format + +git add -A -- src diff --git a/package.json b/package.json index eb89c96..d1a346c 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "build": "npm run build:rust && npm run build:ts", "build:rust": "napi build --platform --release --cargo-cwd rust rust", "build:ts": "node ./scripts/generate-browser-profiles.mjs && tsc && node ./scripts/postbuild.mjs", + "prepare": "node ./scripts/install-git-hooks.mjs", "prepare:publish:main": "node ./scripts/prepare-main-package.mjs", "prepare:publish:platform": "node ./scripts/prepare-platform-package.mjs", "artifacts": "napi artifacts", diff --git a/scripts/install-git-hooks.mjs b/scripts/install-git-hooks.mjs new file mode 100644 index 0000000..fa06281 --- /dev/null +++ b/scripts/install-git-hooks.mjs @@ -0,0 +1,22 @@ +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = resolve(fileURLToPath(new URL('..', import.meta.url))); +const gitDir = resolve(repoRoot, '.git'); +const hooksPath = resolve(repoRoot, '.githooks'); + +if (!existsSync(gitDir) || !existsSync(hooksPath)) { + process.exit(0); +} + +try { + execFileSync('git', ['config', 'core.hooksPath', '.githooks'], { + cwd: repoRoot, + stdio: 'ignore', + }); +} catch (error) { + console.warn('[install-git-hooks] Failed to set core.hooksPath to .githooks'); + console.warn(error instanceof Error ? error.message : String(error)); +} diff --git a/src/native/binding.ts b/src/native/binding.ts index 037c42b..e213c85 100644 --- a/src/native/binding.ts +++ b/src/native/binding.ts @@ -1,4 +1,5 @@ import { execSync } from 'node:child_process'; +import { resolve } from 'node:path'; import type { NativeRequestOptions, NativeResponse, @@ -33,6 +34,14 @@ type NativeTarget = { let nativeBinding: NativeBinding | undefined; +function tryRequire(id: string): T | undefined { + try { + return require(id) as T; + } catch { + return undefined; + } +} + function isMuslRuntime(): boolean { if (process.platform !== 'linux') { return false; @@ -111,28 +120,37 @@ function loadNativeBinding(): NativeBinding { } const attempted: string[] = [target.packageName]; + const packageBinding = tryRequire(target.packageName); - try { - return require(target.packageName) as NativeBinding; - } catch { - attempted.push(`../rust/${target.binaryName}`); + if (packageBinding) { + return packageBinding; } - try { - return require(`../rust/${target.binaryName}`) as NativeBinding; - } catch { - attempted.push('../rust/node-wreq.node'); + const localBinaryPath = resolve(__dirname, '../../rust', target.binaryName); + + attempted.push(localBinaryPath); + + const localPlatformBinding = tryRequire(localBinaryPath); + + if (localPlatformBinding) { + return localPlatformBinding; } - try { - return require('../rust/node-wreq.node') as NativeBinding; - } catch { - throw new Error( - `Failed to load native module for ${platform}-${arch}. ` + - `Tried: ${attempted.join(', ')}. ` + - `Make sure the matching @node-wreq platform package is installed or build the local native module.` - ); + const localGenericBinaryPath = resolve(__dirname, '../../rust/node-wreq.node'); + + attempted.push(localGenericBinaryPath); + + const localGenericBinding = tryRequire(localGenericBinaryPath); + + if (localGenericBinding) { + return localGenericBinding; } + + throw new Error( + `Failed to load native module for ${platform}-${arch}. ` + + `Tried: ${attempted.join(', ')}. ` + + `Make sure the matching @node-wreq platform package is installed or build the local native module.` + ); } export function getBinding(): NativeBinding { From 2699b314b0741643f101b2d5595b4b2f29e931ab Mon Sep 17 00:00:00 2001 From: ruby Date: Sun, 12 Apr 2026 18:53:32 +0400 Subject: [PATCH 11/15] ci: github trusted publishing setup --- .github/workflows/build.yml | 64 ++++++++++++++++++++++++---- .gitignore | 2 + docs/PUBLISHING.md | 25 +++++++++-- package-lock.json | 4 +- package.json | 2 +- scripts/prepare-main-package.mjs | 6 ++- scripts/prepare-platform-package.mjs | 4 +- scripts/publish-version.mjs | 9 ++++ scripts/stub.sh | 51 ++++++++++++++++++++++ 9 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 scripts/publish-version.mjs create mode 100755 scripts/stub.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1802c4..0c7758e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,9 +90,33 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 24 registry-url: 'https://registry.npmjs.org' + - name: Determine npm dist-tag + id: dist_tag + shell: bash + run: | + if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then + echo "value=rc" >> "$GITHUB_OUTPUT" + else + echo "value=latest" >> "$GITHUB_OUTPUT" + fi + + - name: Determine publish version + id: publish_version + shell: bash + run: | + tag="${{ github.event.release.tag_name }}" + version="${tag#v}" + + if [[ -z "$version" ]]; then + echo "Release tag produced an empty version" >&2 + exit 1 + fi + + echo "value=$version" >> "$GITHUB_OUTPUT" + - name: Install dependencies run: npm ci @@ -104,6 +128,8 @@ jobs: - name: Prepare scoped platform package shell: bash + env: + NODE_WREQ_PUBLISH_VERSION: ${{ steps.publish_version.outputs.value }} run: | BINARY_PATH="$(find artifacts -name '*.node' | head -n 1)" node ./scripts/prepare-platform-package.mjs \ @@ -112,9 +138,7 @@ jobs: --outDir ".release/${{ matrix.settings.target }}" - name: Publish scoped platform package - run: npm publish ".release/${{ matrix.settings.target }}" --provenance --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish ".release/${{ matrix.settings.target }}" --access public --tag "${{ steps.dist_tag.outputs.value }}" publish-main: name: Publish main package @@ -131,9 +155,33 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 24 registry-url: 'https://registry.npmjs.org' + - name: Determine npm dist-tag + id: dist_tag + shell: bash + run: | + if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then + echo "value=rc" >> "$GITHUB_OUTPUT" + else + echo "value=latest" >> "$GITHUB_OUTPUT" + fi + + - name: Determine publish version + id: publish_version + shell: bash + run: | + tag="${{ github.event.release.tag_name }}" + version="${tag#v}" + + if [[ -z "$version" ]]; then + echo "Release tag produced an empty version" >&2 + exit 1 + fi + + echo "value=$version" >> "$GITHUB_OUTPUT" + - name: Install dependencies run: npm ci @@ -141,9 +189,9 @@ jobs: run: npm run build:ts - name: Prepare main npm package + env: + NODE_WREQ_PUBLISH_VERSION: ${{ steps.publish_version.outputs.value }} run: npm run prepare:publish:main -- .release/main-package - name: Publish main package - run: npm publish .release/main-package --provenance --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish .release/main-package --access public --tag "${{ steps.dist_tag.outputs.value }}" diff --git a/.gitignore b/.gitignore index f66f99f..c9d34bf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ node_modules/ # Build outputs dist/ .release/ +.release-stubs/ +.tmp-release/ rust/target/ rust/node-wreq.node *.node diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index ca5927e..fb733c6 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -7,6 +7,10 @@ Publishing is split into two layers: - scoped platform packages with the native `.node` binaries - the main `node-wreq` package with JS, types, loader logic, and `optionalDependencies` +The publish version can be overridden at staging time with `NODE_WREQ_PUBLISH_VERSION`. +That lets CI publish from the GitHub release tag even if the repository's root `package.json` +stays on a placeholder or development version locally. + ### Package Structure When published, the package includes: @@ -31,9 +35,9 @@ Scoped package names: #### 1. Prerequisites -- npm account with publish permissions +- npm account with package publish permissions - GitHub repository set up -- `NPM_TOKEN` configured in GitHub Secrets +- npm trusted publishing configured for each package you publish from this workflow #### 1. Update Version @@ -66,6 +70,19 @@ GitHub Actions will automatically: 4. Stage the main package with generated `optionalDependencies` 5. Publish the main package to npm +Workflow requirements for trusted publishing: + +- the trusted publisher must point at this repository and the exact workflow filename: `.github/workflows/build.yml` +- publish jobs must run on GitHub-hosted runners +- publish jobs need `permissions.id-token: write` +- no `NPM_TOKEN` is required for the `npm publish` steps + +Release tag behavior: + +- prereleases publish with npm dist-tag `rc` +- stable releases publish with npm dist-tag `latest` +- the published package version is derived from the GitHub release tag, for example `v1.0.0-rc2` -> `1.0.0-rc2` + ### Local Testing Before Publishing ```bash @@ -91,7 +108,7 @@ If you really need to publish a scoped platform package manually: npm run build:rust -- --target x86_64-unknown-linux-musl # Stage the scoped package -node ./scripts/prepare-platform-package.mjs \ +NODE_WREQ_PUBLISH_VERSION=1.0.0-rc2 node ./scripts/prepare-platform-package.mjs \ --target x86_64-unknown-linux-musl \ --binary rust/node-wreq.linux-x64-musl.node \ --outDir .release/linux-x64-musl @@ -106,7 +123,7 @@ After the platform packages for the same version exist: ```bash npm run build:ts -npm run prepare:publish:main -- .release/main-package +NODE_WREQ_PUBLISH_VERSION=1.0.0-rc2 npm run prepare:publish:main -- .release/main-package npm publish .release/main-package --access public ``` diff --git a/package-lock.json b/package-lock.json index e2341e9..c638e9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "node-wreq", - "version": "0.1.2", + "version": "0.0.0-reserved.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "node-wreq", - "version": "0.1.2", + "version": "0.0.0-reserved.0", "cpu": [ "x64", "arm64" diff --git a/package.json b/package.json index d1a346c..8e35d80 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-wreq", - "version": "0.1.2", + "version": "0.0.0-reserved.0", "description": "Node HTTP client with native browser impersonation, hooks, cookies, redirects, websockets, and timing stats", "main": "./dist/node-wreq.js", "module": "./dist/node-wreq.mjs", diff --git a/scripts/prepare-main-package.mjs b/scripts/prepare-main-package.mjs index 5c6ebd9..bd8e258 100644 --- a/scripts/prepare-main-package.mjs +++ b/scripts/prepare-main-package.mjs @@ -2,6 +2,7 @@ import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { getOptionalDependencyMap } from "./platform-targets.mjs"; +import { resolvePublishVersion } from "./publish-version.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(__dirname, ".."); @@ -10,6 +11,7 @@ const outDir = resolve(repoRoot, process.argv[2] ?? ".release/main-package"); const rootPackage = JSON.parse( await readFile(resolve(repoRoot, "package.json"), "utf8"), ); +const publishVersion = resolvePublishVersion(rootPackage); await rm(outDir, { recursive: true, force: true }); @@ -29,7 +31,7 @@ await cp(resolve(repoRoot, "README.md"), resolve(outDir, "README.md")); const publishPackage = { name: rootPackage.name, - version: rootPackage.version, + version: publishVersion, description: rootPackage.description, main: rootPackage.main, module: rootPackage.module, @@ -44,7 +46,7 @@ const publishPackage = { engines: rootPackage.engines, os: rootPackage.os, cpu: rootPackage.cpu, - optionalDependencies: getOptionalDependencyMap(rootPackage.version), + optionalDependencies: getOptionalDependencyMap(publishVersion), files: ["dist", "docs", "README.md"], }; diff --git a/scripts/prepare-platform-package.mjs b/scripts/prepare-platform-package.mjs index e961b69..dc91b59 100644 --- a/scripts/prepare-platform-package.mjs +++ b/scripts/prepare-platform-package.mjs @@ -2,6 +2,7 @@ import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { basename, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { getPlatformTargetByTriple } from "./platform-targets.mjs"; +import { resolvePublishVersion } from "./publish-version.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(__dirname, ".."); @@ -42,6 +43,7 @@ if (!args.outDir) { const rootPackage = JSON.parse( await readFile(resolve(repoRoot, "package.json"), "utf8"), ); +const publishVersion = resolvePublishVersion(rootPackage); const outDir = resolve(repoRoot, args.outDir); const binarySource = resolve(repoRoot, args.binary); @@ -51,7 +53,7 @@ await cp(binarySource, resolve(outDir, target.binaryName)); const packageJson = { name: target.packageName, - version: rootPackage.version, + version: publishVersion, description: `Prebuilt native binding for ${rootPackage.name} on ${target.target}`, license: rootPackage.license, author: rootPackage.author, diff --git a/scripts/publish-version.mjs b/scripts/publish-version.mjs new file mode 100644 index 0000000..d3150bb --- /dev/null +++ b/scripts/publish-version.mjs @@ -0,0 +1,9 @@ +export function resolvePublishVersion(rootPackage) { + const override = process.env.NODE_WREQ_PUBLISH_VERSION?.trim(); + + if (override) { + return override; + } + + return rootPackage.version; +} diff --git a/scripts/stub.sh b/scripts/stub.sh new file mode 100755 index 0000000..67c02e6 --- /dev/null +++ b/scripts/stub.sh @@ -0,0 +1,51 @@ +VERSION=0.0.0-reserved.0 +OUT=.release-stubs + +rm -rf "$OUT" +mkdir -p "$OUT" + +create_stub () { + DIR="$1" + NAME="$2" + OS_JSON="$3" + CPU_JSON="$4" + LIBC_JSON="$5" + + mkdir -p "$OUT/$DIR" + + cat > "$OUT/$DIR/package.json" < "$OUT/$DIR/README.md" < Date: Sun, 12 Apr 2026 19:03:16 +0400 Subject: [PATCH 12/15] chore: keep feat/v1 implementation after merging main --- package-lock.json | 203 +------------------ package.json | 11 +- rust/Cargo.lock | 24 --- rust/Cargo.toml | 6 - rust/src/websocket.rs | 146 ------------- src/test/{http.spec.ts => node-wreq.spec.ts} | 0 6 files changed, 3 insertions(+), 387 deletions(-) delete mode 100644 rust/src/websocket.rs rename src/test/{http.spec.ts => node-wreq.spec.ts} (100%) diff --git a/package-lock.json b/package-lock.json index d59b0e3..c638e9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,48 +28,7 @@ "ws": "^8.18.3" }, "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" + "node": ">=18.0.0" } }, "node_modules/@eslint-community/eslint-utils": { @@ -1784,166 +1743,6 @@ "node": ">= 0.8.0" } }, - "node_modules/rimraf": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.0.1.tgz", - "integrity": "sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==", - "dev": true, - "license": "ISC", - "dependencies": { - "glob": "^11.0.0", - "package-json-from-dist": "^1.0.0" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", - "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/typescript": { "version": "5.9.3", "dev": true, diff --git a/package.json b/package.json index 4f8978b..8e35d80 100644 --- a/package.json +++ b/package.json @@ -37,14 +37,7 @@ "tls", "http2", "impersonation", - "web-scraping", - "crawler", - "web-scraper", - "wreq", - "ja3", - "tls-fingerprint", - "ja4", - "browser-fingerprint-bypass" + "web-scraping" ], "author": "StopMakingThatBigFace", "license": "MIT", @@ -67,7 +60,7 @@ "ws": "^8.18.3" }, "engines": { - "node": ">=20.0.0" + "node": ">=18.0.0" }, "os": [ "darwin", diff --git a/rust/Cargo.lock b/rust/Cargo.lock index a68afa9..5805c06 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -394,17 +394,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "futures-sink" version = "0.3.31" @@ -803,9 +792,7 @@ name = "node-wreq" version = "0.1.0" dependencies = [ "anyhow", - "futures-util", "neon", - "once_cell", "serde", "serde_json", "strum", @@ -1111,17 +1098,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "shlex" version = "1.3.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e160ee2..0977cc6 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -13,9 +13,6 @@ wreq = { version = "6.0.0-rc.28", default-features = false, features = ["gzip", wreq-util = { version = "3.0.0-rc.10", features = ["emulation-rand", "emulation-serde"] } strum = "0.27.2" -# WebSocket support -futures-util = "0.3" - # Neon for Node.js bindings neon = { version = "1.0", default-features = false, features = ["napi-6"] } @@ -30,9 +27,6 @@ thiserror = "1.0" # Async runtime (if needed) tokio = { version = "1.0", features = ["full"] } -# Global state management -once_cell = "1.20" - [profile.release] opt-level = 3 lto = true diff --git a/rust/src/websocket.rs b/rust/src/websocket.rs deleted file mode 100644 index 12bce25..0000000 --- a/rust/src/websocket.rs +++ /dev/null @@ -1,146 +0,0 @@ -use anyhow::{Context, Result}; -use futures_util::{SinkExt, StreamExt}; -use neon::prelude::*; -use std::collections::HashMap; -use std::sync::{Arc, Mutex as StdMutex}; -use tokio::sync::Mutex; -use wreq::ws::message::Message; -use wreq::ws::WebSocket; -use wreq_util::Emulation; -use once_cell::sync::Lazy; - -// Global storage for WebSocket connections -static WS_CONNECTIONS: Lazy>>> = - Lazy::new(|| StdMutex::new(HashMap::new())); - -static NEXT_WS_ID: Lazy> = Lazy::new(|| StdMutex::new(1)); - -// Global Tokio runtime for WebSocket operations -pub static WS_RUNTIME: Lazy = Lazy::new(|| { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("Failed to create Tokio runtime for WebSockets") -}); - -#[derive(Debug, Clone)] -pub struct WebSocketOptions { - pub url: String, - pub emulation: Emulation, - pub headers: HashMap, - pub proxy: Option, -} - -/// WebSocket connection wrapper -pub struct WsConnection { - sender: Arc>>, -} - -impl WsConnection { - pub fn new( - sender: futures_util::stream::SplitSink, - ) -> Self { - Self { - sender: Arc::new(Mutex::new(sender)), - } - } - - /// Send a text message - pub async fn send_text(&self, text: String) -> Result<()> { - let mut sender = self.sender.lock().await; - sender - .send(Message::text(text)) - .await - .context("Failed to send text message")?; - Ok(()) - } - - /// Send a binary message - pub async fn send_binary(&self, data: Vec) -> Result<()> { - let mut sender = self.sender.lock().await; - sender - .send(Message::binary(data)) - .await - .context("Failed to send binary message")?; - Ok(()) - } - - /// Close the WebSocket connection - pub async fn close(&self) -> Result<()> { - let mut sender = self.sender.lock().await; - sender - .send(Message::close(None)) - .await - .context("Failed to close WebSocket")?; - Ok(()) - } -} - -// Finalize implementation for proper cleanup -impl Finalize for WsConnection {} - -/// Store a WebSocket connection and return its ID -pub fn store_connection(connection: WsConnection) -> u64 { - let mut id_lock = NEXT_WS_ID.lock().unwrap(); - let id = *id_lock; - *id_lock += 1; - drop(id_lock); - - let mut connections = WS_CONNECTIONS.lock().unwrap(); - connections.insert(id, Arc::new(connection)); - id -} - -/// Get a WebSocket connection by ID -pub fn get_connection(id: u64) -> Option> { - let connections = WS_CONNECTIONS.lock().unwrap(); - connections.get(&id).cloned() -} - -/// Remove a WebSocket connection -pub fn remove_connection(id: u64) { - let mut connections = WS_CONNECTIONS.lock().unwrap(); - connections.remove(&id); -} - -/// Create WebSocket connection -pub async fn connect_websocket(options: WebSocketOptions) -> Result<(WsConnection, futures_util::stream::SplitStream)> { - // Build client with emulation and proxy - let mut client_builder = wreq::Client::builder() - .emulation(options.emulation); - - // Apply proxy if present - if let Some(proxy_url) = &options.proxy { - let proxy = wreq::Proxy::all(proxy_url).context("Failed to create proxy")?; - client_builder = client_builder.proxy(proxy); - } - - // Build the client - let client = client_builder - .build() - .context("Failed to build HTTP client")?; - - // Create WebSocket request - let mut request = client.websocket(&options.url); - - // Apply custom headers - for (key, value) in &options.headers { - request = request.header(key, value); - } - - // Send upgrade request - let ws_response = request - .send() - .await - .context("Failed to send WebSocket upgrade request")?; - - // Upgrade to WebSocket - let websocket = ws_response.into_websocket().await?; - - // Split into sender and receiver - let (sender, receiver) = websocket.split(); - - let connection = WsConnection::new(sender); - - Ok((connection, receiver)) -} diff --git a/src/test/http.spec.ts b/src/test/node-wreq.spec.ts similarity index 100% rename from src/test/http.spec.ts rename to src/test/node-wreq.spec.ts From 65dcbbbff1eb7ab6afa671e3c3a6656a17151db8 Mon Sep 17 00:00:00 2001 From: ruby Date: Sun, 12 Apr 2026 19:12:24 +0400 Subject: [PATCH 13/15] chore: update description --- package.json | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8e35d80..7a644bf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "node-wreq", "version": "0.0.0-reserved.0", - "description": "Node HTTP client with native browser impersonation, hooks, cookies, redirects, websockets, and timing stats", + "description": "HTTP client with native TLS, HTTP2, JA3, JA4 browser impersonation backed by wreq's Rust core", "main": "./dist/node-wreq.js", "module": "./dist/node-wreq.mjs", "types": "dist/node-wreq.d.ts", @@ -30,14 +30,29 @@ "format:check": "oxfmt --check \"src/**/*.ts\"" }, "keywords": [ + "anti-bot", + "bypass", "browser", + "browser-fingerprint-bypass", + "browseer-impersonation", + "browser-emulation", + "crawler", + "cloudflare", + "cloudflare-bypass", + "curl-impersonate", + "scrapper", "fingerprint", - "bypass", - "anti-bot", "tls", + "tls-fingerprint", "http2", + "fetch", + "websocket", "impersonation", - "web-scraping" + "web-scraping", + "web-scraper", + "wreq", + "ja3", + "ja4" ], "author": "StopMakingThatBigFace", "license": "MIT", From a3d507ec6a5b06cd7ff7fb9e844cb8db45d9827b Mon Sep 17 00:00:00 2001 From: ruby Date: Sun, 12 Apr 2026 19:17:49 +0400 Subject: [PATCH 14/15] ci: update build configuration for aarch64 and x86_64 targets --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c7758e..06d171e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,12 +22,12 @@ jobs: build: npm run build:rust -- --target x86_64-unknown-linux-gnu - host: ubuntu-latest target: aarch64-unknown-linux-gnu - setup: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - build: CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ npm run build:rust -- --target aarch64-unknown-linux-gnu + setup: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu binutils-aarch64-linux-gnu + build: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ npm run build:rust -- --target aarch64-unknown-linux-gnu - host: ubuntu-latest target: x86_64-unknown-linux-musl setup: sudo apt-get update && sudo apt-get install -y musl-tools - build: CC_x86_64_unknown_linux_musl=musl-gcc npm run build:rust -- --target x86_64-unknown-linux-musl + build: CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc AR_x86_64_unknown_linux_musl=ar CC_x86_64_unknown_linux_musl=musl-gcc CXX_x86_64_unknown_linux_musl=musl-gcc npm run build:rust -- --target x86_64-unknown-linux-musl - host: windows-latest target: x86_64-pc-windows-msvc build: npm run build:rust -- --target x86_64-pc-windows-msvc From da906b1f8afd571fca008b7756bba60ffb9490aa Mon Sep 17 00:00:00 2001 From: ruby Date: Sun, 12 Apr 2026 19:45:29 +0400 Subject: [PATCH 15/15] ci: fix linux native build toolchains --- .github/workflows/build.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06d171e..0c9be86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,14 +20,25 @@ jobs: - host: ubuntu-latest target: x86_64-unknown-linux-gnu build: npm run build:rust -- --target x86_64-unknown-linux-gnu - - host: ubuntu-latest + - host: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu - setup: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu binutils-aarch64-linux-gnu - build: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ npm run build:rust -- --target aarch64-unknown-linux-gnu + build: npm run build:rust -- --target aarch64-unknown-linux-gnu - host: ubuntu-latest target: x86_64-unknown-linux-musl - setup: sudo apt-get update && sudo apt-get install -y musl-tools - build: CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc AR_x86_64_unknown_linux_musl=ar CC_x86_64_unknown_linux_musl=musl-gcc CXX_x86_64_unknown_linux_musl=musl-gcc npm run build:rust -- --target x86_64-unknown-linux-musl + setup: | + set -eux + TOOLCHAIN_URL="https://github.com/troglobit/misc/releases/download/11-20211120/x86_64-linux-musl-cross.tgz" + TOOLCHAIN_ROOT="$HOME/musl-cross" + TOOLCHAIN_DIR="$TOOLCHAIN_ROOT/x86_64-linux-musl-cross" + + if [ ! -d "$TOOLCHAIN_DIR" ]; then + mkdir -p "$TOOLCHAIN_ROOT" + curl -L "$TOOLCHAIN_URL" -o /tmp/x86_64-linux-musl-cross.tgz + tar -xzf /tmp/x86_64-linux-musl-cross.tgz -C "$TOOLCHAIN_ROOT" + fi + + echo "$TOOLCHAIN_DIR/bin" >> "$GITHUB_PATH" + build: CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-linux-musl-gcc CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++ AR_x86_64_unknown_linux_musl=x86_64-linux-musl-ar RANLIB_x86_64_unknown_linux_musl=x86_64-linux-musl-ranlib npm run build:rust -- --target x86_64-unknown-linux-musl - host: windows-latest target: x86_64-pc-windows-msvc build: npm run build:rust -- --target x86_64-pc-windows-msvc