diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5593984949..a0d2a3fe68 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,6 +53,13 @@ jobs: name: dev-pages-react${{ matrix.react }} path: pages/lib/static-default + - name: Upload test utils selectors artifact + if: matrix.react == 18 + uses: actions/upload-artifact@v4 + with: + name: test-utils-selectors + path: lib/components + deploy: needs: quick-build name: deploy${{ matrix.react != 16 && format(' (React {0})', matrix.react) || '' }} @@ -65,3 +72,14 @@ jobs: with: artifact-name: dev-pages-react${{ matrix.react }} deployment-path: pages/lib/static-default + + visual: + name: Visual regression + needs: [quick-build, deploy] + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + uses: ./.github/workflows/visual-regression.yml + secrets: inherit + with: + pr-artifact-name: dev-pages-react18 + test-utils-artifact-name: test-utils-selectors + caller-run-id: ${{ github.run_id }} diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml new file mode 100644 index 0000000000..a34a75b75c --- /dev/null +++ b/.github/workflows/visual-regression.yml @@ -0,0 +1,241 @@ +name: Visual Regression Tests + +on: + workflow_call: + inputs: + pr-artifact-name: + description: 'Name of the artifact containing PR pages (built by the caller workflow).' + required: true + type: string + test-utils-artifact-name: + description: 'Name of the artifact containing test-utils selectors.' + required: true + type: string + caller-run-id: + description: 'The run ID of the calling workflow, used to download artifacts it uploaded.' + required: true + type: string + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + actions: read + deployments: write + +jobs: + # Build the baseline (main branch) pages once and share them across all browser jobs. + build-baseline: + name: Build baseline pages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm i + + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm i + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path "${GITHUB_WORKSPACE}/pages/lib/static-visual-baseline" + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Upload baseline artifact + uses: actions/upload-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + retention-days: 1 + + visual: + name: Visual regression (shard ${{ matrix.shard }}) + needs: [build-baseline] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Set up Chrome and ChromeDriver + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + - name: Install dependencies + run: npm i + + - name: Download PR pages artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.pr-artifact-name }} + path: pages/lib/static-default + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} + + - name: Download baseline artifact + uses: actions/download-artifact@v4 + with: + name: visual-baseline-pages + path: pages/lib/static-visual-baseline + + - name: Download test utils artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.test-utils-artifact-name }} + path: lib/components + github-token: ${{ github.token }} + run-id: ${{ inputs.caller-run-id }} + + # ── Run tests ───────────────────────────────────────────────────────── + + - name: Resolve deployment URLs + id: hosts + run: | + # The deploy workflow creates deployments with environment name = artifact name. + # For React 18, the environment is "dev-pages-react18". + DEPLOY_ENV="dev-pages-react18" + + # --- New host: deployment for this PR --- + echo "Looking for deployment with environment '${DEPLOY_ENV}' for this PR..." + NEW_URL=$(gh api \ + "repos/${REPO}/deployments?environment=${DEPLOY_ENV}&per_page=5" \ + --jq '.[0].id // empty' 2>/dev/null || true) + if [ -n "$NEW_URL" ]; then + DEPLOY_ID="$NEW_URL" + NEW_URL=$(gh api \ + "repos/${REPO}/deployments/${DEPLOY_ID}/statuses" \ + --jq '[.[] | select(.state == "success")] | first | .environment_url // empty' 2>/dev/null || true) + echo "Found current PR deployment: ID=${DEPLOY_ID}, URL=${NEW_URL}" + fi + + # --- Old host: deployment from the last PR that was merged to main --- + OLD_URL="" + LAST_MERGED_SHA=$(gh api \ + "repos/${REPO}/pulls?state=closed&base=main&sort=updated&direction=desc&per_page=10" \ + --jq '[.[] | select(.merged_at != null)] | first | .head.sha // empty' 2>/dev/null || true) + if [ -n "$LAST_MERGED_SHA" ]; then + echo "Last merged PR head SHA: ${LAST_MERGED_SHA:0:8}" + OLD_DEPLOY_ID=$(gh api \ + "repos/${REPO}/deployments?environment=${DEPLOY_ENV}&sha=${LAST_MERGED_SHA}&per_page=1" \ + --jq '.[0].id // empty' 2>/dev/null || true) + if [ -n "$OLD_DEPLOY_ID" ]; then + OLD_URL=$(gh api \ + "repos/${REPO}/deployments/${OLD_DEPLOY_ID}/statuses" \ + --jq '[.[] | select(.state == "success")] | first | .environment_url // empty' 2>/dev/null || true) + echo "Found baseline deployment: ID=${OLD_DEPLOY_ID}, URL=${OLD_URL}" + fi + fi + + echo "new=${NEW_URL}" >> "$GITHUB_OUTPUT" + echo "old=${OLD_URL}" >> "$GITHUB_OUTPUT" + echo "" + echo "=== Result ===" + echo "New host: ${NEW_URL:-(not found)}" + echo "Old host: ${OLD_URL:-(not found)}" + + if [ -z "$NEW_URL" ] || [ -z "$OLD_URL" ]; then + echo "::error::Could not resolve deployment URLs. NEW_URL=${NEW_URL}, OLD_URL=${OLD_URL}" + exit 1 + fi + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + + - name: Run visual regression tests + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js --shard="${SHARD}/2" + env: + TZ: UTC + SHARD: ${{ matrix.shard }} + NEW_HOST: ${{ steps.hosts.outputs.new }} + OLD_HOST: ${{ steps.hosts.outputs.old }} + + - name: Upload diff artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-diffs-shard-${{ matrix.shard }} + path: visual-regression-output/ + retention-days: 14 + + - name: Upload Allure results + if: always() + uses: actions/upload-artifact@v4 + with: + name: allure-results-shard-${{ matrix.shard }} + path: allure-results/ + retention-days: 3 + + report: + name: Generate Allure Report + if: always() + needs: [visual] + runs-on: ubuntu-latest + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Download all Allure results + uses: actions/download-artifact@v4 + with: + pattern: allure-results-shard-* + path: allure-results + merge-multiple: true + + - name: Generate Allure HTML report + run: npx --yes allure generate allure-results -o allure-report + + - name: Upload Allure report artifact + uses: actions/upload-artifact@v4 + with: + name: allure-report + path: allure-report/ + retention-days: 14 + + deploy-report: + name: Deploy Allure Report + if: always() + needs: [report] + uses: cloudscape-design/actions/.github/workflows/deploy.yml@main + secrets: inherit + with: + artifact-name: allure-report + deployment-path: allure-report diff --git a/.gitignore b/.gitignore index db3d62b944..9b14cf8ced 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ coverage lib # generated sources src/index.ts +allure-results +allure-report src/test-utils/dom/index.ts src/test-utils/selectors src/icon/generated diff --git a/build-tools/tasks/package-json.js b/build-tools/tasks/package-json.js index a7c69a53f5..2c447682a1 100644 --- a/build-tools/tasks/package-json.js +++ b/build-tools/tasks/package-json.js @@ -103,6 +103,10 @@ const devPagesPackageJson = generatePackageJson(path.join(workspace.targetPath, const testDefinitionsPackageJson = generatePackageJson(path.join(workspace.targetPath, 'test-definitions'), { name: '@cloudscape-design/test-definitions', + exports: { + '.': './index.js', + './types': './types.js', + }, }); module.exports = parallel([ diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js new file mode 100644 index 0000000000..52ce2f271c --- /dev/null +++ b/build-tools/visual/global-setup.js @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +module.exports = async () => { + const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + await startWebdriver(); +}; diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js new file mode 100644 index 0000000000..0fa05eebfe --- /dev/null +++ b/build-tools/visual/global-teardown.js @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +module.exports = () => { + const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); + shutdownWebdriver(); +}; diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js new file mode 100644 index 0000000000..d52cd606fb --- /dev/null +++ b/build-tools/visual/setup.js @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +/* global jest */ +const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); + +configure({ + browserName: 'ChromeHeadlessIntegration', + browserCreatorOptions: { + seleniumUrl: 'http://localhost:9515', + }, + webdriverOptions: { + baseUrl: 'http://localhost:8080', + }, +}); + +jest.retryTimes(2, { logErrorsBeforeRetry: true }); diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index 525cdf181d..4db82e361b 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -60,11 +60,60 @@ TZ=UTC npx jest -u -c jest.unit.config.js src/ ``` ## Visual Regression Tests -> **Note:** The components repository does not have visual regression tests on GitHub. This section applies to other repositories such as chat-components, code-view, chart-components, and board-components. +Visual regression tests run automatically when opening a pull request in GitHub (see `.github/workflows/visual-regression.yml`). -Visual regression tests for permutation pages run automatically when opening a pull request in GitHub. +They compare permutation pages between the PR build and a baseline build of `main`, both served locally in the same CI job. Each side installs from its own `package-lock.json` via a git worktree, so dependency changes in the PR are handled correctly and unpinned updates in sister repositories affect both sides equally. -To check results: look at the "Visual Regression Tests" action in the PR. The "Test for regressions" step logs which pages failed. For a full report, download the `visual-regression-snapshots-results` artifact from the action summary. +### How it works -If there are unexpected regressions, fix your pull request. -If the changes are expected, call this out in your pull request comments. +1. The PR pages are built and served on port 8080. +2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081. +3. The single test runner (`test/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ. + +### Running locally + +``` +npm run test:visual +``` + +This handles the full build and comparison in one command. If both outputs are already built, skip the build step: + +``` +NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js +``` + +(Requires both servers to be running — start the PR build with `npm run start:integ` on port 8080 and the baseline build on port 8081, or set `NEW_HOST` / `OLD_HOST` env vars to point at different hosts.) + +### Adding tests for a new component + +Create `test/definitions/visual/.ts`: + +```ts +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'my-component', + tests: [ + { + description: 'permutations', + path: 'my-component/permutations', + }, + ], +}; + +export default suite; +``` + +Then run the generation script to pick it up automatically: + +```bash +node build-tools/visual/generate-tests.js +``` + +This generates both the test runner (`test/visual/my-component.test.ts`) and updates `test/definitions/index.ts`. No manual imports needed. + +### Reviewing failures + +If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary. + +If the diff is expected (intentional visual change), note it in your PR description. diff --git a/eslint.config.mjs b/eslint.config.mjs index 0d9423aa5b..b92a169696 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -225,7 +225,7 @@ export default tsEslint.config( }, }, { - files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**'], + files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/definitions/**'], rules: { // useBrowser is not a hook 'react-hooks/rules-of-hooks': 'off', diff --git a/jest.visual.config.js b/jest.visual.config.js new file mode 100644 index 0000000000..98938768b3 --- /dev/null +++ b/jest.visual.config.js @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const path = require('path'); +const os = require('os'); + +module.exports = { + verbose: true, + testEnvironment: 'allure-jest/node', + testEnvironmentOptions: { + resultsDir: 'allure-results', + }, + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + tsconfig: 'tsconfig.visual.json', + }, + ], + }, + reporters: ['default', 'github-actions'], + testTimeout: 240_000, // 4min — pages can be tall and slow to capture + maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), + globalSetup: '/build-tools/visual/global-setup.js', + globalTeardown: '/build-tools/visual/global-teardown.js', + setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], + moduleFileExtensions: ['js', 'ts'], + testMatch: ['/test/visual/**/*.test.ts'], +}; diff --git a/package-lock.json b/package-lock.json index c4d0123dfc..48e18b734b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -60,6 +61,7 @@ "@types/react-test-renderer": "^16.9.12", "@types/react-transition-group": "^4.4.4", "@types/webpack-env": "^1.16.3", + "allure-jest": "^3.9.0", "axe-core": "^4.7.2", "babel-jest": "^29.7.0", "change-case": "^4.1.2", @@ -96,6 +98,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", @@ -3521,9 +3524,9 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", - "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.2.tgz", + "integrity": "sha512-5EUZSUIc37H6aIXyWO0Z4y8NlF8NnjgmqeQgOGiswAU7pY0HOo16ho4+alIWmSfdZnjqBRawMsP3I5YqLSn6kw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -4796,6 +4799,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pngjs": { "version": "6.0.5", "dev": true, @@ -5842,6 +5855,58 @@ "dev": true, "license": "MIT" }, + "node_modules/allure-jest": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/allure-jest/-/allure-jest-3.9.0.tgz", + "integrity": "sha512-hEW4DKjvb3engGoHUPQaDEdyrFkUxQnqULiSQAehL1eDEggqdPbQro86Nch8Cj1yuIqUTn9UP1FMuuuwl/5jnQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "allure-js-commons": "3.9.0" + }, + "peerDependencies": { + "jest": ">=24.8.0", + "jest-circus": ">=24.8.0", + "jest-cli": ">=24.8.0", + "jest-environment-jsdom": ">=24.8.0", + "jest-environment-node": ">=24.8.0" + }, + "peerDependenciesMeta": { + "jest": { + "optional": true + }, + "jest-circus": { + "optional": true + }, + "jest-cli": { + "optional": true + }, + "jest-environment-jsdom": { + "optional": true + }, + "jest-environment-node": { + "optional": true + } + } + }, + "node_modules/allure-js-commons": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/allure-js-commons/-/allure-js-commons-3.9.0.tgz", + "integrity": "sha512-uVQcGE6MWIvGR/zW1XEUwHXUQa1EJKY0Cah+0TZK1qKuw6ptyhftDr34XE3wExTyCZirRrI98dbRtPeYYuyI+g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "md5": "^2.3.0" + }, + "peerDependencies": { + "allure-playwright": "3.9.0" + }, + "peerDependenciesMeta": { + "allure-playwright": { + "optional": true + } + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "dev": true, @@ -7104,6 +7169,16 @@ "node": ">=10" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/cheerio": { "version": "1.1.0", "dev": true, @@ -7212,6 +7287,20 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "dev": true, @@ -7798,6 +7887,16 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/css-declaration-sorter": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-7.3.1.tgz", @@ -8728,6 +8827,13 @@ "dev": true, "license": "MIT" }, + "node_modules/devtools-protocol": { + "version": "0.0.1608973", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz", + "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/diff-sequences": { "version": "29.6.3", "dev": true, @@ -12509,6 +12615,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true, + "license": "MIT" + }, "node_modules/is-builtin-module": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-5.0.0.tgz", @@ -15188,6 +15301,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/mdn-data": { "version": "2.12.2", "dev": true, @@ -17685,6 +17810,25 @@ "node": ">=6" } }, + "node_modules/puppeteer-core": { + "version": "24.43.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.43.1.tgz", + "integrity": "sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.2", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1608973", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.20.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "dev": true, @@ -21099,6 +21243,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "dev": true, + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "dev": true, @@ -21616,6 +21767,13 @@ "node": ">=18.20.0" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/webdriver/node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -22393,6 +22551,16 @@ "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 368682e67d..3aaa1ef485 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -83,6 +84,7 @@ "@types/react-test-renderer": "^16.9.12", "@types/react-transition-group": "^4.4.4", "@types/webpack-env": "^1.16.3", + "allure-jest": "^3.9.0", "axe-core": "^4.7.2", "babel-jest": "^29.7.0", "change-case": "^4.1.2", @@ -119,6 +121,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", diff --git a/test/definitions/index.ts b/test/definitions/index.ts index 91e90a89f7..67970b0ab6 100644 --- a/test/definitions/index.ts +++ b/test/definitions/index.ts @@ -1,9 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually. -// Each component has its own test definition file. -// Import them here manually to form the full test suite. import { TestSuite } from './types'; +export { TestSuite, TestDefinition, ScreenshotType, ScreenshotTestConfiguration } from './types'; import actionCard from './visual/action-card'; import alert from './visual/alert'; diff --git a/test/definitions/types.ts b/test/definitions/types.ts index 8cdca9996a..adaaf9f23c 100644 --- a/test/definitions/types.ts +++ b/test/definitions/types.ts @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import type { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; +import type createWrapper from '../../lib/components/test-utils/selectors'; + +export type Wrapper = ReturnType; + export interface ScreenshotTestConfiguration { width?: number; height?: number; @@ -9,7 +13,7 @@ export interface ScreenshotTestConfiguration { // 'screenshotArea' — captures the .screenshot-area element on the page. // 'permutations' — captures the entire page and crops permutations out of it. -export type ScreenshotType = 'screenshotArea' | 'permutations'; +export type ScreenshotType = 'screenshotArea' | 'permutations' | 'viewport'; export interface TestDefinition { description: string; @@ -17,7 +21,7 @@ export interface TestDefinition { screenshotType: ScreenshotType; queryParams?: Record; configuration?: ScreenshotTestConfiguration; - setup?: (page: ScreenshotPageObject) => Promise; + setup?: (page: ScreenshotPageObject, wrapper: Wrapper) => Promise; } export interface TestSuite { diff --git a/test/definitions/utils.ts b/test/definitions/utils.ts new file mode 100644 index 0000000000..862e1834d9 --- /dev/null +++ b/test/definitions/utils.ts @@ -0,0 +1,161 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { attachment } from 'allure-js-commons'; + +import { cropAndCompare } from '@cloudscape-design/browser-test-tools/image-utils'; +import { ScreenshotPageObject, ScreenshotWithOffset } from '@cloudscape-design/browser-test-tools/page-objects'; + +import createWrapper from '../../lib/components/test-utils/selectors'; +import { TestDefinition, TestSuite } from './types'; + +const screenshotAreaSelector = '.screenshot-area'; +const defaultWindowSize = { width: 1200, height: 800 }; + +// NEW_HOST serves the PR's pages, OLD_HOST serves the baseline (main) pages. +const newHost = process.env.NEW_HOST || 'http://localhost:8080'; +const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; + +const wrapper = createWrapper(); + +function buildUrl(host: string, path: string, queryParams?: Record): string { + const params = new URLSearchParams(queryParams); + const qs = params.toString(); + return `${host}/#/${path}${qs ? `?${qs}` : ''}`; +} + +function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { + return (item as TestDefinition).path !== undefined; +} + +/** + * Attaches a visual comparison to the Allure report using the built-in image diff viewer. + */ +async function attachDiffImages( + result: { firstImage: Buffer; secondImage: Buffer; diffImage: Buffer | null }, + testName: string +): Promise { + const diffPayload = JSON.stringify({ + expected: `data:image/png;base64,${result.secondImage.toString('base64')}`, + actual: `data:image/png;base64,${result.firstImage.toString('base64')}`, + diff: result.diffImage ? `data:image/png;base64,${result.diffImage.toString('base64')}` : undefined, + }); + + await attachment(testName, diffPayload, { + contentType: 'application/vnd.allure.image.diff', + fileExtension: 'imagediff', + } as any); +} + +/** + * Registers all test suites with a single shared browser session per worker. + * This avoids the per-test session creation overhead. + */ +export function runTestSuites(suites: Array) { + let browser: WebdriverIO.Browser; + + beforeAll(async () => { + const { default: getBrowserCreator } = await import('@cloudscape-design/browser-test-tools/browser'); + const creator = getBrowserCreator('ChromeHeadlessIntegration', 'local', { + seleniumUrl: 'http://localhost:9515', + }); + browser = await creator.getBrowser({ width: defaultWindowSize.width, height: defaultWindowSize.height }); + }); + + afterAll(async () => { + await browser?.deleteSession(); + }); + + registerSuites(suites, () => browser); +} + +function registerSuites(suites: Array, getBrowser: () => WebdriverIO.Browser) { + for (const item of suites) { + if (isTestDefinition(item)) { + registerTest(item, getBrowser); + } else { + describe(item.description, () => { + registerSuites(item.tests, getBrowser); + }); + } + } +} + +/** + * Navigates to a URL, waits for the screenshot area, and runs any setup interactions. + */ +async function preparePage( + browser: WebdriverIO.Browser, + page: ScreenshotPageObject, + url: string, + testDef: TestDefinition, + windowSize?: { width?: number; height?: number } +): Promise { + await browser.setWindowSize( + windowSize?.width ?? defaultWindowSize.width, + windowSize?.height ?? defaultWindowSize.height + ); + const params = new URLSearchParams({ motionDisabled: 'true' }); + await browser.url(`${url}?${params.toString()}`); + await page.waitForVisible(screenshotAreaSelector); + if (testDef.setup) { + await testDef.setup(page, wrapper); + } +} + +/** + * Captures a screenshot based on the test's screenshotType. + */ +function capture(page: ScreenshotPageObject, testDef: TestDefinition): Promise { + if (testDef.screenshotType === 'viewport') { + return page.captureViewport(); + } + return page.captureBySelector(screenshotAreaSelector); +} + +function registerTest(testDef: TestDefinition, getBrowser: () => WebdriverIO.Browser) { + test(testDef.description, async () => { + const browser = getBrowser(); + const page = new ScreenshotPageObject(browser); + + const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); + const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); + + await preparePage(browser, page, newUrl, testDef, testDef.configuration); + const newScreenshot = await capture(page, testDef); + + await preparePage(browser, page, oldUrl, testDef, testDef.configuration); + const oldScreenshot = await capture(page, testDef); + + const result = await cropAndCompare(newScreenshot, oldScreenshot); + + if (testDef.screenshotType === 'permutations') { + if (result.diffPixels === 0) { + return; + } + await preparePage(browser, page, newUrl, testDef, testDef.configuration); + const newPermutations = await page.capturePermutations(); + + await preparePage(browser, page, oldUrl, testDef, testDef.configuration); + const oldPermutations = await page.capturePermutations(); + + expect(newPermutations.length).toBe(oldPermutations.length); + const permFailures: number[] = []; + const attachmentPromises: Promise[] = []; + for (let i = 0; i < newPermutations.length; i++) { + const permResult = await cropAndCompare(newPermutations[i], oldPermutations[i]); + attachmentPromises.push(attachDiffImages(permResult, `Permutation #${i + 1}`)); + if (permResult.diffPixels !== 0) { + permFailures.push(i); + } + } + await Promise.all(attachmentPromises); + expect(permFailures).toEqual([]); + return; + } + + // Always attach for visibility in the Allure report. + await attachDiffImages(result, testDef.description); + + expect(result.diffPixels).toBe(0); + }); +} diff --git a/test/definitions/visual/alert.ts b/test/definitions/visual/alert.ts index 3220c79e7e..a4169741e3 100644 --- a/test/definitions/visual/alert.ts +++ b/test/definitions/visual/alert.ts @@ -21,11 +21,13 @@ const suite: TestSuite = { description: `width ${width}px`, tests: [ { + configuration: { width }, description: 'permutations', path: 'alert/permutations', screenshotType: 'permutations' as const, }, { + configuration: { width }, description: 'custom types', path: 'alert/style-custom-types', screenshotType: 'screenshotArea' as const, diff --git a/test/visual/action-card.test.ts b/test/visual/action-card.test.ts new file mode 100644 index 0000000000..9e62b89ebf --- /dev/null +++ b/test/visual/action-card.test.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually. +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/action-card'; + +runTestSuites([suite]); diff --git a/test/visual/alert.test.ts b/test/visual/alert.test.ts new file mode 100644 index 0000000000..95af4acc78 --- /dev/null +++ b/test/visual/alert.test.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// Auto-generated by build-tools/visual/generate-tests.js — do not edit manually. +import { runTestSuites } from '../definitions/utils'; +import suite from '../definitions/visual/alert'; + +runTestSuites([suite]); diff --git a/tsconfig.test-definitions.json b/tsconfig.test-definitions.json index 30e82b4047..3f5961a78b 100644 --- a/tsconfig.test-definitions.json +++ b/tsconfig.test-definitions.json @@ -16,5 +16,5 @@ "skipLibCheck": true }, "include": ["test/definitions", "test/types.ts"], - "exclude": [] + "exclude": ["test/definitions/utils.ts"] } diff --git a/tsconfig.visual.json b/tsconfig.visual.json new file mode 100644 index 0000000000..be61d962ef --- /dev/null +++ b/tsconfig.visual.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.integ.json", + "include": ["test/definitions/utils.ts", "test/visual/**/*.test.ts", "types"] +}