diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index b599dba896..0abaee3d32 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -428,12 +428,28 @@ jobs: with: model: ${{ env.IOS_DEVICE }} os_version: ${{ env.IOS_VERSION }} + # Cirrus Labs Tart VMs need more time to fully boot the simulator before + # Maestro can connect; without this the boot races with driver startup. + wait_for_boot: true + # Skip erasing the simulator before boot — each Maestro flow already + # reinstalls the app via clearState, and the erase adds overhead that + # makes the simulator less stable on nested-virtualisation Tart VMs. + erase_before_boot: false + + - name: Warm up iOS simulator + if: ${{ matrix.platform == 'ios' }} + run: | + # Tart VMs are very slow right after boot. Launch a stock app so + # that SpringBoard, backboardd, and other system services finish + # their post-boot initialisation before Maestro tries to connect. + xcrun simctl launch booted com.apple.Preferences || true + sleep 5 + xcrun simctl terminate booted com.apple.Preferences || true - name: Run tests on iOS if: ${{ matrix.platform == 'ios' }} env: - # Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time) - MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000 + MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test - name: Upload logs diff --git a/dev-packages/e2e-tests/cli.mjs b/dev-packages/e2e-tests/cli.mjs index fded8479b3..da6ca500e2 100755 --- a/dev-packages/e2e-tests/cli.mjs +++ b/dev-packages/e2e-tests/cli.mjs @@ -290,20 +290,50 @@ if (actions.includes('test')) { if (!sentryAuthToken) { console.log('Skipping maestro test due to unavailable or empty SENTRY_AUTH_TOKEN'); } else { + const maxAttempts = 3; + const maestroDir = path.join(e2eDir, 'maestro'); + const flowFiles = fs.readdirSync(maestroDir) + .filter(f => f.endsWith('.yml') && !fs.statSync(path.join(maestroDir, f)).isDirectory()) + .sort(); + + console.log(`Found ${flowFiles.length} test flows: ${flowFiles.join(', ')}`); + + const results = []; + try { - execSync( - `maestro test maestro \ - --env=APP_ID="${appId}" \ - --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \ - --debug-output maestro-logs \ - --flatten-debug-output`, - { - stdio: 'inherit', - cwd: e2eDir, - }, - ); + for (const flowFile of flowFiles) { + const flowName = flowFile.replace('.yml', ''); + let passed = false; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const label = `[${flowName}] Attempt ${attempt}/${maxAttempts}`; + console.log(`\n${'='.repeat(60)}\n${label}\n${'='.repeat(60)}`); + try { + execFileSync('maestro', [ + 'test', `maestro/${flowFile}`, + '--env', `APP_ID=${appId}`, + '--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`, + '--debug-output', 'maestro-logs', + '--flatten-debug-output', + ], { + stdio: 'inherit', + cwd: e2eDir, + }); + console.log(`${label} — PASSED`); + passed = true; + break; + } catch (error) { + console.error(`${label} — FAILED`); + if (attempt < maxAttempts) { + console.log(`Retrying ${flowName}…`); + } + } + } + + results.push({ flowName, passed }); + } } finally { - // Always redact sensitive data, even if the test fails + // Always redact sensitive data, even if a test fails const redactScript = ` if [[ "$(uname)" == "Darwin" ]]; then find ./maestro-logs -type f -exec sed -i '' "s/${sentryAuthToken}/[REDACTED]/g" {} + @@ -320,5 +350,20 @@ if (actions.includes('test')) { console.warn('Failed to redact sensitive data from logs:', error.message); } } + + // Print summary + console.log(`\n${'='.repeat(60)}\nTest Summary\n${'='.repeat(60)}`); + const failed = []; + for (const { flowName, passed } of results) { + const icon = passed ? 'PASS' : 'FAIL'; + console.log(` ${icon} ${flowName}`); + if (!passed) failed.push(flowName); + } + + if (failed.length > 0) { + console.error(`\n${failed.length}/${results.length} flows failed after ${maxAttempts} attempts: ${failed.join(', ')}`); + process.exit(1); + } + console.log(`\nAll ${results.length} flows passed.`); } } diff --git a/dev-packages/e2e-tests/maestro/crash.yml b/dev-packages/e2e-tests/maestro/crash.yml index 4a2c41675f..2cab0ff2d2 100644 --- a/dev-packages/e2e-tests/maestro/crash.yml +++ b/dev-packages/e2e-tests/maestro/crash.yml @@ -4,6 +4,11 @@ jsEngine: graaljs - runFlow: utils/launchTestAppClear.yml - tapOn: "Crash" -- launchApp +# Use clearState to reinstall the app after the intentional crash. +# Without clearState, Sentry reads the pending crash report on relaunch and +# crashes immediately (~82ms), which then triggers iOS crash-loop protection +# and causes the next test in the suite to also fail. +- launchApp: + clearState: true - runFlow: utils/assertTestReady.yml