From 944e5597b51325479de425ae1c491610dd6cd1a7 Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Tue, 3 Mar 2026 15:26:13 +0100 Subject: [PATCH 1/7] fix(ci): Fix iOS E2E flakiness on Cirrus Labs runners After the react-native-test job was moved from GitHub-hosted macos-26 to Cirrus Labs Tart VMs (macos-tahoe-xcode:26.2.0), iOS simulators take longer to fully boot in the new virtualised environment. With `wait_for_boot` defaulting to false, Maestro was racing to connect before the simulator was ready, causing different failures on each run. - Add `wait_for_boot: true` to `futureware-tech/simulator-action` so the job blocks until the simulator has fully completed booting before Maestro connects. - Bump `MAESTRO_DRIVER_STARTUP_TIMEOUT` from 120s to 180s to give additional headroom for the Cirrus Labs runner environment. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/e2e-v2.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index b599dba896..7d11fd480a 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -428,12 +428,15 @@ jobs: with: model: ${{ env.IOS_DEVICE }} os_version: ${{ env.IOS_VERSION }} + # Cirrus Labs Tart VMs need more time to fully boot the simulator before + # Maestro can connect; without this the boot races with driver startup. + wait_for_boot: true - name: Run tests on iOS if: ${{ matrix.platform == 'ios' }} env: # Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time) - MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000 + MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test - name: Upload logs From e11b7706170e2cd4e3e11b7b67ca958404f1aae2 Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Tue, 3 Mar 2026 16:10:54 +0100 Subject: [PATCH 2/7] fix(e2e): Prevent crash-loop after nativeCrash test on iOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After crash.yml taps "Crash" (Sentry.nativeCrash()), the plain `launchApp` (without clearState) causes the app to crash immediately on relaunch (~82ms) because the Sentry SDK reads the pending crash report during initialisation and hits a failure path. This writes a second crash report on top of the first, triggering iOS's simulator crash-loop guard for the bundle ID. The cascade: 1. nativeCrash → crash report #1 written 2. launchApp (no clearState) → app crashes on startup → crash report #2 3. Next test (captureMessage) gets the crash-loop ban → instant exit on launch Fix: add `clearState: true` to the post-crash launchApp so Maestro reinstalls the app, clearing both the crash report and the crash-loop state before assertTestReady runs. Co-Authored-By: Claude Sonnet 4.6 --- dev-packages/e2e-tests/maestro/crash.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/maestro/crash.yml b/dev-packages/e2e-tests/maestro/crash.yml index 4a2c41675f..2cab0ff2d2 100644 --- a/dev-packages/e2e-tests/maestro/crash.yml +++ b/dev-packages/e2e-tests/maestro/crash.yml @@ -4,6 +4,11 @@ jsEngine: graaljs - runFlow: utils/launchTestAppClear.yml - tapOn: "Crash" -- launchApp +# Use clearState to reinstall the app after the intentional crash. +# Without clearState, Sentry reads the pending crash report on relaunch and +# crashes immediately (~82ms), which then triggers iOS crash-loop protection +# and causes the next test in the suite to also fail. +- launchApp: + clearState: true - runFlow: utils/assertTestReady.yml From 06792e16ebf15f28055555a5b25d0b11c08547ed Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Tue, 3 Mar 2026 17:06:57 +0100 Subject: [PATCH 3/7] fix(ci): Add simulator warm-up and disable erase_before_boot for Tart VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The iOS E2E tests have been consistently failing since the migration to Cirrus Labs Tart VMs (c1cade43). The nested virtualisation makes the simulator slower to stabilise, causing Maestro's XCTest driver to lose communication with the app on first launches. Two fixes: 1. Set erase_before_boot: false — each Maestro flow already reinstalls the app via clearState, so erasing the entire simulator is redundant and adds overhead that destabilises the simulator on Tart VMs. 2. Add a warm-up step that launches and terminates Settings.app so that SpringBoard and other system services finish post-boot initialisation before Maestro connects. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/e2e-v2.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index 7d11fd480a..e352cbdedb 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -431,6 +431,21 @@ jobs: # Cirrus Labs Tart VMs need more time to fully boot the simulator before # Maestro can connect; without this the boot races with driver startup. wait_for_boot: true + # Skip erasing the simulator before boot — each Maestro flow already + # reinstalls the app via clearState, and the erase adds overhead that + # makes the simulator less stable on nested-virtualisation Tart VMs. + erase_before_boot: false + + - name: Warm up iOS simulator + if: ${{ matrix.platform == 'ios' }} + run: | + # Launch and terminate a stock app so that SpringBoard, backboardd, + # and other system services finish their post-boot initialisation. + # Without this warm-up Maestro's XCTest driver often fails to + # communicate with the app on first launch inside Tart VMs. + xcrun simctl launch booted com.apple.Preferences + sleep 3 + xcrun simctl terminate booted com.apple.Preferences - name: Run tests on iOS if: ${{ matrix.platform == 'ios' }} From 54daf491cfaeeda1a3b146f4dab2144c14e041cc Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Tue, 3 Mar 2026 17:43:07 +0100 Subject: [PATCH 4/7] fix(ci): Retry iOS E2E suite up to 3 times on Tart VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cirrus Labs Tart VMs intermittently fail individual app launches — the app process exits before the JS bundle finishes loading, causing Maestro to report "App crashed or stopped". A single retry of the full suite is the most reliable way to absorb this flakiness. Also increased the warmup sleep from 3s to 5s to give SpringBoard more time to settle on the slow nested-virtualisation runners. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/e2e-v2.yml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index e352cbdedb..e4f4b204ea 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -439,20 +439,36 @@ jobs: - name: Warm up iOS simulator if: ${{ matrix.platform == 'ios' }} run: | - # Launch and terminate a stock app so that SpringBoard, backboardd, - # and other system services finish their post-boot initialisation. - # Without this warm-up Maestro's XCTest driver often fails to - # communicate with the app on first launch inside Tart VMs. + # Tart VMs are very slow right after boot. Launch a stock app so + # that SpringBoard, backboardd, and other system services finish + # their post-boot initialisation before Maestro tries to connect. xcrun simctl launch booted com.apple.Preferences - sleep 3 + sleep 5 xcrun simctl terminate booted com.apple.Preferences - name: Run tests on iOS if: ${{ matrix.platform == 'ios' }} env: - # Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time) MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 - run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test + run: | + # Retry the full suite up to 3 times. Cirrus Labs Tart VMs + # occasionally fail individual app launches (the app process + # exits before the JS bundle finishes loading), so a retry is + # the most reliable way to get a green run. + for attempt in 1 2 3; do + echo "::group::Attempt $attempt of 3" + if ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test; then + echo "::endgroup::" + echo "Tests passed on attempt $attempt" + exit 0 + fi + echo "::endgroup::" + if [ "$attempt" -lt 3 ]; then + echo "Tests failed on attempt $attempt — retrying…" + fi + done + echo "Tests failed after 3 attempts" + exit 1 - name: Upload logs if: ${{ always() }} From b2626106018297b3d31586868bb525a80bd11f70 Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Tue, 3 Mar 2026 18:52:55 +0100 Subject: [PATCH 5/7] fix(e2e): Retry each Maestro flow individually up to 3 times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of retrying the entire test suite, run each flow file individually with up to 3 attempts. This is more effective because different flows fail randomly on Tart VMs — retrying only the failed flow is faster and avoids re-running flows that already passed. The CLI now: 1. Lists all .yml files in the maestro/ directory 2. Runs each flow with `maestro test ` 3. On failure, retries the same flow up to 2 more times 4. Prints a summary of all results at the end Removes the suite-level retry wrapper from the workflow since per-flow retries in the CLI are more targeted. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/e2e-v2.yml | 20 +--------- dev-packages/e2e-tests/cli.mjs | 70 ++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index e4f4b204ea..17cd04cbed 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -450,25 +450,7 @@ jobs: if: ${{ matrix.platform == 'ios' }} env: MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 - run: | - # Retry the full suite up to 3 times. Cirrus Labs Tart VMs - # occasionally fail individual app launches (the app process - # exits before the JS bundle finishes loading), so a retry is - # the most reliable way to get a green run. - for attempt in 1 2 3; do - echo "::group::Attempt $attempt of 3" - if ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test; then - echo "::endgroup::" - echo "Tests passed on attempt $attempt" - exit 0 - fi - echo "::endgroup::" - if [ "$attempt" -lt 3 ]; then - echo "Tests failed on attempt $attempt — retrying…" - fi - done - echo "Tests failed after 3 attempts" - exit 1 + run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test - name: Upload logs if: ${{ always() }} diff --git a/dev-packages/e2e-tests/cli.mjs b/dev-packages/e2e-tests/cli.mjs index fded8479b3..6c666f1d50 100755 --- a/dev-packages/e2e-tests/cli.mjs +++ b/dev-packages/e2e-tests/cli.mjs @@ -290,20 +290,51 @@ if (actions.includes('test')) { if (!sentryAuthToken) { console.log('Skipping maestro test due to unavailable or empty SENTRY_AUTH_TOKEN'); } else { + const maxAttempts = 3; + const maestroDir = path.join(e2eDir, 'maestro'); + const flowFiles = fs.readdirSync(maestroDir) + .filter(f => f.endsWith('.yml') && !fs.statSync(path.join(maestroDir, f)).isDirectory()) + .sort(); + + console.log(`Found ${flowFiles.length} test flows: ${flowFiles.join(', ')}`); + + const results = []; + try { - execSync( - `maestro test maestro \ - --env=APP_ID="${appId}" \ - --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \ - --debug-output maestro-logs \ - --flatten-debug-output`, - { - stdio: 'inherit', - cwd: e2eDir, - }, - ); + for (const flowFile of flowFiles) { + const flowName = flowFile.replace('.yml', ''); + let passed = false; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const label = `[${flowName}] Attempt ${attempt}/${maxAttempts}`; + console.log(`\n${'='.repeat(60)}\n${label}\n${'='.repeat(60)}`); + try { + execSync( + `maestro test "maestro/${flowFile}" \ + --env=APP_ID="${appId}" \ + --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \ + --debug-output maestro-logs \ + --flatten-debug-output`, + { + stdio: 'inherit', + cwd: e2eDir, + }, + ); + console.log(`${label} — PASSED`); + passed = true; + break; + } catch (error) { + console.error(`${label} — FAILED`); + if (attempt < maxAttempts) { + console.log(`Retrying ${flowName}…`); + } + } + } + + results.push({ flowName, passed }); + } } finally { - // Always redact sensitive data, even if the test fails + // Always redact sensitive data, even if a test fails const redactScript = ` if [[ "$(uname)" == "Darwin" ]]; then find ./maestro-logs -type f -exec sed -i '' "s/${sentryAuthToken}/[REDACTED]/g" {} + @@ -320,5 +351,20 @@ if (actions.includes('test')) { console.warn('Failed to redact sensitive data from logs:', error.message); } } + + // Print summary + console.log(`\n${'='.repeat(60)}\nTest Summary\n${'='.repeat(60)}`); + const failed = []; + for (const { flowName, passed } of results) { + const icon = passed ? 'PASS' : 'FAIL'; + console.log(` ${icon} ${flowName}`); + if (!passed) failed.push(flowName); + } + + if (failed.length > 0) { + console.error(`\n${failed.length}/${results.length} flows failed after ${maxAttempts} attempts: ${failed.join(', ')}`); + process.exit(1); + } + console.log(`\nAll ${results.length} flows passed.`); } } From f7cb8905794818ce668a8f0d4cfa44eaa1523ea7 Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Tue, 3 Mar 2026 19:32:08 +0100 Subject: [PATCH 6/7] fix(e2e): Use execFileSync to avoid shell injection in maestro command Address CodeQL finding by using execFileSync with an argument array instead of execSync with a template string. This avoids shell interpolation of filesystem-sourced flow file names. Co-Authored-By: Claude Opus 4.6 --- dev-packages/e2e-tests/cli.mjs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/dev-packages/e2e-tests/cli.mjs b/dev-packages/e2e-tests/cli.mjs index 6c666f1d50..da6ca500e2 100755 --- a/dev-packages/e2e-tests/cli.mjs +++ b/dev-packages/e2e-tests/cli.mjs @@ -309,17 +309,16 @@ if (actions.includes('test')) { const label = `[${flowName}] Attempt ${attempt}/${maxAttempts}`; console.log(`\n${'='.repeat(60)}\n${label}\n${'='.repeat(60)}`); try { - execSync( - `maestro test "maestro/${flowFile}" \ - --env=APP_ID="${appId}" \ - --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \ - --debug-output maestro-logs \ - --flatten-debug-output`, - { - stdio: 'inherit', - cwd: e2eDir, - }, - ); + execFileSync('maestro', [ + 'test', `maestro/${flowFile}`, + '--env', `APP_ID=${appId}`, + '--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`, + '--debug-output', 'maestro-logs', + '--flatten-debug-output', + ], { + stdio: 'inherit', + cwd: e2eDir, + }); console.log(`${label} — PASSED`); passed = true; break; From 4d9b775ffc54231467582b731c4c514a696db74e Mon Sep 17 00:00:00 2001 From: Antonis Lilis Date: Wed, 4 Mar 2026 11:45:29 +0100 Subject: [PATCH 7/7] fix(ci): Add || true to simulator warm-up commands The warm-up step is best-effort and should not fail the build if the Preferences app fails to launch or terminate. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/e2e-v2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index 17cd04cbed..0abaee3d32 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -442,9 +442,9 @@ jobs: # Tart VMs are very slow right after boot. Launch a stock app so # that SpringBoard, backboardd, and other system services finish # their post-boot initialisation before Maestro tries to connect. - xcrun simctl launch booted com.apple.Preferences + xcrun simctl launch booted com.apple.Preferences || true sleep 5 - xcrun simctl terminate booted com.apple.Preferences + xcrun simctl terminate booted com.apple.Preferences || true - name: Run tests on iOS if: ${{ matrix.platform == 'ios' }}