Skip to content
Draft
20 changes: 18 additions & 2 deletions .github/workflows/e2e-v2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -428,12 +428,28 @@ jobs:
with:
model: ${{ env.IOS_DEVICE }}
os_version: ${{ env.IOS_VERSION }}
# Cirrus Labs Tart VMs need more time to fully boot the simulator before
# Maestro can connect; without this the boot races with driver startup.
wait_for_boot: true
# Skip erasing the simulator before boot — each Maestro flow already
# reinstalls the app via clearState, and the erase adds overhead that
# makes the simulator less stable on nested-virtualisation Tart VMs.
erase_before_boot: false

- name: Warm up iOS simulator
if: ${{ matrix.platform == 'ios' }}
run: |
# Tart VMs are very slow right after boot. Launch a stock app so
# that SpringBoard, backboardd, and other system services finish
# their post-boot initialisation before Maestro tries to connect.
xcrun simctl launch booted com.apple.Preferences || true
sleep 5
xcrun simctl terminate booted com.apple.Preferences || true

- name: Run tests on iOS
if: ${{ matrix.platform == 'ios' }}
env:
# Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time)
MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000
MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000
run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test

- name: Upload logs
Expand Down
69 changes: 57 additions & 12 deletions dev-packages/e2e-tests/cli.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -290,20 +290,50 @@ if (actions.includes('test')) {
if (!sentryAuthToken) {
console.log('Skipping maestro test due to unavailable or empty SENTRY_AUTH_TOKEN');
} else {
const maxAttempts = 3;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have mixed feelings about it because it basically is an attempt to fix the issue with flakiness by trying it three times to run the same test. I also wonder if it works considering that the flakiness is consistent + it doesn't seem like it guarantees to work.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good point @alwx 👍
I don't like this much either because it's a workaround. I don't have other ideas on this at this point though. It seems that the tests have started failing randomly (not always the same test fails) thus the retries attempt to overcome this by making individual automatic test retries rather than manually rerunning the whole test.

const maestroDir = path.join(e2eDir, 'maestro');
const flowFiles = fs.readdirSync(maestroDir)
.filter(f => f.endsWith('.yml') && !fs.statSync(path.join(maestroDir, f)).isDirectory())
.sort();

console.log(`Found ${flowFiles.length} test flows: ${flowFiles.join(', ')}`);

const results = [];

try {
execSync(
`maestro test maestro \
--env=APP_ID="${appId}" \
--env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
--debug-output maestro-logs \
--flatten-debug-output`,
{
stdio: 'inherit',
cwd: e2eDir,
},
);
for (const flowFile of flowFiles) {
const flowName = flowFile.replace('.yml', '');
let passed = false;

for (let attempt = 1; attempt <= maxAttempts; attempt++) {
const label = `[${flowName}] Attempt ${attempt}/${maxAttempts}`;
console.log(`\n${'='.repeat(60)}\n${label}\n${'='.repeat(60)}`);
try {
execFileSync('maestro', [
'test', `maestro/${flowFile}`,
'--env', `APP_ID=${appId}`,
'--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`,
'--debug-output', 'maestro-logs',
'--flatten-debug-output',
], {
stdio: 'inherit',
cwd: e2eDir,
});
console.log(`${label} — PASSED`);
passed = true;
break;
} catch (error) {
console.error(`${label} — FAILED`);
if (attempt < maxAttempts) {
console.log(`Retrying ${flowName}…`);
}
}
}

results.push({ flowName, passed });
}
} finally {
// Always redact sensitive data, even if the test fails
// Always redact sensitive data, even if a test fails
const redactScript = `
if [[ "$(uname)" == "Darwin" ]]; then
find ./maestro-logs -type f -exec sed -i '' "s/${sentryAuthToken}/[REDACTED]/g" {} +
Expand All @@ -320,5 +350,20 @@ if (actions.includes('test')) {
console.warn('Failed to redact sensitive data from logs:', error.message);
}
}

// Print summary
console.log(`\n${'='.repeat(60)}\nTest Summary\n${'='.repeat(60)}`);
const failed = [];
for (const { flowName, passed } of results) {
const icon = passed ? 'PASS' : 'FAIL';
console.log(` ${icon} ${flowName}`);
if (!passed) failed.push(flowName);
}

if (failed.length > 0) {
console.error(`\n${failed.length}/${results.length} flows failed after ${maxAttempts} attempts: ${failed.join(', ')}`);
process.exit(1);
}
console.log(`\nAll ${results.length} flows passed.`);
}
}
7 changes: 6 additions & 1 deletion dev-packages/e2e-tests/maestro/crash.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ jsEngine: graaljs
- runFlow: utils/launchTestAppClear.yml
- tapOn: "Crash"

- launchApp
# Use clearState to reinstall the app after the intentional crash.
# Without clearState, Sentry reads the pending crash report on relaunch and
# crashes immediately (~82ms), which then triggers iOS crash-loop protection
# and causes the next test in the suite to also fail.
- launchApp:
clearState: true

- runFlow: utils/assertTestReady.yml
Loading