From 2b8b544414948a8ccdae31ae5e9e7e29f17d4da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:30:29 +0200 Subject: [PATCH 01/20] automatic tests --- .gitignore | 2 + e2e/README.md | 65 +++ e2e/in-app-tutorials.spec.js | 36 ++ e2e/lib/gdevelopEditor.js | 92 ++++ e2e/lib/tutorialPlayer.js | 502 ++++++++++++++++++ e2e/playwright.config.js | 65 +++ package-lock.json | 74 +++ package.json | 3 + .../check-in-app-tutorial-selectors.spec.js | 208 ++++++++ scripts/check-in-app-tutorial-selectors.js | 107 ++++ scripts/lib/InAppTutorialSelectorChecker.js | 442 +++++++++++++++ 11 files changed, 1596 insertions(+) create mode 100644 e2e/README.md create mode 100644 e2e/in-app-tutorials.spec.js create mode 100644 e2e/lib/gdevelopEditor.js create mode 100644 e2e/lib/tutorialPlayer.js create mode 100644 e2e/playwright.config.js create mode 100644 scripts/__tests__/check-in-app-tutorial-selectors.spec.js create mode 100644 scripts/check-in-app-tutorial-selectors.js create mode 100644 scripts/lib/InAppTutorialSelectorChecker.js diff --git a/.gitignore b/.gitignore index 5ce736c..b119814 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .DS_Store /dist *.autosave +/test-results +/playwright-report diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..d5dddaa --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,65 @@ +# In-app tutorials end-to-end tests + +These tests play every in-app tutorial of this repository against a real +GDevelop editor (web build), like a user would: for each step of a tutorial +flow, the expected action is performed (click the highlighted element, fill the +input, drag an object to the scene, launch a preview...), and the test checks +that the tutorial orchestrator advances to the next step. If a step cannot be +completed, the tutorial is broken and the test fails with the step index, the +selector involved and a screenshot/trace. + +They complement the static selector check +(`npm run check-in-app-tutorial-selectors`), which quickly catches selectors +referencing elements that no longer exist in the editor, without running it. + +## How it works + +- The editor is opened with the `in-app-tutorial-test-mode` URL parameter: the + orchestrator (see `InAppTutorialOrchestrator.js` in GDevelop) then exposes + its current state (step index, interpolated element to highlight, trigger) + on `window.inAppTutorialTestModeState`. +- Requests to the in-app tutorials API/CDN are intercepted so the **local** + tutorial files (with meta steps expanded) are used instead of the deployed + ones. Initial project templates are still fetched from the real CDN, so a + broken template URL is caught too. +- The tutorial is started through the regular + `?initial-dialog=guided-lesson&tutorial-id=` flow. + +## Running locally + +Requirements: a GDevelop checkout with `npm install` ran in `newIDE/app`, +located next to this repository (or set `GDEVELOP_ROOT_PATH`), and +`npx playwright install chromium` ran once in this repository. + +```bash +# Run all tutorials (starts the editor dev server if not already running): +npm run test-in-app-tutorials + +# Against an already-running editor, for a single tutorial: +GDEVELOP_EDITOR_URL=http://localhost:3000 TUTORIAL_IDS=timer npm run test-in-app-tutorials + +# Inspect a failure: +npx playwright show-trace test-results//trace.zip +``` + +## Environment variables + +| Variable | Description | +| -------------------- | ------------------------------------------------------------------ | +| `GDEVELOP_EDITOR_URL`| URL of a running editor (default `http://localhost:3000`). | +| `GDEVELOP_ROOT_PATH` | Path to the GDevelop checkout (default: `./GDevelop` or `../GDevelop`). | +| `TUTORIAL_IDS` | Comma-separated tutorial ids to test (default: all). | + +## How steps are played + +| Trigger | Action performed | +| ---------------------- | ---------------------------------------------------------------------- | +| `presenceOfElement` | Click the highlighted element, or type the text in bold/quotes from the tooltip if the element is a text input (search bars). | +| `absenceOfElement` | Click the highlighted element (e.g. an Apply/OK button). | +| `editorIsActive` | Click the highlighted editor tab. | +| `valueEquals` | Fill the highlighted field with the expected value. | +| `valueHasChanged` | Fill the highlighted field with a test value. | +| `instanceAddedOnScene` | Drag the highlighted object list item onto the scene canvas. | +| `objectAddedInLayout` | Click the highlighted element (e.g. the asset store add button). | +| `previewLaunched` | Click the preview button and wait for the preview window. | +| `clickOnTooltipButton` | Click the tooltip button with the expected label. | diff --git a/e2e/in-app-tutorials.spec.js b/e2e/in-app-tutorials.spec.js new file mode 100644 index 0000000..cd56862 --- /dev/null +++ b/e2e/in-app-tutorials.spec.js @@ -0,0 +1,36 @@ +// @ts-check +/** + * Plays every in-app tutorial of this repository against a running GDevelop + * editor, and fails if a tutorial step cannot be completed. + * + * Run with: npx playwright test --config e2e/playwright.config.js + * Filter tutorials with: TUTORIAL_IDS=timer,healthBar + */ +const { test } = require('@playwright/test'); +const { + loadAllTutorials, + serveLocalTutorials, + startTutorial, +} = require('./lib/gdevelopEditor'); +const { playTutorial } = require('./lib/tutorialPlayer'); + +const allTutorials = loadAllTutorials(); +const tutorialIdsFilter = process.env.TUTORIAL_IDS + ? process.env.TUTORIAL_IDS.split(',').map((id) => id.trim()) + : null; +const tutorials = tutorialIdsFilter + ? allTutorials.filter((tutorial) => tutorialIdsFilter.includes(tutorial.id)) + : allTutorials; + +for (const tutorial of tutorials) { + test(`in-app tutorial: ${tutorial.id}`, async ({ page, context }) => { + await serveLocalTutorials(context, allTutorials); + await startTutorial(page, tutorial.id); + await playTutorial({ + page, + context, + tutorial: tutorial.content, + log: (message) => console.log(`[${tutorial.id}] ${message}`), + }); + }); +} diff --git a/e2e/lib/gdevelopEditor.js b/e2e/lib/gdevelopEditor.js new file mode 100644 index 0000000..d8bc82e --- /dev/null +++ b/e2e/lib/gdevelopEditor.js @@ -0,0 +1,92 @@ +// @ts-check +/** + * Helpers to serve the local tutorials to a running GDevelop editor and start + * a tutorial like a user would. + */ +const path = require('path'); +const fs = require('fs'); +const { InAppTutorial } = require('../../scripts/lib/InAppTutorial'); + +const inAppTutorialsPath = path.join(__dirname, '../../tutorials/in-app'); + +/** + * Loads all local tutorials, with their meta steps expanded the same way they + * are expanded at deploy time. + * @returns {Array<{ id: string, shortHeader: any, content: any }>} + */ +const loadAllTutorials = () => { + return fs + .readdirSync(inAppTutorialsPath) + .filter((fileName) => fileName.endsWith('.json')) + .map((fileName) => { + const tutorial = new InAppTutorial( + path.join(inAppTutorialsPath, fileName) + ); + tutorial.processFlowMetaSteps(); + return { + id: tutorial.id, + shortHeader: tutorial.buildShortHeader(), + content: JSON.parse(tutorial.toString()), + }; + }); +}; + +/** + * Intercepts the editor's network requests about in-app tutorials so that the + * local tutorials are used instead of the deployed ones. Requests to the + * initial project templates (`in-app-tutorials/templates/...`) are not + * intercepted: a broken template URL is a real breakage the test should catch. + * @param {import('@playwright/test').BrowserContext} context + * @param {Array<{ id: string, shortHeader: any, content: any }>} tutorials + */ +const serveLocalTutorials = async (context, tutorials) => { + await context.route('**/in-app-tutorial-short-header*', (route) => + route.fulfill({ json: tutorials.map((tutorial) => tutorial.shortHeader) }) + ); + // Matches the short headers' contentUrl + // (https://resources.gdevelop-app.com/in-app-tutorials/.json). `*` does + // not match `/`, so template files in subfolders are not intercepted. + await context.route('**/in-app-tutorials/*.json', (route, request) => { + const id = path.basename(new URL(request.url()).pathname, '.json'); + const tutorial = tutorials.find((tutorial) => tutorial.id === id); + if (!tutorial) return route.continue(); + return route.fulfill({ json: tutorial.content }); + }); +}; + +/** + * Opens the editor and starts the given tutorial through the same flow as a + * user clicking a lesson card: the start dialog is opened via the + * `initial-dialog=guided-lesson` URL argument, and the dialog's primary button + * is clicked (which creates the template project and starts the tutorial). + * @param {import('@playwright/test').Page} page + * @param {string} tutorialId + */ +const startTutorial = async (page, tutorialId) => { + await page.goto( + `/?initial-dialog=guided-lesson&tutorial-id=${tutorialId}&in-app-tutorial-test-mode` + ); + // The start dialog can take a while to show: the editor loads WebAssembly + // and fetches the tutorial short headers first. + const startButton = page.getByRole('button', { name: "Let's go" }); + await startButton.click({ timeout: 3 * 60 * 1000 }); + // The tutorial is started once the orchestrator exposes its state (the + // template project has to be downloaded and opened first). + await page.waitForFunction( + () => !!window['inAppTutorialTestModeState'], + undefined, + { timeout: 3 * 60 * 1000 } + ); +}; + +/** @param {import('@playwright/test').Page} page */ +const getTutorialState = async (page) => { + return await page.evaluate(() => window['inAppTutorialTestModeState']); +}; + +module.exports = { + loadAllTutorials, + serveLocalTutorials, + startTutorial, + getTutorialState, +}; diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js new file mode 100644 index 0000000..5f51802 --- /dev/null +++ b/e2e/lib/tutorialPlayer.js @@ -0,0 +1,502 @@ +// @ts-check +/** + * Plays an in-app tutorial like a user would: for each step, performs the + * action expected by the step's `nextStepTrigger` and checks that the + * orchestrator advances to the next step. If it does not, the tutorial is + * considered broken at this step. + */ +const { getTutorialState } = require('./gdevelopEditor'); + +/** How long to wait for the orchestrator to advance after performing an action. + * The orchestrator polls the DOM/project every 0.5-1s, so this includes at + * least a few polling cycles plus UI animations. */ +const STEP_ADVANCE_TIMEOUT_MS = 20 * 1000; +/** How many times to re-perform an action before declaring the step broken. */ +const MAX_ATTEMPTS_PER_STEP = 3; + +/** @param {any} translatedText @returns {string} */ +const getEnglishMessage = (translatedText) => { + if (!translatedText) return ''; + if (typeof translatedText === 'string') return translatedText; + if (translatedText.messageByLocale) { + return translatedText.messageByLocale.en || ''; + } + return ''; +}; + +/** + * Extracts the text a user is asked to type from a step tooltip. Tutorials + * consistently write it in bold ("Search for **Scene timer**.") or between + * quotes ('Search for “layer”.'). + * @param {any} step + * @returns {string | null} + */ +const extractTextToType = (step) => { + const description = getEnglishMessage((step.tooltip || {}).description); + const boldMatch = description.match(/\*\*(.+?)\*\*/); + if (boldMatch) return boldMatch[1]; + const quoteMatch = description.match(/[“"']([^”"']+)[”"']/); + if (quoteMatch) return quoteMatch[1]; + return null; +}; + +/** All the bold texts of a step tooltip, in order. + * @param {any} step + * @returns {string[]} + */ +const extractAllBoldTexts = (step) => { + const description = getEnglishMessage((step.tooltip || {}).description); + return [...description.matchAll(/\*\*(.+?)\*\*/g)].map((match) => match[1]); +}; + +class TutorialStepError extends Error { + /** + * @param {string} message + * @param {{ stepIndex: number, step: any, state: any }} details + */ + constructor(message, details) { + super(message); + this.name = 'TutorialStepError'; + this.details = details; + } +} + +/** + * Rebuilds the editor tab selector for an expected editor, the same way the + * orchestrator does (see getEditorTabSelector in InAppTutorialOrchestrator). + * @param {{ editor: string, scene?: string } | null} expectedEditor + * @returns {string | null} + */ +const getEditorTabSelector = (expectedEditor) => { + if (!expectedEditor) return null; + if (expectedEditor.editor === 'Home') { + return 'button[id="tab-start-page-button"]'; + } + const sceneNameFilter = expectedEditor.scene + ? `[data-scene="${expectedEditor.scene}"]` + : ''; + return `button[id^="tab"][data-type="${ + expectedEditor.editor === 'Scene' ? 'layout' : 'layout-events' + }"]${sceneNameFilter}`; +}; + +/** + * Fills the input designated by the selector with the given value, handling + * fields where the id is on a wrapper rather than on the input itself, as + * well as select fields. + * @param {import('@playwright/test').Page} page + * @param {string} selector + * @param {string} value + */ +const fillField = async (page, selector, value, { anyValue = false } = {}) => { + const element = page.locator(selector).first(); + await element.waitFor({ state: 'visible', timeout: 10 * 1000 }); + const tagName = await element.evaluate((node) => node.tagName); + + /** @param {import('@playwright/test').Locator} select */ + const changeNativeSelect = async (select) => { + if (anyValue) { + // Any different option satisfies a valueHasChanged trigger. + const newValue = await select.evaluate((node) => { + const selectNode = /** @type {HTMLSelectElement} */ (node); + const options = [...selectNode.options]; + const other = options.find( + (option) => option.value !== selectNode.value + ); + return other ? other.value : null; + }); + if (newValue === null) { + throw new Error(`No other option to select in "${selector}".`); + } + await select.selectOption(newValue); + } else { + await select.selectOption(value); + } + }; + + if (tagName === 'SELECT') { + await changeNativeSelect(element); + return; + } + let input; + if (tagName === 'INPUT' || tagName === 'TEXTAREA') { + input = element; + } else { + const nativeSelect = element.locator('select').first(); + if (await nativeSelect.count()) { + await changeNativeSelect(nativeSelect); + return; + } + input = element.locator('input, textarea').first(); + if (!(await input.count())) { + throw new Error(`Could not find an input to fill in "${selector}".`); + } + } + const inputType = await input.evaluate((node) => node.getAttribute('type')); + if (inputType === 'checkbox') { + // Toggle (valueHasChanged) or set (valueEquals "true"/"false") the + // checkbox. + if (!anyValue && (value === 'true' || value === 'false')) { + await input.setChecked(value === 'true', { force: true }); + } else { + await input.click({ force: true }); + } + return; + } + if (inputType === 'hidden') { + // Material-UI select: open the menu and pick the option. + await element.click(); + const options = page.locator( + '[role="listbox"] [role="option"], [role="menu"] li' + ); + if (anyValue) { + await page + .locator( + '[role="listbox"] [role="option"]:not([aria-selected="true"]), [role="menu"] li:not([aria-selected="true"])' + ) + .first() + .click(); + } else { + await options.filter({ hasText: value }).first().click(); + } + return; + } + await input.fill(value); +}; + +/** + * Performs the user action expected by the current step. + * @param {{ + * page: import('@playwright/test').Page, + * context: import('@playwright/test').BrowserContext, + * step: any, + * state: any, + * attempt: number, + * log: (message: string) => void, + * }} options + */ +const performStepAction = async ({ + page, + context, + step, + state, + attempt, + log, +}) => { + const trigger = step.nextStepTrigger || {}; + const highlightedElementSelector = state.elementToHighlightId; + + if (trigger.clickOnTooltipButton) { + const buttonLabel = getEnglishMessage(trigger.clickOnTooltipButton); + const button = page + .locator('#in-app-tutorial-tooltip-displayer') + .getByRole('button', { name: buttonLabel }); + try { + await button.click({ timeout: 5 * 1000 }); + } catch (error) { + // The tooltip can be folded (only the avatar is visible): unfold it by + // clicking the avatar, then click the button. + await page + .locator('#in-app-tutorial-avatar') + .click({ timeout: 5 * 1000 }); + await button.click({ timeout: 5 * 1000 }); + } + return; + } + + if (trigger.previewLaunched) { + const popupPromise = context + .waitForEvent('page', { timeout: STEP_ADVANCE_TIMEOUT_MS }) + .catch(() => null); + await page + .locator(highlightedElementSelector || '#toolbar-preview-button') + .first() + .click({ timeout: 10 * 1000 }); + // Some steps target a split/menu button (e.g. "Launch preview in... > + // 2 previews in 2 windows"): follow the bold texts of the tooltip through + // the menus that opened. + for (const boldText of extractAllBoldTexts(step)) { + const menuItem = page + .getByRole('menuitem', { + name: boldText.replace(/(\.|…)+$/, ''), + }) + .first(); + if (await menuItem.isVisible().catch(() => false)) { + await menuItem.click({ timeout: 5000 }).catch(() => {}); + } + } + // The preview opens in a new page: keep it open (the trigger fires when + // the launch resolves), it is closed by the caller once the step advanced. + await popupPromise; + return; + } + + if (trigger.valueEquals !== undefined) { + if (!highlightedElementSelector) { + throw new Error('A valueEquals step must have an element to highlight.'); + } + await fillField( + page, + highlightedElementSelector, + String(trigger.valueEquals) + ); + return; + } + + if (trigger.valueHasChanged) { + if (!highlightedElementSelector) { + throw new Error( + 'A valueHasChanged step must have an element to highlight.' + ); + } + await fillField(page, highlightedElementSelector, 'Test value 123', { + anyValue: true, + }); + return; + } + + if (trigger.instanceAddedOnScene) { + if (!highlightedElementSelector) { + throw new Error( + 'An instanceAddedOnScene step must have an element to highlight.' + ); + } + const source = page.locator(highlightedElementSelector).first(); + await source.waitFor({ state: 'visible', timeout: 10 * 1000 }); + const canvas = page + .locator('#scene-editor[data-active="true"] canvas') + .first(); + // Manual mouse drag: the objects list uses mouse-based drag and drop onto + // the scene canvas. + const sourceBox = await source.boundingBox(); + const canvasBox = await canvas.boundingBox(); + if (!sourceBox || !canvasBox) { + throw new Error('Could not find the drag source or the scene canvas.'); + } + await page.mouse.move( + sourceBox.x + sourceBox.width / 2, + sourceBox.y + sourceBox.height / 2 + ); + await page.mouse.down(); + // Move in several small steps so that drag events are properly emitted. + await page.mouse.move( + canvasBox.x + canvasBox.width / 2, + canvasBox.y + canvasBox.height / 2, + { steps: 20 } + ); + await page.waitForTimeout(200); + await page.mouse.up(); + return; + } + + // Default action (presenceOfElement, absenceOfElement, editorIsActive, + // objectAddedInLayout): click the highlighted element — unless it is a text + // input (e.g. a search bar): in that case type the text the tooltip asks + // for, so that the expected element (e.g. a search result) appears. + if (highlightedElementSelector) { + const element = page.locator(highlightedElementSelector).first(); + await element.waitFor({ state: 'visible', timeout: 10 * 1000 }); + const isTextInput = await element.evaluate((node) => { + const input = + node.tagName === 'INPUT' || node.tagName === 'TEXTAREA' + ? node + : node.querySelector('input, textarea'); + return ( + !!input && + !['hidden', 'checkbox', 'radio'].includes( + input.getAttribute('type') || 'text' + ) + ); + }); + const textToType = extractTextToType(step); + if (isTextInput && textToType) { + await fillField(page, highlightedElementSelector, textToType); + return; + } + const sceneCanvas = page + .locator('#scene-editor[data-active="true"] canvas') + .first(); + if (step.interactsWithCanvas && (await sceneCanvas.count())) { + // The step also requires clicking something in the scene (e.g. "click + // the terrain"): probe a few points of the scene canvas. The highlighted + // element is often a panel toggle that can already be in the expected + // state, so it is only clicked on retries. + if (attempt > 1) { + await element.click({ timeout: 10 * 1000 }); + } + const canvasBox = await sceneCanvas.boundingBox(); + if (canvasBox) { + const probePoints = [ + [0.3, 0.65], + [0.15, 0.5], + [0.6, 0.85], + ]; + const [fractionX, fractionY] = + probePoints[(attempt - 1) % probePoints.length]; + await page.mouse.click( + canvasBox.x + canvasBox.width * fractionX, + canvasBox.y + canvasBox.height * fractionY + ); + } + return; + } + // Opening the object editor from an objects list item requires a double + // click. Also alternate to a double click on retries, in case a single + // click was not the expected interaction. + const shouldDoubleClick = + (trigger.presenceOfElement === '#object-editor-dialog' && + highlightedElementSelector.includes('data-object-name')) || + attempt % 2 === 0; + if (shouldDoubleClick) { + await element.dblclick({ timeout: 10 * 1000 }); + } else { + await element.click({ timeout: 10 * 1000 }); + } + return; + } + + if (trigger.absenceOfElement || trigger.presenceOfElement) { + // No element to interact with: the expected change may happen on its own + // (or cannot be automated, e.g. a login): just wait. + log('No element to highlight: waiting for the trigger to be satisfied.'); + return; + } + + throw new Error( + `Don't know how to perform the action for trigger: ${JSON.stringify( + trigger + )}.` + ); +}; + +/** + * Waits until the orchestrator's step index changes (or the end dialog is + * displayed). Returns true if it did before the timeout. + * @param {import('@playwright/test').Page} page + * @param {number} currentStepIndex + * @returns {Promise} + */ +const waitForStepChange = async (page, currentStepIndex) => { + try { + await page.waitForFunction( + (stepIndex) => { + const state = window['inAppTutorialTestModeState']; + return ( + !!state && (state.stepIndex !== stepIndex || state.displayEndDialog) + ); + }, + currentStepIndex, + { timeout: STEP_ADVANCE_TIMEOUT_MS } + ); + return true; + } catch (error) { + return false; + } +}; + +/** + * Plays the whole tutorial. Resolves when the end dialog is displayed, throws + * a TutorialStepError when a step cannot be completed. + * @param {{ + * page: import('@playwright/test').Page, + * context: import('@playwright/test').BrowserContext, + * tutorial: any, + * log?: (message: string) => void, + * }} options + */ +const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { + const flow = tutorial.flow; + let lastHandledStepIndex = -1; + let attemptsForCurrentStep = 0; + + for (;;) { + const state = await getTutorialState(page); + if (!state) { + throw new TutorialStepError( + 'The tutorial state is not available anymore (was the tutorial exited?).', + { stepIndex: lastHandledStepIndex, step: null, state: null } + ); + } + if (state.displayEndDialog) { + log(`Tutorial "${tutorial.id}" completed (${flow.length} steps).`); + return; + } + + // Close any extra page (preview window) once the main flow is back in the + // editor. + for (const otherPage of context.pages()) { + if (otherPage !== page && !state.wrongEditorInfoOpen) { + await otherPage.close().catch(() => {}); + } + } + + if (state.stepIndex !== lastHandledStepIndex) { + if (state.stepIndex < lastHandledStepIndex) { + log( + `⚠️ Went back from step ${lastHandledStepIndex} to ${state.stepIndex} (a dialog was probably closed).` + ); + } + lastHandledStepIndex = state.stepIndex; + attemptsForCurrentStep = 0; + } + + const step = flow[state.stepIndex]; + + if (state.wrongEditorInfoOpen) { + const tabSelector = getEditorTabSelector(state.expectedEditor); + log( + `Wrong editor open: switching to ${JSON.stringify( + state.expectedEditor + )}.` + ); + if (tabSelector) { + await page + .locator(tabSelector) + .first() + .click({ timeout: 10 * 1000 }); + await page.waitForTimeout(1500); + continue; + } + } + + attemptsForCurrentStep++; + log( + `Step ${state.stepIndex + 1}/${ + flow.length + } (attempt ${attemptsForCurrentStep}): ` + + `highlight=${ + state.elementToHighlightId || 'none' + } trigger=${JSON.stringify(step.nextStepTrigger || {}).slice(0, 120)}` + ); + + let actionError = null; + try { + await performStepAction({ + page, + context, + step, + state, + attempt: attemptsForCurrentStep, + log, + }); + } catch (error) { + actionError = error; + log(`Action failed: ${error.message}`); + } + + const advanced = await waitForStepChange(page, state.stepIndex); + if (!advanced && attemptsForCurrentStep >= MAX_ATTEMPTS_PER_STEP) { + throw new TutorialStepError( + `Tutorial "${tutorial.id}" is broken at step ${state.stepIndex} ` + + `(highlighted element: ${state.elementToHighlightId || 'none'}, ` + + `trigger: ${JSON.stringify(step.nextStepTrigger || {})})` + + (actionError + ? `. The action could not be performed: ${actionError.message}` + : '. The action was performed but the tutorial did not advance.'), + { stepIndex: state.stepIndex, step, state } + ); + } + } +}; + +module.exports = { playTutorial, TutorialStepError }; diff --git a/e2e/playwright.config.js b/e2e/playwright.config.js new file mode 100644 index 0000000..7b90047 --- /dev/null +++ b/e2e/playwright.config.js @@ -0,0 +1,65 @@ +// @ts-check +const { defineConfig } = require('@playwright/test'); +const path = require('path'); + +/** + * End-to-end tests that play the in-app tutorials against a running GDevelop + * editor (web build). + * + * Environment variables: + * - GDEVELOP_EDITOR_URL: URL of a running editor (default http://localhost:3000). + * If the editor is not running, it is started from GDEVELOP_ROOT_PATH. + * - GDEVELOP_ROOT_PATH: path to a GDevelop checkout with `npm install` ran in + * newIDE/app (default: ./GDevelop or ../GDevelop relative to this repository). + * - TUTORIAL_IDS: comma-separated list of tutorial ids to test (default: all). + */ + +const fs = require('fs'); + +const findGDevelopRootPath = () => { + if (process.env.GDEVELOP_ROOT_PATH) { + return path.resolve(process.cwd(), process.env.GDEVELOP_ROOT_PATH); + } + const candidates = [ + path.join(__dirname, '../GDevelop'), + path.join(__dirname, '../../GDevelop'), + ]; + for (const candidate of candidates) { + if (fs.existsSync(path.join(candidate, 'newIDE/app'))) return candidate; + } + return null; +}; + +const editorUrl = process.env.GDEVELOP_EDITOR_URL || 'http://localhost:3000'; +const gdevelopRootPath = findGDevelopRootPath(); + +module.exports = defineConfig({ + testDir: __dirname, + // Playing a whole tutorial can take a while: each step advance is detected + // by the editor with 0.5-1s polls. + timeout: 10 * 60 * 1000, + // The editor is a heavy page (WebAssembly, PIXI): don't run too many at once. + workers: 2, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [['list'], ['github']] : [['list']], + use: { + baseURL: editorUrl, + viewport: { width: 1600, height: 900 }, + screenshot: 'only-on-failure', + trace: 'retain-on-failure', + }, + webServer: gdevelopRootPath + ? { + command: `npm start --prefix ${path.join( + gdevelopRootPath, + 'newIDE/app' + )}`, + url: editorUrl, + reuseExistingServer: true, + timeout: 20 * 60 * 1000, + env: { BROWSER: 'none' }, + } + : undefined, +}); diff --git a/package-lock.json b/package-lock.json index 07571af..464b3e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "shelljs": "^0.8.4" }, "devDependencies": { + "@playwright/test": "^1.61.1", "@types/jest": "^26.0.23", "@types/minimist": "^1.2.1", "@types/shelljs": "^0.8.8", @@ -898,6 +899,22 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -3700,6 +3717,38 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -5207,6 +5256,15 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "requires": { + "playwright": "1.61.1" + } + }, "@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -7350,6 +7408,22 @@ "find-up": "^4.0.0" } }, + "playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "requires": { + "fsevents": "2.3.2", + "playwright-core": "1.61.1" + } + }, + "playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true + }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", diff --git a/package.json b/package.json index ac3e7be..32447d5 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "description": "Repository of tutorials for GDevelop", "scripts": { "test": "jest scripts/", + "check-in-app-tutorial-selectors": "node scripts/check-in-app-tutorial-selectors.js", + "test-in-app-tutorials": "playwright test --config e2e/playwright.config.js", "format": "prettier --write \"scripts/**/*.ts\" \"scripts/**/*.js\"", "build": "node scripts/generate-database.js", "deploy": "node scripts/deploy.js", @@ -20,6 +22,7 @@ "shelljs": "^0.8.4" }, "devDependencies": { + "@playwright/test": "^1.61.1", "@types/jest": "^26.0.23", "@types/minimist": "^1.2.1", "@types/shelljs": "^0.8.8", diff --git a/scripts/__tests__/check-in-app-tutorial-selectors.spec.js b/scripts/__tests__/check-in-app-tutorial-selectors.spec.js new file mode 100644 index 0000000..57a6ef7 --- /dev/null +++ b/scripts/__tests__/check-in-app-tutorial-selectors.spec.js @@ -0,0 +1,208 @@ +// @ts-check +const { + splitTopLevelCommas, + extractIdTokens, + extractAttributeNames, + extractSelectorUsages, + collectDeclaredDataKeys, + checkCssSelector, + checkTutorial, +} = require('../lib/InAppTutorialSelectorChecker'); + +describe('splitTopLevelCommas', () => { + it('splits a selector list on top-level commas only', () => { + expect(splitTopLevelCommas('#a, #b')).toEqual(['#a', '#b']); + expect( + splitTopLevelCommas( + 'div:is([data-open=true], :not([aria-hidden=true])) #a' + ) + ).toEqual(['div:is([data-open=true], :not([aria-hidden=true])) #a']); + }); +}); + +describe('extractIdTokens', () => { + it('extracts #id, [id=], [id^=], [id$=] tokens', () => { + expect(extractIdTokens('#object-editor-dialog #apply-button')).toEqual([ + 'object-editor-dialog', + 'apply-button', + ]); + expect(extractIdTokens('[id^="scene-item"][data-default="true"]')).toEqual([ + 'scene-item', + ]); + expect( + extractIdTokens('[id$=online-web-button][id^=launch-export]') + ).toEqual(['online-web-button', 'launch-export']); + expect(extractIdTokens('[id="Z Order"]')).toEqual(['Z Order']); + }); +}); + +describe('extractAttributeNames', () => { + it('extracts attribute names but not id', () => { + expect( + extractAttributeNames('#a[data-active="true"][id^=tab][data-open]') + ).toEqual(['data-active', 'data-open']); + }); +}); + +describe('checkCssSelector', () => { + const sourceBlob = [ + 'id="add-new-object-button"', + 'id="data-active"', + 'const id = `parameter-${props.parameterIndex}-expression-field`;', + "data={{ isFiltered: 'true' }}", + ].join('\n'); + + it('accepts an id found verbatim in the sources', () => { + expect(checkCssSelector('#add-new-object-button', sourceBlob)).toEqual([]); + }); + + it('rejects an id absent from the sources', () => { + expect(checkCssSelector('#removed-button', sourceBlob)[0]).toMatch( + /removed-button/ + ); + }); + + it('accepts a selector list when at least one alternative is valid', () => { + expect( + checkCssSelector('#removed-button, #add-new-object-button', sourceBlob) + ).toEqual([]); + }); + + it('accepts dynamic id families when their construction site exists', () => { + expect( + checkCssSelector('#parameter-2-expression-field', sourceBlob) + ).toEqual([]); + }); + + it('rejects dynamic id families when their construction site is gone', () => { + expect(checkCssSelector('#variable-0-name', sourceBlob)[0]).toMatch( + /variable-0-name/ + ); + }); + + it('accepts property-name ids that cannot be checked statically', () => { + expect(checkCssSelector('#FollowOnY', sourceBlob)).toEqual([]); + expect(checkCssSelector('[id="Z Order"]', sourceBlob)).toEqual([]); + }); + + it('accepts known camelCase dataset attributes', () => { + expect( + checkCssSelector( + '#add-new-object-button[data-is-filtered="false"]', + sourceBlob + ) + ).toEqual([]); + }); + + it('rejects unknown dataset attributes', () => { + expect( + checkCssSelector( + '#add-new-object-button[data-default="true"]', + sourceBlob + )[0] + ).toMatch(/data-default/); + }); +}); + +describe('extractSelectorUsages and collectDeclaredDataKeys', () => { + const tutorial = { + id: 'test', + initialProjectData: { gameScene: 'GameScene' }, + flow: [ + { + id: 'Start', + elementToHighlightId: '#a', + nextStepTrigger: { presenceOfElement: '#b' }, + shortcuts: [{ trigger: { absenceOfElement: '#c' } }], + }, + { + elementToHighlightId: 'objectInObjectsList:player', + nextStepTrigger: { instanceAddedOnScene: 'player' }, + mapProjectData: { player: 'sceneLastObjectName:gameScene' }, + }, + ], + }; + + it('extracts selectors from steps and shortcuts', () => { + const usages = extractSelectorUsages(tutorial); + expect(usages.map((usage) => usage.selector)).toEqual([ + '#a', + '#b', + '#c', + 'objectInObjectsList:player', + ]); + expect(usages[2].field).toBe('shortcuts[0].trigger.absenceOfElement'); + }); + + it('collects declared data keys', () => { + expect(Array.from(collectDeclaredDataKeys(tutorial)).sort()).toEqual([ + 'gameScene', + 'player', + ]); + }); +}); + +describe('checkTutorial', () => { + const sourceBlob = 'id="a"\nid="b"\nid="c"'; + + it('validates project data accessors against declared keys', () => { + const tutorial = { + id: 'test', + initialProjectData: { gameScene: 'GameScene' }, + flow: [ + { + elementToHighlightId: 'objectInObjectsList:notDeclared', + nextStepTrigger: { presenceOfElement: '#a' }, + }, + ], + }; + const problems = checkTutorial(tutorial, sourceBlob); + expect(problems).toHaveLength(1); + expect(problems[0].reason).toMatch(/notDeclared/); + }); + + it('validates mapProjectData accessors', () => { + const tutorial = { + id: 'test', + initialProjectData: { gameScene: 'GameScene' }, + flow: [{ mapProjectData: { player: 'sceneLastObjectName:missingKey' } }], + }; + const problems = checkTutorial(tutorial, sourceBlob); + expect(problems).toHaveLength(1); + expect(problems[0].field).toBe('mapProjectData.player'); + }); + + it('validates editorIsActive triggers', () => { + const tutorial = { + id: 'test', + initialProjectData: { gameScene: 'GameScene' }, + flow: [ + { nextStepTrigger: { editorIsActive: 'gameScene:EventsSheet' } }, + { nextStepTrigger: { editorIsActive: 'Home' } }, + { nextStepTrigger: { editorIsActive: 'gameScene:UnknownEditor' } }, + ], + }; + const problems = checkTutorial(tutorial, sourceBlob); + expect(problems).toHaveLength(1); + expect(problems[0].reason).toMatch(/UnknownEditor/); + }); + + it('returns no problem for a fully valid tutorial', () => { + const tutorial = { + id: 'test', + initialProjectData: { gameScene: 'GameScene' }, + editorSwitches: { Start: { editor: 'Scene', scene: 'gameScene' } }, + flow: [ + { + elementToHighlightId: '#a', + nextStepTrigger: { presenceOfElement: '#b' }, + }, + { + elementToHighlightId: 'editorTab:gameScene:EventsSheet', + nextStepTrigger: { editorIsActive: 'gameScene:EventsSheet' }, + }, + ], + }; + expect(checkTutorial(tutorial, sourceBlob)).toEqual([]); + }); +}); diff --git a/scripts/check-in-app-tutorial-selectors.js b/scripts/check-in-app-tutorial-selectors.js new file mode 100644 index 0000000..8d06b6d --- /dev/null +++ b/scripts/check-in-app-tutorial-selectors.js @@ -0,0 +1,107 @@ +// @ts-check +/** + * Checks that every DOM selector used by the in-app tutorials still exists in + * the GDevelop editor sources (newIDE/app/src). This catches the most common + * way tutorials break: an id is renamed or removed in the editor. + * + * Usage: node scripts/check-in-app-tutorial-selectors.js --gdevelop-root-path ../GDevelop + */ +const shell = require('shelljs'); +const path = require('path'); +const fs = require('fs'); +const args = require('minimist')(process.argv.slice(2)); +const { checkTutorial } = require('./lib/InAppTutorialSelectorChecker'); +const { InAppTutorial } = require('./lib/InAppTutorial'); + +const inAppTutorialsPath = path.join(__dirname, '../tutorials/in-app'); + +const findGDevelopRootPath = () => { + if (args['gdevelop-root-path']) { + return path.resolve(process.cwd(), args['gdevelop-root-path']); + } + // Default locations: a GDevelop checkout inside this repository (as done in + // CI) or next to it. + const candidates = [ + path.join(__dirname, '../GDevelop'), + path.join(__dirname, '../../GDevelop'), + ]; + for (const candidate of candidates) { + if (fs.existsSync(path.join(candidate, 'newIDE/app/src'))) { + return candidate; + } + } + shell.echo( + '❌ Could not find a GDevelop checkout. Pass --gdevelop-root-path with the path to the GDevelop repository.' + ); + shell.exit(1); + return ''; +}; + +/** + * Concatenates all source files of newIDE/app/src into one string, used for + * substring checks. + * @param {string} sourcePath + * @returns {string} + */ +const readSourceBlob = (sourcePath) => { + /** @type {string[]} */ + const chunks = []; + const walk = (/** @type {string} */ directoryPath) => { + for (const entry of fs.readdirSync(directoryPath, { + withFileTypes: true, + })) { + const entryPath = path.join(directoryPath, entry.name); + if (entry.isDirectory()) { + if (entry.name !== 'node_modules') walk(entryPath); + } else if (/\.(js|jsx|ts|tsx)$/.test(entry.name)) { + chunks.push(fs.readFileSync(entryPath, 'utf8')); + } + } + }; + walk(sourcePath); + return chunks.join('\n'); +}; + +const gdevelopRootPath = findGDevelopRootPath(); +const sourcePath = path.join(gdevelopRootPath, 'newIDE/app/src'); +if (!fs.existsSync(sourcePath)) { + shell.echo(`❌ ${sourcePath} does not exist.`); + shell.exit(1); +} + +shell.echo(`ℹ️ Reading editor sources from ${sourcePath}...`); +const sourceBlob = readSourceBlob(sourcePath); + +const tutorialFileNames = fs + .readdirSync(inAppTutorialsPath) + .filter((fileName) => fileName.endsWith('.json')); + +let totalProblemsCount = 0; +for (const fileName of tutorialFileNames) { + // Load through InAppTutorial to expand meta steps ("add-behavior", + // "launch-preview"...) the same way they are expanded at deploy time. + const tutorial = new InAppTutorial(path.join(inAppTutorialsPath, fileName)); + tutorial.processFlowMetaSteps(); + const problems = checkTutorial(tutorial, sourceBlob); + if (problems.length === 0) { + shell.echo(`✅ ${fileName}`); + } else { + shell.echo(`❌ ${fileName}:`); + for (const problem of problems) { + const location = + problem.stepIndex !== null ? `step ${problem.stepIndex}` : 'tutorial'; + shell.echo( + ` - ${location}, ${problem.field} = "${problem.selector}": ${problem.reason}` + ); + } + totalProblemsCount += problems.length; + } +} + +if (totalProblemsCount > 0) { + shell.echo( + `\n❌ ${totalProblemsCount} problem(s) found. These selectors will prevent tutorial steps from being triggered or highlighted in the editor.` + ); + shell.exit(1); +} +shell.echo(`\n✅ All ${tutorialFileNames.length} tutorials passed.`); diff --git a/scripts/lib/InAppTutorialSelectorChecker.js b/scripts/lib/InAppTutorialSelectorChecker.js new file mode 100644 index 0000000..6553ed3 --- /dev/null +++ b/scripts/lib/InAppTutorialSelectorChecker.js @@ -0,0 +1,442 @@ +// @ts-check +/** + * Static checker for in-app tutorial selectors. + * + * Tutorials reference DOM elements of the GDevelop editor through CSS + * selectors (`elementToHighlightId`, `presenceOfElement`, `absenceOfElement`). + * When an id is renamed or removed in the editor, the tutorial silently breaks. + * This module extracts every selector used by a tutorial and verifies that the + * ids/attributes it relies on still exist in the GDevelop `newIDE/app/src` + * sources. + */ + +/** + * @typedef {Object} SelectorUsage + * @property {number} stepIndex + * @property {string | undefined} stepId + * @property {string} field Where the selector comes from (e.g. "elementToHighlightId"). + * @property {string} selector + */ + +/** + * @typedef {Object} Problem + * @property {string} tutorialId + * @property {number | null} stepIndex + * @property {string} field + * @property {string} selector + * @property {string} reason + */ + +/** + * Selector prefixes interpolated at runtime by InAppTutorialOrchestrator + * (see newIDE/app/src/InAppTutorial/InAppTutorialOrchestrator.js). + * The suffix is a key of the tutorial project data, not a DOM id. + */ +const PROJECT_DATA_ACCESSOR_PREFIXES = [ + 'objectInObjectsList:', + 'sceneInProjectManager:', + 'objectInObjectOrResourceSelector:', + 'editorTab:', +]; + +const KNOWN_EDITOR_TYPES = ['Scene', 'EventsSheet', 'Home']; + +/** + * Ids that are built dynamically in the editor sources (template literals, + * concatenations...) and therefore cannot be found verbatim. Each family is + * identified by a pattern on the id used in tutorials, and by one or more + * "needles": strings that must be present in the sources for the construction + * site of this id family to be considered still alive. + */ +const DYNAMIC_ID_FAMILIES = [ + { pattern: /^instruction-item-/, needles: ['instruction-item-'] }, + { + pattern: /^instruction-or-expression-/, + needles: ['instruction-or-expression-'], + }, + { pattern: /^object-item-/, needles: ['object-item-'] }, + { pattern: /^behavior-item-/, needles: ['behavior-item-'] }, + { pattern: /^behavior-parameters-/, needles: ['behavior-parameters-'] }, + { pattern: /^extension-list-item-/, needles: ['extension-list-item-'] }, + { pattern: /^asset-card-/, needles: ['asset-card-'] }, + { pattern: /^asset-pack-/, needles: ['asset-pack-'] }, + { pattern: /^scene-item-\d+$/, needles: ['scene-item-${'] }, + { pattern: /^project-manager-tab-/, needles: ['project-manager-tab-${'] }, + { pattern: /^tab-/, needles: ['tab-${'] }, + { pattern: /^event-\d/, needles: ['event-${'] }, + { pattern: /^parameter-\d+-/, needles: ['parameter-${'] }, + { pattern: /^variable-\d+-/, needles: ['variable-${'] }, + { pattern: /^layer-\d+$/, needles: ['layer-${'] }, + { + pattern: /^layer-selected-(checked|unchecked)$/, + needles: ['layer-selected-${'], + }, + { + pattern: /^add-(action|condition)-button-empty$/, + needles: ["'add-condition-button'", "'add-action-button'", "'-empty'"], + }, + { + pattern: /^open-(number|string)-expression-popover-button$/, + needles: ['-expression-popover-button'], + }, + { pattern: /^launch-export-/, needles: ['launch-export-${'] }, + { + pattern: /^online-web-button$/, + needles: ['launch-export-${', 'online-web'], + }, +]; + +/** + * `data-*` attributes that are set through a React `data`/dataset object with + * a camelCase key (e.g. `data={{ isFiltered: ... }}` renders + * `data-is-filtered`), so the kebab-case attribute never appears verbatim in + * the sources. Only keys that have been manually verified to exist should be + * listed here; for each of them the camelCase key followed by ':' must still + * be found in the sources. + */ +const KNOWN_CAMEL_CASE_DATASET_KEYS = [ + 'effective', + 'global', + 'groupName', + 'isFiltered', +]; + +/** @param {string} attribute @returns {string} */ +const toCamelCaseDatasetKey = (attribute) => + attribute + .replace(/^data-/, '') + .replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()); + +/** + * Splits a selector list on commas that are not nested inside parentheses or + * brackets (e.g. `:is(a, b)` is not split). + * @param {string} selector + * @returns {string[]} + */ +const splitTopLevelCommas = (selector) => { + /** @type {string[]} */ + const parts = []; + let depth = 0; + let current = ''; + for (const character of selector) { + if (character === '(' || character === '[') depth++; + else if (character === ')' || character === ']') depth--; + if (character === ',' && depth === 0) { + parts.push(current.trim()); + current = ''; + } else { + current += character; + } + } + if (current.trim()) parts.push(current.trim()); + return parts; +}; + +/** + * Extracts the ids a selector relies on: `#some-id`, `[id="some-id"]`, + * `[id^=prefix]`, `[id$=suffix]`. + * @param {string} selector + * @returns {string[]} + */ +const extractIdTokens = (selector) => { + /** @type {string[]} */ + const ids = []; + for (const match of selector.matchAll(/#([\w-]+)/g)) { + ids.push(match[1]); + } + for (const match of selector.matchAll( + /\[id[\^$*]?=["']?([\w -]+?)["']?\]/g + )) { + ids.push(match[1]); + } + return ids; +}; + +/** + * Extracts the attribute names (other than `id`) a selector relies on, e.g. + * `data-active` in `[data-active="true"]` or `[data-active]`. + * @param {string} selector + * @returns {string[]} + */ +const extractAttributeNames = (selector) => { + /** @type {string[]} */ + const attributes = []; + for (const match of selector.matchAll(/\[([a-zA-Z-]+)(?:[\^$*]?=|\])/g)) { + if (match[1] !== 'id') attributes.push(match[1]); + } + return attributes; +}; + +/** + * @param {string} id + * @param {string} sourceBlob + * @returns {string | null} A failure reason, or null if the id is fine. + */ +const checkId = (id, sourceBlob) => { + if (sourceBlob.includes(id)) return null; + // Editor ids are kebab-case: an id containing an uppercase letter and no + // dash is a property/behavior field name generated from project data (e.g. + // `#FollowOnY`, `#playerNumber`, `[id="Z Order"]`) and cannot be checked + // statically. + if (/^[A-Z]/.test(id) || (/[A-Z]/.test(id) && !id.includes('-'))) return null; + for (const family of DYNAMIC_ID_FAMILIES) { + if (family.pattern.test(id)) { + const missingNeedles = family.needles.filter( + (needle) => !sourceBlob.includes(needle) + ); + if (missingNeedles.length === 0) return null; + return `id "${id}" matches a dynamic id family but its construction site was not found in the editor sources (missing: ${missingNeedles.join( + ', ' + )})`; + } + } + return `id "${id}" was not found in the editor sources`; +}; + +/** + * @param {string} attribute + * @param {string} sourceBlob + * @returns {string | null} A failure reason, or null if the attribute is fine. + */ +const checkAttribute = (attribute, sourceBlob) => { + if (sourceBlob.includes(attribute)) return null; + const camelCaseKey = toCamelCaseDatasetKey(attribute); + if ( + KNOWN_CAMEL_CASE_DATASET_KEYS.includes(camelCaseKey) && + sourceBlob.includes(camelCaseKey + ':') + ) { + return null; + } + return `attribute "${attribute}" was not found in the editor sources`; +}; + +/** + * Checks a raw CSS selector (no project data accessor prefix) against the + * editor sources. A selector list (comma-separated) passes if at least one of + * its alternatives passes, matching `querySelector` semantics. + * @param {string} selector + * @param {string} sourceBlob + * @returns {string[]} Failure reasons (empty if the selector is fine). + */ +const checkCssSelector = (selector, sourceBlob) => { + const alternatives = splitTopLevelCommas(selector); + /** @type {string[]} */ + const allReasons = []; + for (const alternative of alternatives) { + /** @type {string[]} */ + const reasons = []; + for (const id of extractIdTokens(alternative)) { + const reason = checkId(id, sourceBlob); + if (reason) reasons.push(reason); + } + for (const attribute of extractAttributeNames(alternative)) { + const reason = checkAttribute(attribute, sourceBlob); + if (reason) reasons.push(reason); + } + if (reasons.length === 0) return []; // One alternative is fully valid. + allReasons.push(...reasons); + } + return allReasons; +}; + +/** + * Collects all the project data keys a tutorial declares + * (`initialProjectData` and every step's `mapProjectData`). + * @param {any} tutorial + * @returns {Set} + */ +const collectDeclaredDataKeys = (tutorial) => { + const keys = new Set(Object.keys(tutorial.initialProjectData || {})); + for (const step of tutorial.flow || []) { + for (const key of Object.keys(step.mapProjectData || {})) { + keys.add(key); + } + } + return keys; +}; + +/** + * Extracts every selector usage of a tutorial flow. + * @param {any} tutorial + * @returns {SelectorUsage[]} + */ +const extractSelectorUsages = (tutorial) => { + /** @type {SelectorUsage[]} */ + const usages = []; + /** @type {any[]} */ (tutorial.flow || []).forEach((step, stepIndex) => { + /** @param {string} field @param {string | undefined} selector */ + const add = (field, selector) => { + if (typeof selector === 'string') { + usages.push({ stepIndex, stepId: step.id, field, selector }); + } + }; + add('elementToHighlightId', step.elementToHighlightId); + const trigger = step.nextStepTrigger || {}; + add('nextStepTrigger.presenceOfElement', trigger.presenceOfElement); + add('nextStepTrigger.absenceOfElement', trigger.absenceOfElement); + add('nextStepTrigger.editorIsActive', trigger.editorIsActive); + /** @type {any[]} */ (step.shortcuts || []).forEach( + /** @param {any} shortcut @param {number} shortcutIndex */ ( + shortcut, + shortcutIndex + ) => { + const shortcutTrigger = shortcut.trigger || {}; + add( + `shortcuts[${shortcutIndex}].trigger.presenceOfElement`, + shortcutTrigger.presenceOfElement + ); + add( + `shortcuts[${shortcutIndex}].trigger.absenceOfElement`, + shortcutTrigger.absenceOfElement + ); + } + ); + }); + return usages; +}; + +/** + * Checks a project data accessor selector (e.g. `objectInObjectsList:player`) + * or an `editorIsActive` value against the keys declared by the tutorial. + * @param {string} selector + * @param {string} field + * @param {Set} declaredDataKeys + * @returns {string[]} Failure reasons. + */ +const checkProjectDataAccessor = (selector, field, declaredDataKeys) => { + if (field === 'nextStepTrigger.editorIsActive') { + // Format: ":" or "Home". + if (selector === 'Home') return []; + const [sceneKey, editorType] = selector.split(':'); + /** @type {string[]} */ + const reasons = []; + if (!editorType || !KNOWN_EDITOR_TYPES.includes(editorType)) { + reasons.push( + `editorIsActive value "${selector}" does not use a known editor type (${KNOWN_EDITOR_TYPES.join( + ', ' + )})` + ); + } + if (sceneKey && !declaredDataKeys.has(sceneKey)) { + reasons.push( + `editorIsActive value "${selector}" references project data key "${sceneKey}" which is not declared in initialProjectData or any mapProjectData` + ); + } + return reasons; + } + + if (selector.startsWith('editorTab:')) { + // Format: "editorTab::" or "editorTab:Home". + const [, sceneKey, editorType] = selector.split(':'); + if (sceneKey === 'Home' && !editorType) return []; + /** @type {string[]} */ + const reasons = []; + if (!editorType || !KNOWN_EDITOR_TYPES.includes(editorType)) { + reasons.push( + `selector "${selector}" does not use a known editor type (${KNOWN_EDITOR_TYPES.join( + ', ' + )})` + ); + } + if (sceneKey && sceneKey !== 'Home' && !declaredDataKeys.has(sceneKey)) { + reasons.push( + `selector "${selector}" references project data key "${sceneKey}" which is not declared in initialProjectData or any mapProjectData` + ); + } + return reasons; + } + + const dataKey = selector.slice(selector.indexOf(':') + 1); + if (!declaredDataKeys.has(dataKey)) { + return [ + `selector "${selector}" references project data key "${dataKey}" which is not declared in initialProjectData or any mapProjectData`, + ]; + } + return []; +}; + +/** + * Checks the `mapProjectData` and `editorSwitches` declarations of a tutorial. + * @param {any} tutorial + * @param {Set} declaredDataKeys + * @returns {Problem[]} + */ +const checkProjectDataDeclarations = (tutorial, declaredDataKeys) => { + /** @type {Problem[]} */ + const problems = []; + /** @type {any[]} */ (tutorial.flow || []).forEach((step, stepIndex) => { + for (const [key, accessor] of Object.entries(step.mapProjectData || {})) { + const isValid = + accessor === 'projectLastSceneName' || + (typeof accessor === 'string' && + accessor.startsWith('sceneLastObjectName:') && + declaredDataKeys.has(accessor.split(':')[1])); + if (!isValid) { + problems.push({ + tutorialId: tutorial.id, + stepIndex, + field: `mapProjectData.${key}`, + selector: String(accessor), + reason: `unknown or invalid project data accessor "${accessor}"`, + }); + } + } + }); + for (const [stepId, editorSwitch] of Object.entries( + tutorial.editorSwitches || {} + )) { + const sceneKey = /** @type {any} */ (editorSwitch).scene; + if (sceneKey && !declaredDataKeys.has(sceneKey)) { + problems.push({ + tutorialId: tutorial.id, + stepIndex: null, + field: `editorSwitches.${stepId}`, + selector: String(sceneKey), + reason: `references project data key "${sceneKey}" which is not declared in initialProjectData or any mapProjectData`, + }); + } + } + return problems; +}; + +/** + * Checks a whole tutorial against the editor sources. + * @param {any} tutorial + * @param {string} sourceBlob Concatenated content of all newIDE/app/src files. + * @returns {Problem[]} + */ +const checkTutorial = (tutorial, sourceBlob) => { + const declaredDataKeys = collectDeclaredDataKeys(tutorial); + /** @type {Problem[]} */ + const problems = checkProjectDataDeclarations(tutorial, declaredDataKeys); + + for (const usage of extractSelectorUsages(tutorial)) { + const isProjectDataAccessor = + PROJECT_DATA_ACCESSOR_PREFIXES.some((prefix) => + usage.selector.startsWith(prefix) + ) || usage.field === 'nextStepTrigger.editorIsActive'; + const reasons = isProjectDataAccessor + ? checkProjectDataAccessor(usage.selector, usage.field, declaredDataKeys) + : checkCssSelector(usage.selector, sourceBlob); + for (const reason of reasons) { + problems.push({ + tutorialId: tutorial.id, + stepIndex: usage.stepIndex, + field: usage.field, + selector: usage.selector, + reason, + }); + } + } + return problems; +}; + +module.exports = { + splitTopLevelCommas, + extractIdTokens, + extractAttributeNames, + extractSelectorUsages, + collectDeclaredDataKeys, + checkCssSelector, + checkTutorial, +}; From 24a1f0b910e321f846c1e8489dda776ea9234fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:39:33 +0200 Subject: [PATCH 02/20] Add CI workflow playing in-app tutorials with videos as artifacts - GitHub Action on the automatic-tests branch: checks out GDevelop (automatic-tests branch), runs the static selector check, then plays every tutorial with Playwright against the editor dev server. - A video of each tutorial being played is uploaded as artifact (14 days), for passing and failing runs alike, along with traces and screenshots. - Known broken tutorials (flingGame, plinkoMultiplier) are played and recorded but expected to fail; the selector check skips them via --ignore. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 67 +++++++++++++++++++++ e2e/in-app-tutorials.spec.js | 17 ++++++ e2e/playwright.config.js | 9 ++- scripts/check-in-app-tutorial-selectors.js | 16 ++++- 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-in-app-tutorials.yml diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml new file mode 100644 index 0000000..df4683d --- /dev/null +++ b/.github/workflows/test-in-app-tutorials.yml @@ -0,0 +1,67 @@ +# Plays every in-app tutorial against the GDevelop editor (web build) and +# fails if a tutorial step cannot be completed. A video of each tutorial being +# played is uploaded as an artifact, for passing and failing runs alike. +# +# For the moment this runs on the automatic-tests branch only, against the +# automatic-tests branch of GDevelop (which contains the orchestrator test +# hook). Once merged, switch both refs and add a schedule (hourly cron). +name: Test in-app tutorials + +on: + push: + branches: [automatic-tests] + workflow_dispatch: + +jobs: + test-in-app-tutorials: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout tutorials + uses: actions/checkout@v4 + + - name: Checkout GDevelop + uses: actions/checkout@v4 + with: + repository: 4ian/GDevelop + ref: automatic-tests + path: GDevelop + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: | + package-lock.json + GDevelop/newIDE/app/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Install GDevelop editor dependencies + run: npm ci + working-directory: GDevelop/newIDE/app + + # Fast static check: every selector used by the tutorials must still + # exist in the editor sources. Known broken tutorials are skipped — + # remove them from --ignore (and from KNOWN_BROKEN_TUTORIAL_IDS in + # e2e/in-app-tutorials.spec.js) once fixed. + - name: Check tutorial selectors against the editor sources + run: npm run check-in-app-tutorial-selectors -- --gdevelop-root-path ./GDevelop --ignore flingGame,plinkoMultiplier,joystick + + - name: Install Playwright browser + run: npx playwright install --with-deps chromium + + # Starts the editor dev server (from ./GDevelop) and plays every + # tutorial. Known broken tutorials are expected to fail. + - name: Play all in-app tutorials + run: npm run test-in-app-tutorials + + - name: Upload videos, screenshots and traces + if: always() + uses: actions/upload-artifact@v4 + with: + name: tutorial-videos-and-traces + path: test-results/ + retention-days: 14 + if-no-files-found: warn diff --git a/e2e/in-app-tutorials.spec.js b/e2e/in-app-tutorials.spec.js index cd56862..958dd18 100644 --- a/e2e/in-app-tutorials.spec.js +++ b/e2e/in-app-tutorials.spec.js @@ -14,6 +14,19 @@ const { } = require('./lib/gdevelopEditor'); const { playTutorial } = require('./lib/tutorialPlayer'); +/** + * Tutorials that are currently known to be broken (see the findings of + * `npm run check-in-app-tutorial-selectors`). They are still played (and + * recorded) but are expected to fail. Remove a tutorial from this list once it + * is fixed — the test will then fail with "unexpectedly passed" as a reminder. + * - flingGame: references removed editor elements (#layer-name, + * [data-default]) and requires a logged-in user for its leaderboard steps. + * - plinkoMultiplier: first step requires a logged-in user + * (absenceOfElement: #login-now), and it references the removed + * #project-manager-drawer-close element. + */ +const KNOWN_BROKEN_TUTORIAL_IDS = ['flingGame', 'plinkoMultiplier']; + const allTutorials = loadAllTutorials(); const tutorialIdsFilter = process.env.TUTORIAL_IDS ? process.env.TUTORIAL_IDS.split(',').map((id) => id.trim()) @@ -24,6 +37,10 @@ const tutorials = tutorialIdsFilter for (const tutorial of tutorials) { test(`in-app tutorial: ${tutorial.id}`, async ({ page, context }) => { + test.fail( + KNOWN_BROKEN_TUTORIAL_IDS.includes(tutorial.id), + 'This tutorial is known to be broken.' + ); await serveLocalTutorials(context, allTutorials); await startTutorial(page, tutorial.id); await playTutorial({ diff --git a/e2e/playwright.config.js b/e2e/playwright.config.js index 7b90047..1ead490 100644 --- a/e2e/playwright.config.js +++ b/e2e/playwright.config.js @@ -49,6 +49,9 @@ module.exports = defineConfig({ viewport: { width: 1600, height: 900 }, screenshot: 'only-on-failure', trace: 'retain-on-failure', + // On CI, record a video of every tutorial (uploaded as artifact), so that + // each run can be reviewed and failures can be diagnosed. + video: process.env.CI ? 'on' : 'retain-on-failure', }, webServer: gdevelopRootPath ? { @@ -59,7 +62,11 @@ module.exports = defineConfig({ url: editorUrl, reuseExistingServer: true, timeout: 20 * 60 * 1000, - env: { BROWSER: 'none' }, + env: { + BROWSER: 'none', + // The editor compilation (webpack) is memory hungry on CI. + NODE_OPTIONS: '--max-old-space-size=7168', + }, } : undefined, }); diff --git a/scripts/check-in-app-tutorial-selectors.js b/scripts/check-in-app-tutorial-selectors.js index 8d06b6d..58fc47f 100644 --- a/scripts/check-in-app-tutorial-selectors.js +++ b/scripts/check-in-app-tutorial-selectors.js @@ -5,6 +5,8 @@ * way tutorials break: an id is renamed or removed in the editor. * * Usage: node scripts/check-in-app-tutorial-selectors.js --gdevelop-root-path ../GDevelop + * Options: + * - --ignore : tutorial ids to skip (known broken tutorials). */ const shell = require('shelljs'); const path = require('path'); @@ -76,8 +78,20 @@ const tutorialFileNames = fs .readdirSync(inAppTutorialsPath) .filter((fileName) => fileName.endsWith('.json')); +const ignoredTutorialIds = new Set( + typeof args['ignore'] === 'string' + ? args['ignore'].split(',').filter(Boolean) + : [] +); + let totalProblemsCount = 0; +let checkedTutorialsCount = 0; for (const fileName of tutorialFileNames) { + if (ignoredTutorialIds.has(path.basename(fileName, '.json'))) { + shell.echo(`⚠️ ${fileName} skipped (known broken tutorial).`); + continue; + } + checkedTutorialsCount++; // Load through InAppTutorial to expand meta steps ("add-behavior", // "launch-preview"...) the same way they are expanded at deploy time. const tutorial = new InAppTutorial(path.join(inAppTutorialsPath, fileName)); @@ -104,4 +118,4 @@ if (totalProblemsCount > 0) { ); shell.exit(1); } -shell.echo(`\n✅ All ${tutorialFileNames.length} tutorials passed.`); +shell.echo(`\n✅ All ${checkedTutorialsCount} checked tutorials passed.`); From b90540c1452b074b0a03c5b6aa39afefb14ef7a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:27:14 +0200 Subject: [PATCH 03/20] Make tutorial player robust to slow CI environments - Wait for menus/submenus to render when following the bold texts of a tooltip through a split menu button (2-players preview), and dismiss stray menus before retrying a preview launch. - Dispatch a direct click on the tooltip button when it never stabilizes (tooltip anchored to the bouncing avatar), with bounded timeouts. - Set a global actionTimeout so no action can ever block until the test timeout. Co-Authored-By: Claude Fable 5 --- e2e/README.md | 13 +++++++++++++ e2e/lib/tutorialPlayer.js | 32 +++++++++++++++++++++++++++----- e2e/playwright.config.js | 2 ++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index d5dddaa..b4aba8b 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -25,6 +25,19 @@ referencing elements that no longer exist in the editor, without running it. - The tutorial is started through the regular `?initial-dialog=guided-lesson&tutorial-id=` flow. +## CI + +The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static +selector check and plays every tutorial on each push (currently on the +`automatic-tests` branch, against the `automatic-tests` branch of GDevelop). A +video of each tutorial being played is uploaded as an artifact +(`tutorial-videos-and-traces`), for passing and failing runs alike. + +Known broken tutorials are listed in `KNOWN_BROKEN_TUTORIAL_IDS` +(`e2e/in-app-tutorials.spec.js`) and in the `--ignore` flag of the selector +check step: they are still played and recorded, but expected to fail. When +fixing one, remove it from both places. + ## Running locally Requirements: a GDevelop checkout with `npm install` ran in `newIDE/app`, diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 5f51802..7be9ba9 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -198,13 +198,30 @@ const performStepAction = async ({ // clicking the avatar, then click the button. await page .locator('#in-app-tutorial-avatar') - .click({ timeout: 5 * 1000 }); - await button.click({ timeout: 5 * 1000 }); + .click({ timeout: 5 * 1000 }) + .catch(() => {}); + try { + await button.click({ timeout: 5 * 1000 }); + } catch (secondError) { + // The tooltip can be anchored to the bouncing avatar: the button then + // never stops moving so a regular click never considers it stable. + // Dispatch the click directly. + await button.waitFor({ state: 'attached', timeout: 5 * 1000 }); + await button.evaluate((node) => node.click(), undefined, { + timeout: 5 * 1000, + }); + } } return; } if (trigger.previewLaunched) { + if (attempt > 1) { + // A menu opened by a previous attempt may still be there, its backdrop + // blocking the toolbar: dismiss it. + await page.keyboard.press('Escape'); + await page.waitForTimeout(500); + } const popupPromise = context .waitForEvent('page', { timeout: STEP_ADVANCE_TIMEOUT_MS }) .catch(() => null); @@ -214,15 +231,20 @@ const performStepAction = async ({ .click({ timeout: 10 * 1000 }); // Some steps target a split/menu button (e.g. "Launch preview in... > // 2 previews in 2 windows"): follow the bold texts of the tooltip through - // the menus that opened. + // the menus that opened. Menus and submenus can take a moment to render, + // so wait for each item (not all bold texts are menu items, e.g. "down + // arrow": those are skipped after the wait times out). for (const boldText of extractAllBoldTexts(step)) { const menuItem = page .getByRole('menuitem', { name: boldText.replace(/(\.|…)+$/, ''), }) .first(); - if (await menuItem.isVisible().catch(() => false)) { - await menuItem.click({ timeout: 5000 }).catch(() => {}); + try { + await menuItem.waitFor({ state: 'visible', timeout: 4000 }); + await menuItem.click({ timeout: 5000 }); + } catch (error) { + // Not a menu item: ignore. } } // The preview opens in a new page: keep it open (the trigger fires when diff --git a/e2e/playwright.config.js b/e2e/playwright.config.js index 1ead490..f51e1c6 100644 --- a/e2e/playwright.config.js +++ b/e2e/playwright.config.js @@ -47,6 +47,8 @@ module.exports = defineConfig({ use: { baseURL: editorUrl, viewport: { width: 1600, height: 900 }, + // Safety net: no action should ever block until the test timeout. + actionTimeout: 15 * 1000, screenshot: 'only-on-failure', trace: 'retain-on-failure', // On CI, record a video of every tutorial (uploaded as artifact), so that From 51fb8927d4d0124b9ce2438b04e7f693ad811d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:36:50 +0200 Subject: [PATCH 04/20] Click tutorial tooltip buttons at their current position The tooltip is anchored to the bouncing avatar and never stops moving, so a regular Playwright click (which waits for stability) intermittently times out, especially on slow CI machines. Wait for visibility, then click at the element's current coordinates. Co-Authored-By: Claude Fable 5 --- e2e/lib/tutorialPlayer.js | 47 +++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 7be9ba9..306f0de 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -49,6 +49,20 @@ const extractAllBoldTexts = (step) => { return [...description.matchAll(/\*\*(.+?)\*\*/g)].map((match) => match[1]); }; +/** + * Clicks at the element's current position with a raw mouse click, bypassing + * Playwright's stability check — for elements that never stop moving (the + * tutorial avatar bounces indefinitely, and so does the tooltip anchored to + * it). + * @param {import('@playwright/test').Page} page + * @param {import('@playwright/test').Locator} locator + */ +const clickAtCurrentPosition = async (page, locator) => { + const box = await locator.boundingBox({ timeout: 5 * 1000 }); + if (!box) throw new Error('Element to click is not visible.'); + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); +}; + class TutorialStepError extends Error { /** * @param {string} message @@ -190,28 +204,23 @@ const performStepAction = async ({ const buttonLabel = getEnglishMessage(trigger.clickOnTooltipButton); const button = page .locator('#in-app-tutorial-tooltip-displayer') - .getByRole('button', { name: buttonLabel }); + .getByRole('button', { name: buttonLabel }) + .first(); + // The tooltip is often anchored to the bouncing avatar: it never stops + // moving, so a regular click (which waits for the element to be stable) + // would time out. Click at the current position instead. try { - await button.click({ timeout: 5 * 1000 }); + await button.waitFor({ state: 'visible', timeout: 8 * 1000 }); } catch (error) { - // The tooltip can be folded (only the avatar is visible): unfold it by - // clicking the avatar, then click the button. - await page - .locator('#in-app-tutorial-avatar') - .click({ timeout: 5 * 1000 }) - .catch(() => {}); - try { - await button.click({ timeout: 5 * 1000 }); - } catch (secondError) { - // The tooltip can be anchored to the bouncing avatar: the button then - // never stops moving so a regular click never considers it stable. - // Dispatch the click directly. - await button.waitFor({ state: 'attached', timeout: 5 * 1000 }); - await button.evaluate((node) => node.click(), undefined, { - timeout: 5 * 1000, - }); - } + // The tooltip is probably folded (only the avatar is visible): + // clicking the avatar unfolds it. + await clickAtCurrentPosition( + page, + page.locator('#in-app-tutorial-avatar') + ); + await button.waitFor({ state: 'visible', timeout: 8 * 1000 }); } + await clickAtCurrentPosition(page, button); return; } From 3f2974504f5665dc9d12357be2824997c33b7e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:22:48 +0200 Subject: [PATCH 05/20] Report page state (error boundary, open dialogs) in step failures The tutorial tooltip hides itself when an error boundary is displayed or when another dialog is stacked above the one hosting the highlighted element: include both in the failure message to diagnose CI-only failures. Co-Authored-By: Claude Fable 5 --- e2e/lib/tutorialPlayer.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 306f0de..a89ffaa 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -517,13 +517,33 @@ const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { const advanced = await waitForStepChange(page, state.stepIndex); if (!advanced && attemptsForCurrentStep >= MAX_ATTEMPTS_PER_STEP) { + // The tutorial tooltip/highlighter hides itself in some situations + // (error boundary displayed, another dialog opened above): include + // those in the report as they usually explain "element not found" + // failures. + const pageDiagnostics = await page + .evaluate(() => { + const dialogTitles = [ + ...document.querySelectorAll('[role="dialog"] h2'), + ].map((title) => (title.textContent || '').slice(0, 60)); + return { + hasErrorBoundary: !!document.querySelector('[data-error-boundary]'), + dialogTitles, + }; + }) + .catch(() => null); throw new TutorialStepError( `Tutorial "${tutorial.id}" is broken at step ${state.stepIndex} ` + `(highlighted element: ${state.elementToHighlightId || 'none'}, ` + `trigger: ${JSON.stringify(step.nextStepTrigger || {})})` + (actionError ? `. The action could not be performed: ${actionError.message}` - : '. The action was performed but the tutorial did not advance.'), + : '. The action was performed but the tutorial did not advance.') + + (pageDiagnostics + ? ` Page state: error boundary displayed: ${ + pageDiagnostics.hasErrorBoundary + }, open dialogs: [${pageDiagnostics.dialogTitles.join(' | ')}].` + : ''), { stepIndex: state.stepIndex, step, state } ); } From e333abcbc72285cebdabe40d752ae4306430fc65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:32:18 +0200 Subject: [PATCH 06/20] Log tooltip popper state when the tooltip button is not found Sharpen failure diagnostics (dialog count, element presence, tooltip text) and dump each tooltip popper's computed visibility and buttons when a clickOnTooltipButton step cannot find its button, to diagnose the remaining CI-only tilemapPlatformer failure. Co-Authored-By: Claude Fable 5 --- e2e/lib/tutorialPlayer.js | 55 ++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index a89ffaa..ceded34 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -202,9 +202,13 @@ const performStepAction = async ({ if (trigger.clickOnTooltipButton) { const buttonLabel = getEnglishMessage(trigger.clickOnTooltipButton); + // There can be several tooltip poppers in the DOM (a previous step's + // tooltip whose exit transition did not complete keeps the same id): + // only consider the visible button. const button = page - .locator('#in-app-tutorial-tooltip-displayer') + .locator('[id="in-app-tutorial-tooltip-displayer"]') .getByRole('button', { name: buttonLabel }) + .filter({ visible: true }) .first(); // The tooltip is often anchored to the bouncing avatar: it never stops // moving, so a regular click (which waits for the element to be stable) @@ -212,6 +216,29 @@ const performStepAction = async ({ try { await button.waitFor({ state: 'visible', timeout: 8 * 1000 }); } catch (error) { + const tooltipState = await page + .evaluate(() => + [ + ...document.querySelectorAll( + '[id="in-app-tutorial-tooltip-displayer"]' + ), + ].map((popper) => ({ + visibility: getComputedStyle(popper).visibility, + display: getComputedStyle(popper).display, + width: Math.round(popper.getBoundingClientRect().width), + height: Math.round(popper.getBoundingClientRect().height), + buttons: [...popper.querySelectorAll('button')].map((button) => ({ + text: (button.textContent || '').slice(0, 25), + width: Math.round(button.getBoundingClientRect().width), + height: Math.round(button.getBoundingClientRect().height), + visibility: getComputedStyle(button).visibility, + })), + })) + ) + .catch(() => null); + log( + `Tooltip button not visible. Poppers: ${JSON.stringify(tooltipState)}` + ); // The tooltip is probably folded (only the avatar is visible): // clicking the avatar unfolds it. await clickAtCurrentPosition( @@ -522,15 +549,25 @@ const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { // those in the report as they usually explain "element not found" // failures. const pageDiagnostics = await page - .evaluate(() => { - const dialogTitles = [ - ...document.querySelectorAll('[role="dialog"] h2'), - ].map((title) => (title.textContent || '').slice(0, 60)); + .evaluate((elementToHighlightId) => { + const tooltip = document.querySelector( + '#in-app-tutorial-tooltip-displayer' + ); return { hasErrorBoundary: !!document.querySelector('[data-error-boundary]'), - dialogTitles, + openDialogsCount: + document.querySelectorAll('[role="dialog"]').length, + highlightedElementExists: elementToHighlightId + ? !!document.querySelector(elementToHighlightId) + : null, + tooltipText: tooltip + ? (tooltip.textContent || '').trim().slice(0, 120) + : null, + avatarDisplayed: !!document.querySelector( + '#in-app-tutorial-avatar' + ), }; - }) + }, state.elementToHighlightId || null) .catch(() => null); throw new TutorialStepError( `Tutorial "${tutorial.id}" is broken at step ${state.stepIndex} ` + @@ -540,9 +577,7 @@ const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { ? `. The action could not be performed: ${actionError.message}` : '. The action was performed but the tutorial did not advance.') + (pageDiagnostics - ? ` Page state: error boundary displayed: ${ - pageDiagnostics.hasErrorBoundary - }, open dialogs: [${pageDiagnostics.dialogTitles.join(' | ')}].` + ? ` Page state: ${JSON.stringify(pageDiagnostics)}.` : ''), { stepIndex: state.stepIndex, step, state } ); From b6d3e806deacdb7d5827b813f742c1cc65a63754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:46:48 +0200 Subject: [PATCH 07/20] Locate tooltip buttons with CSS instead of role, wait longer for submenus When a modal dialog is open, Material UI sets aria-hidden on the other root containers including the tutorial tooltip popper: the visible button is then absent from the accessibility tree and getByRole cannot find it. Use a CSS + text locator instead. Also give submenus more time to render on slow CI machines. Co-Authored-By: Claude Fable 5 --- e2e/lib/tutorialPlayer.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index ceded34..27070bc 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -202,12 +202,14 @@ const performStepAction = async ({ if (trigger.clickOnTooltipButton) { const buttonLabel = getEnglishMessage(trigger.clickOnTooltipButton); - // There can be several tooltip poppers in the DOM (a previous step's - // tooltip whose exit transition did not complete keeps the same id): - // only consider the visible button. + // A CSS locator is used (not getByRole): when a modal dialog is open, + // Material UI sets aria-hidden on the other root containers including the + // tutorial popper, removing the (visible) button from the accessibility + // tree. Also only consider the visible button: several poppers with the + // same id can coexist in the DOM. const button = page - .locator('[id="in-app-tutorial-tooltip-displayer"]') - .getByRole('button', { name: buttonLabel }) + .locator('[id="in-app-tutorial-tooltip-displayer"] button') + .filter({ hasText: buttonLabel }) .filter({ visible: true }) .first(); // The tooltip is often anchored to the bouncing avatar: it never stops @@ -277,7 +279,7 @@ const performStepAction = async ({ }) .first(); try { - await menuItem.waitFor({ state: 'visible', timeout: 4000 }); + await menuItem.waitFor({ state: 'visible', timeout: 8000 }); await menuItem.click({ timeout: 5000 }); } catch (error) { // Not a menu item: ignore. From 069921d820b6ad3cc8efd24106acd27d46b5dd7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:12:37 +0200 Subject: [PATCH 08/20] Dispatch menu item clicks directly to avoid the submenu hover race A submenu closes 75ms after the pointer leaves its parent menu item (MaterialUIMenuImplementation): moving the mouse from the parent to a submenu item can close the submenu before the click lands on slow CI machines. Dispatch the click without any pointer movement instead, and use a CSS locator (menus can be aria-hidden when a dialog is open). Co-Authored-By: Claude Fable 5 --- e2e/lib/tutorialPlayer.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 27070bc..37faac7 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -271,16 +271,22 @@ const performStepAction = async ({ // 2 previews in 2 windows"): follow the bold texts of the tooltip through // the menus that opened. Menus and submenus can take a moment to render, // so wait for each item (not all bold texts are menu items, e.g. "down - // arrow": those are skipped after the wait times out). + // arrow": those are skipped after the wait times out). The clicks are + // dispatched directly (no mouse movement): a submenu closes 75ms after + // the pointer leaves its parent menu item, so moving the mouse to the + // submenu item can close it on slow machines. A CSS locator is used as + // menus can be aria-hidden when a dialog is open. for (const boldText of extractAllBoldTexts(step)) { const menuItem = page - .getByRole('menuitem', { - name: boldText.replace(/(\.|…)+$/, ''), - }) + .locator('[role="menuitem"]') + .filter({ hasText: boldText.replace(/(\.|…)+$/, '') }) + .filter({ visible: true }) .first(); try { await menuItem.waitFor({ state: 'visible', timeout: 8000 }); - await menuItem.click({ timeout: 5000 }); + await menuItem.evaluate((node) => node.click(), undefined, { + timeout: 5000, + }); } catch (error) { // Not a menu item: ignore. } From 71fc5d808c418b2ea71a454f311f554a7c5ab3ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:34:43 +0200 Subject: [PATCH 09/20] Dismiss unexpected dialogs that block a tutorial step click When a click is intercepted by a dialog that does not contain the element the tutorial points to (e.g. an error alert opened above the object editor on CI), close it like a user would and log its content. Co-Authored-By: Claude Fable 5 --- e2e/lib/tutorialPlayer.js | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 37faac7..2559810 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -547,7 +547,39 @@ const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { }); } catch (error) { actionError = error; - log(`Action failed: ${error.message}`); + log(`Action failed: ${error.message.split('\n')[0]}`); + if (error.message.includes('intercepts pointer events')) { + // An unexpected dialog (e.g. an error alert) opened above the current + // one and blocks the click: dismiss any dialog that does not contain + // the element the tutorial points to, like a user would. + const dismissedDialogs = await page + .evaluate((selector) => { + const target = selector ? document.querySelector(selector) : null; + const dismissed = []; + for (const dialog of document.querySelectorAll('[role="dialog"]')) { + if (target && dialog.contains(target)) continue; + const closeButton = [...dialog.querySelectorAll('button')].find( + (button) => + /^(close|ok|cancel|got it)$/i.test( + (button.textContent || '').trim() + ) + ); + if (closeButton) { + dismissed.push((dialog.textContent || '').trim().slice(0, 120)); + closeButton.click(); + } + } + return dismissed; + }, state.elementToHighlightId || null) + .catch(() => []); + if (dismissedDialogs.length) { + log( + `Dismissed unexpected dialog(s): ${JSON.stringify( + dismissedDialogs + )}` + ); + } + } } const advanced = await waitForStepChange(page, state.stepIndex); From 0b81c4814f0e0e5ae6a07c13d2e005d33fd85d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:53:49 +0200 Subject: [PATCH 10/20] Upload tutorial videos as mp4, separate from traces - Convert Playwright's webm recordings to mp4 (H.264) on CI so they play natively on macOS (QuickTime/Quick Look). - Split the artifact: 'tutorial-videos' (small, videos + screenshots) and 'test-traces' (heavy, ~50MB per failed test, failures only). Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 30 ++++++++++++++++++--- e2e/README.md | 7 +++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index df4683d..7181b38 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -57,11 +57,35 @@ jobs: - name: Play all in-app tutorials run: npm run test-in-app-tutorials - - name: Upload videos, screenshots and traces + # Playwright records webm, which does not play natively on macOS: + # convert to mp4 (H.264) so the videos open with QuickTime/Quick Look. + - name: Convert videos to mp4 + if: always() + run: | + find test-results -name "*.webm" -print0 | while IFS= read -r -d '' file; do + ffmpeg -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file" + done + + - name: Upload videos and screenshots if: always() uses: actions/upload-artifact@v4 with: - name: tutorial-videos-and-traces - path: test-results/ + name: tutorial-videos + path: | + test-results/**/*.mp4 + test-results/**/*.png + test-results/**/error-context.md retention-days: 14 if-no-files-found: warn + + # Traces are heavy (~50MB per failed test, with a DOM snapshot of every + # action): uploaded separately so downloading the videos stays fast. + # Inspect with: npx playwright show-trace trace.zip + - name: Upload traces (failures only) + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-traces + path: test-results/**/trace.zip + retention-days: 14 + if-no-files-found: ignore diff --git a/e2e/README.md b/e2e/README.md index b4aba8b..0415a7d 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -30,8 +30,11 @@ referencing elements that no longer exist in the editor, without running it. The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static selector check and plays every tutorial on each push (currently on the `automatic-tests` branch, against the `automatic-tests` branch of GDevelop). A -video of each tutorial being played is uploaded as an artifact -(`tutorial-videos-and-traces`), for passing and failing runs alike. +video of each tutorial being played (converted to mp4, playable natively on +macOS) is uploaded as the `tutorial-videos` artifact, for passing and failing +runs alike. Playwright traces of failed tests are uploaded separately as +`test-traces` (they are heavy — ~50MB per failed test); inspect one with +`npx playwright show-trace trace.zip`. Known broken tutorials are listed in `KNOWN_BROKEN_TUTORIAL_IDS` (`e2e/in-app-tutorials.spec.js`) and in the `--ignore` flag of the selector From 8a0d5d4dbc6c81a5e431bc0c012cf6503ddeb4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:11:34 +0200 Subject: [PATCH 11/20] Install ffmpeg before converting tutorial videos ffmpeg is not preinstalled on current ubuntu-latest runner images. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index 7181b38..be0f204 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -62,6 +62,7 @@ jobs: - name: Convert videos to mp4 if: always() run: | + sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg find test-results -name "*.webm" -print0 | while IFS= read -r -d '' file; do ffmpeg -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file" done From 73f95c641135a3c5103711bfde5e450367ff9085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:30:11 +0200 Subject: [PATCH 12/20] Pass -nostdin to ffmpeg so it does not consume the file list ffmpeg reads stdin by default, swallowing the piped find output inside the while loop and corrupting the remaining file paths. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index be0f204..10cb19f 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -64,7 +64,7 @@ jobs: run: | sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg find test-results -name "*.webm" -print0 | while IFS= read -r -d '' file; do - ffmpeg -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file" + ffmpeg -nostdin -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file" done - name: Upload videos and screenshots From 8c848b373c747756bdf164aac24d86a361e54a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:24:42 +0200 Subject: [PATCH 13/20] Fix joystick data key and the drawer-close checker false positive - joystick: mapProjectData referenced the undeclared key 'level' instead of 'gameScene' (it only worked through the orchestrator's fallback to the first scene). - plinkoMultiplier's #project-manager-drawer-close selectors are actually valid: DrawerTopBar builds its close button id from its own id. Teach the checker this pattern; only flingGame remains ignored. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 2 +- e2e/in-app-tutorials.spec.js | 4 ++-- .../__tests__/check-in-app-tutorial-selectors.spec.js | 9 +++++++++ scripts/lib/InAppTutorialSelectorChecker.js | 11 +++++++++++ tutorials/in-app/joystick.json | 2 +- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index 10cb19f..4e2e37a 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -47,7 +47,7 @@ jobs: # remove them from --ignore (and from KNOWN_BROKEN_TUTORIAL_IDS in # e2e/in-app-tutorials.spec.js) once fixed. - name: Check tutorial selectors against the editor sources - run: npm run check-in-app-tutorial-selectors -- --gdevelop-root-path ./GDevelop --ignore flingGame,plinkoMultiplier,joystick + run: npm run check-in-app-tutorial-selectors -- --gdevelop-root-path ./GDevelop --ignore flingGame - name: Install Playwright browser run: npx playwright install --with-deps chromium diff --git a/e2e/in-app-tutorials.spec.js b/e2e/in-app-tutorials.spec.js index 958dd18..ba0213d 100644 --- a/e2e/in-app-tutorials.spec.js +++ b/e2e/in-app-tutorials.spec.js @@ -22,8 +22,8 @@ const { playTutorial } = require('./lib/tutorialPlayer'); * - flingGame: references removed editor elements (#layer-name, * [data-default]) and requires a logged-in user for its leaderboard steps. * - plinkoMultiplier: first step requires a logged-in user - * (absenceOfElement: #login-now), and it references the removed - * #project-manager-drawer-close element. + * (absenceOfElement: #login-now) — remove once the tests use a test + * account. */ const KNOWN_BROKEN_TUTORIAL_IDS = ['flingGame', 'plinkoMultiplier']; diff --git a/scripts/__tests__/check-in-app-tutorial-selectors.spec.js b/scripts/__tests__/check-in-app-tutorial-selectors.spec.js index 57a6ef7..6b06cfa 100644 --- a/scripts/__tests__/check-in-app-tutorial-selectors.spec.js +++ b/scripts/__tests__/check-in-app-tutorial-selectors.spec.js @@ -85,6 +85,15 @@ describe('checkCssSelector', () => { expect(checkCssSelector('[id="Z Order"]', sourceBlob)).toEqual([]); }); + it('accepts drawer close buttons built from the drawer id', () => { + const blobWithDrawer = + sourceBlob + '\nid={`${props.id}-close`}\nid="my-drawer"'; + expect(checkCssSelector('#my-drawer-close', blobWithDrawer)).toEqual([]); + expect(checkCssSelector('#other-drawer-close', blobWithDrawer)[0]).toMatch( + /other-drawer-close/ + ); + }); + it('accepts known camelCase dataset attributes', () => { expect( checkCssSelector( diff --git a/scripts/lib/InAppTutorialSelectorChecker.js b/scripts/lib/InAppTutorialSelectorChecker.js index 6553ed3..21f1a57 100644 --- a/scripts/lib/InAppTutorialSelectorChecker.js +++ b/scripts/lib/InAppTutorialSelectorChecker.js @@ -179,6 +179,17 @@ const checkId = (id, sourceBlob) => { // `#FollowOnY`, `#playerNumber`, `[id="Z Order"]`) and cannot be checked // statically. if (/^[A-Z]/.test(id) || (/[A-Z]/.test(id) && !id.includes('-'))) return null; + // DrawerTopBar builds its buttons ids from its own id + // (`${props.id}-close` / `${props.id}-icon`): accept them when the base id + // still exists. + const drawerButtonMatch = id.match(/^(.+)-(close|icon)$/); + if ( + drawerButtonMatch && + sourceBlob.includes('${props.id}-' + drawerButtonMatch[2]) && + sourceBlob.includes(drawerButtonMatch[1]) + ) { + return null; + } for (const family of DYNAMIC_ID_FAMILIES) { if (family.pattern.test(id)) { const missingNeedles = family.needles.filter( diff --git a/tutorials/in-app/joystick.json b/tutorials/in-app/joystick.json index 3b92c43..625bd12 100644 --- a/tutorials/in-app/joystick.json +++ b/tutorials/in-app/joystick.json @@ -531,7 +531,7 @@ "objectAddedInLayout": true }, "mapProjectData": { - "joystick": "sceneLastObjectName:level" + "joystick": "sceneLastObjectName:gameScene" }, "tooltip": { "description": { From 34c5ac46997dc1f888a2ef4ebd68c0be71abda6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:01:54 +0200 Subject: [PATCH 14/20] Play plinkoMultiplier fully: in-flow login, right click, multi-drop - Log in with the dev test account through the tutorial's own flow: when an absenceOfElement step targets an existing element (e.g. #login-now), click it and complete the login dialog that opens. - Follow 'Right click ... and select **X**' tooltips: open the context menu and click the bold menu items. - Drop as many instances as instancesCount requires, at spread positions. - Verify filled values stick and retype like a user when the editor resets an uncontrolled input (variables list). - Dismiss unexpected dialogs whenever a step fails to advance (the dev leaderboard-creation 404 dialog is expected: template leaderboards only exist in production). - plinkoMultiplier is no longer expected to fail; only flingGame remains. Co-Authored-By: Claude Fable 5 --- e2e/README.md | 9 ++ e2e/in-app-tutorials.spec.js | 7 +- e2e/lib/gdevelopEditor.js | 38 +++++++ e2e/lib/tutorialPlayer.js | 195 ++++++++++++++++++++++------------- 4 files changed, 173 insertions(+), 76 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 0415a7d..c538d1b 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -41,6 +41,15 @@ Known broken tutorials are listed in `KNOWN_BROKEN_TUTORIAL_IDS` check step: they are still played and recorded, but expected to fail. When fixing one, remove it from both places. +## Known dev-environment behaviors + +- When opening a template that uses leaderboards (plinkoMultiplier) with a + logged-in user, the leaderboard auto-creation fails with a 404: template + leaderboards only exist in production. This is expected on dev — the player + dismisses the error dialog ("Abandon") and the tutorial continues. +- The test account (see `TEST_ACCOUNT` in `e2e/lib/gdevelopEditor.js`) is a + dev-environment account: it contains no secret. + ## Running locally Requirements: a GDevelop checkout with `npm install` ran in `newIDE/app`, diff --git a/e2e/in-app-tutorials.spec.js b/e2e/in-app-tutorials.spec.js index ba0213d..ea393d0 100644 --- a/e2e/in-app-tutorials.spec.js +++ b/e2e/in-app-tutorials.spec.js @@ -20,12 +20,9 @@ const { playTutorial } = require('./lib/tutorialPlayer'); * recorded) but are expected to fail. Remove a tutorial from this list once it * is fixed — the test will then fail with "unexpectedly passed" as a reminder. * - flingGame: references removed editor elements (#layer-name, - * [data-default]) and requires a logged-in user for its leaderboard steps. - * - plinkoMultiplier: first step requires a logged-in user - * (absenceOfElement: #login-now) — remove once the tests use a test - * account. + * [data-default]). */ -const KNOWN_BROKEN_TUTORIAL_IDS = ['flingGame', 'plinkoMultiplier']; +const KNOWN_BROKEN_TUTORIAL_IDS = ['flingGame']; const allTutorials = loadAllTutorials(); const tutorialIdsFilter = process.env.TUTORIAL_IDS diff --git a/e2e/lib/gdevelopEditor.js b/e2e/lib/gdevelopEditor.js index d8bc82e..1c93105 100644 --- a/e2e/lib/gdevelopEditor.js +++ b/e2e/lib/gdevelopEditor.js @@ -54,6 +54,43 @@ const serveLocalTutorials = async (context, tutorials) => { }); }; +/** + * Test account used for the tutorials that require an authenticated user + * (leaderboards...). This is a dev-environment account (the tests run against + * api-dev.gdevelop.io): no secret here. + */ +const TEST_ACCOUNT = { + email: + process.env.GDEVELOP_TEST_ACCOUNT_EMAIL || 'clement+playwright@gdevelop.io', + password: process.env.GDEVELOP_TEST_ACCOUNT_PASSWORD || 'gdevelop', +}; + +/** + * If the login dialog is (or becomes) visible, fills it with the test account + * and submits it, like a user would. Returns true if a login was performed. + * @param {import('@playwright/test').Page} page + * @returns {Promise} + */ +const completeLoginDialogIfPresent = async (page) => { + const loginDialog = page.locator('#login-dialog'); + try { + await loginDialog.waitFor({ state: 'visible', timeout: 5 * 1000 }); + } catch (error) { + return false; + } + await loginDialog + .locator('input:not([type="password"])') + .first() + .fill(TEST_ACCOUNT.email); + await loginDialog + .locator('input[type="password"]') + .first() + .fill(TEST_ACCOUNT.password); + await page.locator('#login-button').click(); + await loginDialog.waitFor({ state: 'hidden', timeout: 60 * 1000 }); + return true; +}; + /** * Opens the editor and starts the given tutorial through the same flow as a * user clicking a lesson card: the start dialog is opened via the @@ -87,6 +124,7 @@ const getTutorialState = async (page) => { module.exports = { loadAllTutorials, serveLocalTutorials, + completeLoginDialogIfPresent, startTutorial, getTutorialState, }; diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 2559810..5eb6962 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -5,7 +5,10 @@ * orchestrator advances to the next step. If it does not, the tutorial is * considered broken at this step. */ -const { getTutorialState } = require('./gdevelopEditor'); +const { + getTutorialState, + completeLoginDialogIfPresent, +} = require('./gdevelopEditor'); /** How long to wait for the orchestrator to advance after performing an action. * The orchestrator polls the DOM/project every 0.5-1s, so this includes at @@ -49,6 +52,36 @@ const extractAllBoldTexts = (step) => { return [...description.matchAll(/\*\*(.+?)\*\*/g)].map((match) => match[1]); }; +/** + * Clicks the menu items matching the bold texts of the step tooltip, in + * order, in the menus that are open. Menus and submenus can take a moment to + * render, so each item is waited for (bold texts that are not menu items, + * e.g. "down arrow", are skipped after the wait times out). The clicks are + * dispatched directly (no mouse movement): a submenu closes 75ms after the + * pointer leaves its parent menu item, so moving the mouse to the submenu + * item can close it on slow machines. A CSS locator is used as menus can be + * aria-hidden when a dialog is open. + * @param {import('@playwright/test').Page} page + * @param {any} step + */ +const followBoldTextsThroughMenus = async (page, step) => { + for (const boldText of extractAllBoldTexts(step)) { + const menuItem = page + .locator('[role="menuitem"]') + .filter({ hasText: boldText.replace(/(\.|…)+$/, '') }) + .filter({ visible: true }) + .first(); + try { + await menuItem.waitFor({ state: 'visible', timeout: 8000 }); + await menuItem.evaluate((node) => node.click(), undefined, { + timeout: 5000, + }); + } catch (error) { + // Not a menu item: ignore. + } + } +}; + /** * Clicks at the element's current position with a raw mouse click, bypassing * Playwright's stability check — for elements that never stop moving (the @@ -176,6 +209,16 @@ const fillField = async (page, selector, value, { anyValue = false } = {}) => { return; } await input.fill(value); + // Some fields are uncontrolled inputs that the editor can reset when + // another field commits its pending change (e.g. the variables list): + // verify the value stuck, and retype it like a user would otherwise. + await page.waitForTimeout(400); + const currentValue = await input.inputValue().catch(() => null); + if (currentValue !== null && currentValue !== value) { + await input.click(); + await input.press('ControlOrMeta+a'); + await input.pressSequentially(value, { delay: 30 }); + } }; /** @@ -269,28 +312,8 @@ const performStepAction = async ({ .click({ timeout: 10 * 1000 }); // Some steps target a split/menu button (e.g. "Launch preview in... > // 2 previews in 2 windows"): follow the bold texts of the tooltip through - // the menus that opened. Menus and submenus can take a moment to render, - // so wait for each item (not all bold texts are menu items, e.g. "down - // arrow": those are skipped after the wait times out). The clicks are - // dispatched directly (no mouse movement): a submenu closes 75ms after - // the pointer leaves its parent menu item, so moving the mouse to the - // submenu item can close it on slow machines. A CSS locator is used as - // menus can be aria-hidden when a dialog is open. - for (const boldText of extractAllBoldTexts(step)) { - const menuItem = page - .locator('[role="menuitem"]') - .filter({ hasText: boldText.replace(/(\.|…)+$/, '') }) - .filter({ visible: true }) - .first(); - try { - await menuItem.waitFor({ state: 'visible', timeout: 8000 }); - await menuItem.evaluate((node) => node.click(), undefined, { - timeout: 5000, - }); - } catch (error) { - // Not a menu item: ignore. - } - } + // the menus that opened. + await followBoldTextsThroughMenus(page, step); // The preview opens in a new page: keep it open (the trigger fires when // the launch resolves), it is closed by the caller once the step advanced. await popupPromise; @@ -339,19 +362,27 @@ const performStepAction = async ({ if (!sourceBox || !canvasBox) { throw new Error('Could not find the drag source or the scene canvas.'); } - await page.mouse.move( - sourceBox.x + sourceBox.width / 2, - sourceBox.y + sourceBox.height / 2 - ); - await page.mouse.down(); - // Move in several small steps so that drag events are properly emitted. - await page.mouse.move( - canvasBox.x + canvasBox.width / 2, - canvasBox.y + canvasBox.height / 2, - { steps: 20 } - ); - await page.waitForTimeout(200); - await page.mouse.up(); + // The step can require several instances (`instancesCount`): drop each + // one at a different position. + const instancesCount = trigger.instancesCount || 1; + for (let index = 0; index < instancesCount; index++) { + const dropFractionX = + 0.3 + (0.4 * index) / Math.max(instancesCount - 1, 1); + await page.mouse.move( + sourceBox.x + sourceBox.width / 2, + sourceBox.y + sourceBox.height / 2 + ); + await page.mouse.down(); + // Move in several small steps so that drag events are properly emitted. + await page.mouse.move( + canvasBox.x + canvasBox.width * dropFractionX, + canvasBox.y + canvasBox.height / 2, + { steps: 20 } + ); + await page.waitForTimeout(200); + await page.mouse.up(); + await page.waitForTimeout(300); + } return; } @@ -362,6 +393,15 @@ const performStepAction = async ({ if (highlightedElementSelector) { const element = page.locator(highlightedElementSelector).first(); await element.waitFor({ state: 'visible', timeout: 10 * 1000 }); + // Steps like "Right click on GameScene and select Edit scene variables": + // open the context menu and follow the bold texts through it. + if ( + /right.?click/i.test(getEnglishMessage((step.tooltip || {}).description)) + ) { + await element.click({ button: 'right', timeout: 10 * 1000 }); + await followBoldTextsThroughMenus(page, step); + return; + } const isTextInput = await element.evaluate((node) => { const input = node.tagName === 'INPUT' || node.tagName === 'TEXTAREA' @@ -421,9 +461,24 @@ const performStepAction = async ({ return; } - if (trigger.absenceOfElement || trigger.presenceOfElement) { - // No element to interact with: the expected change may happen on its own - // (or cannot be automated, e.g. a login): just wait. + if (trigger.absenceOfElement) { + // No element highlighted, and an element must disappear: if it is there, + // click it — that is the natural interaction to make it go away (e.g. + // plinkoMultiplier's "#login-now" button, which opens the login dialog). + const target = page.locator(trigger.absenceOfElement).first(); + if (await target.isVisible().catch(() => false)) { + await target.click({ timeout: 10 * 1000 }); + if (await completeLoginDialogIfPresent(page)) { + log('Logged in with the test account.'); + } + return; + } + log('Element to make disappear is already absent: waiting.'); + return; + } + + if (trigger.presenceOfElement) { + // No element to interact with: the expected change may happen on its own. log('No element to highlight: waiting for the trigger to be satisfied.'); return; } @@ -548,41 +603,39 @@ const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { } catch (error) { actionError = error; log(`Action failed: ${error.message.split('\n')[0]}`); - if (error.message.includes('intercepts pointer events')) { - // An unexpected dialog (e.g. an error alert) opened above the current - // one and blocks the click: dismiss any dialog that does not contain - // the element the tutorial points to, like a user would. - const dismissedDialogs = await page - .evaluate((selector) => { - const target = selector ? document.querySelector(selector) : null; - const dismissed = []; - for (const dialog of document.querySelectorAll('[role="dialog"]')) { - if (target && dialog.contains(target)) continue; - const closeButton = [...dialog.querySelectorAll('button')].find( - (button) => - /^(close|ok|cancel|got it)$/i.test( - (button.textContent || '').trim() - ) - ); - if (closeButton) { - dismissed.push((dialog.textContent || '').trim().slice(0, 120)); - closeButton.click(); - } - } - return dismissed; - }, state.elementToHighlightId || null) - .catch(() => []); - if (dismissedDialogs.length) { - log( - `Dismissed unexpected dialog(s): ${JSON.stringify( - dismissedDialogs - )}` - ); - } - } } const advanced = await waitForStepChange(page, state.stepIndex); + if (!advanced) { + // An unexpected dialog (e.g. an error alert) may have opened above the + // current one, blocking the step: dismiss any dialog that does not + // contain the element the tutorial points to, like a user would. + const dismissedDialogs = await page + .evaluate((selector) => { + const target = selector ? document.querySelector(selector) : null; + const dismissed = []; + for (const dialog of document.querySelectorAll('[role="dialog"]')) { + if (target && dialog.contains(target)) continue; + const closeButton = [...dialog.querySelectorAll('button')].find( + (button) => + /^(close|ok|cancel|got it|abandon)$/i.test( + (button.textContent || '').trim() + ) + ); + if (closeButton) { + dismissed.push((dialog.textContent || '').trim().slice(0, 120)); + closeButton.click(); + } + } + return dismissed; + }, state.elementToHighlightId || null) + .catch(() => []); + if (dismissedDialogs.length) { + log( + `Dismissed unexpected dialog(s): ${JSON.stringify(dismissedDialogs)}` + ); + } + } if (!advanced && attemptsForCurrentStep >= MAX_ATTEMPTS_PER_STEP) { // The tutorial tooltip/highlighter hides itself in some situations // (error boundary displayed, another dialog opened above): include From 8f3b1ed07e9df637fa2f2419cd209a9d14ede031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:54:47 +0200 Subject: [PATCH 15/20] Play the tutorials on desktop, tablet and mobile layouts - Three Playwright projects matching the editor's responsive thresholds: desktop (1600x900), tablet (1024x768), mobile (844x390 landscape). - CI runs them as parallel matrix jobs, each uploading its own tutorial-videos- and test-traces- artifacts. - Mobile player adaptations, mirroring real user behavior: select then drag with the drop in the visible canvas area, reopen the objects drawer when the object is hidden, and search a virtualized list for the expected item using the tooltip's bold text. - tilemapPlatformer is marked known broken on mobile: at the tile painting step the properties drawer closes and the tutorial leaves the user without any guidance. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 14 ++++-- e2e/README.md | 17 ++++--- e2e/in-app-tutorials.spec.js | 18 ++++++- e2e/lib/tutorialPlayer.js | 54 ++++++++++++++++++++- e2e/playwright.config.js | 20 +++++++- 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index 4e2e37a..5d9fd88 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -14,6 +14,13 @@ on: jobs: test-in-app-tutorials: + # One job per editor layout (see the projects in e2e/playwright.config.js): + # they run in parallel and each uploads its own videos. + strategy: + fail-fast: false + matrix: + project: [desktop, tablet, mobile] + name: test-in-app-tutorials (${{ matrix.project }}) runs-on: ubuntu-latest timeout-minutes: 90 steps: @@ -47,6 +54,7 @@ jobs: # remove them from --ignore (and from KNOWN_BROKEN_TUTORIAL_IDS in # e2e/in-app-tutorials.spec.js) once fixed. - name: Check tutorial selectors against the editor sources + if: matrix.project == 'desktop' run: npm run check-in-app-tutorial-selectors -- --gdevelop-root-path ./GDevelop --ignore flingGame - name: Install Playwright browser @@ -55,7 +63,7 @@ jobs: # Starts the editor dev server (from ./GDevelop) and plays every # tutorial. Known broken tutorials are expected to fail. - name: Play all in-app tutorials - run: npm run test-in-app-tutorials + run: npm run test-in-app-tutorials -- --project=${{ matrix.project }} # Playwright records webm, which does not play natively on macOS: # convert to mp4 (H.264) so the videos open with QuickTime/Quick Look. @@ -71,7 +79,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: tutorial-videos + name: tutorial-videos-${{ matrix.project }} path: | test-results/**/*.mp4 test-results/**/*.png @@ -86,7 +94,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-traces + name: test-traces-${{ matrix.project }} path: test-results/**/trace.zip retention-days: 14 if-no-files-found: ignore diff --git a/e2e/README.md b/e2e/README.md index c538d1b..d3f7b2d 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -29,12 +29,17 @@ referencing elements that no longer exist in the editor, without running it. The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static selector check and plays every tutorial on each push (currently on the -`automatic-tests` branch, against the `automatic-tests` branch of GDevelop). A -video of each tutorial being played (converted to mp4, playable natively on -macOS) is uploaded as the `tutorial-videos` artifact, for passing and failing -runs alike. Playwright traces of failed tests are uploaded separately as -`test-traces` (they are heavy — ~50MB per failed test); inspect one with -`npx playwright show-trace trace.zip`. +`automatic-tests` branch, against the `automatic-tests` branch of GDevelop). +The tutorials are played on the three editor layouts in parallel jobs — +desktop (1600×900), tablet (1024×768) and mobile (844×390 landscape), matching +the editor's responsive thresholds. A video of each tutorial being played +(converted to mp4, playable natively on macOS) is uploaded per layout as the +`tutorial-videos-` artifacts, for passing and failing runs alike. +Playwright traces of failed tests are uploaded separately as +`test-traces-` (they are heavy — ~50MB per failed test); inspect one +with `npx playwright show-trace trace.zip`. + +Run a single layout locally with `--project=desktop|tablet|mobile`. Known broken tutorials are listed in `KNOWN_BROKEN_TUTORIAL_IDS` (`e2e/in-app-tutorials.spec.js`) and in the `--ignore` flag of the selector diff --git a/e2e/in-app-tutorials.spec.js b/e2e/in-app-tutorials.spec.js index ea393d0..1f44420 100644 --- a/e2e/in-app-tutorials.spec.js +++ b/e2e/in-app-tutorials.spec.js @@ -24,6 +24,15 @@ const { playTutorial } = require('./lib/tutorialPlayer'); */ const KNOWN_BROKEN_TUTORIAL_IDS = ['flingGame']; +/** + * Tutorials known to be broken on the mobile layout only. + * - tilemapPlatformer: after selecting the terrain, the instance properties + * panel (a drawer on mobile) closes again, #freehandBrush disappears and + * the tutorial tooltip is not displayed anymore: a mobile user is left + * without any guidance. + */ +const KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS = ['tilemapPlatformer']; + const allTutorials = loadAllTutorials(); const tutorialIdsFilter = process.env.TUTORIAL_IDS ? process.env.TUTORIAL_IDS.split(',').map((id) => id.trim()) @@ -33,9 +42,14 @@ const tutorials = tutorialIdsFilter : allTutorials; for (const tutorial of tutorials) { - test(`in-app tutorial: ${tutorial.id}`, async ({ page, context }) => { + test(`in-app tutorial: ${tutorial.id}`, async ({ + page, + context, + }, testInfo) => { test.fail( - KNOWN_BROKEN_TUTORIAL_IDS.includes(tutorial.id), + KNOWN_BROKEN_TUTORIAL_IDS.includes(tutorial.id) || + (testInfo.project.name === 'mobile' && + KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS.includes(tutorial.id)), 'This tutorial is known to be broken.' ); await serveLocalTutorials(context, allTutorials); diff --git a/e2e/lib/tutorialPlayer.js b/e2e/lib/tutorialPlayer.js index 5eb6962..40115e5 100644 --- a/e2e/lib/tutorialPlayer.js +++ b/e2e/lib/tutorialPlayer.js @@ -351,7 +351,21 @@ const performStepAction = async ({ ); } const source = page.locator(highlightedElementSelector).first(); - await source.waitFor({ state: 'visible', timeout: 10 * 1000 }); + try { + await source.waitFor({ state: 'visible', timeout: 5 * 1000 }); + } catch (error) { + // On small layouts the objects list is a drawer that can be closed: + // open it like a user would. + await page + .locator( + '#scene-editor[data-active="true"] #toolbar-open-objects-panel-button' + ) + .first() + .click({ timeout: 5 * 1000 }); + await source.waitFor({ state: 'visible', timeout: 8 * 1000 }); + } + // The object can also be below the fold of the list. + await source.scrollIntoViewIfNeeded().catch(() => {}); const canvas = page .locator('#scene-editor[data-active="true"] canvas') .first(); @@ -362,6 +376,14 @@ const performStepAction = async ({ if (!sourceBox || !canvasBox) { throw new Error('Could not find the drag source or the scene canvas.'); } + // Select the object first ("Select then drag", as the mobile tooltips + // say — harmless on desktop), then drag. Drop in the upper part of the + // canvas: on mobile the lower part is covered by the objects drawer. + await page.mouse.click( + sourceBox.x + sourceBox.width / 2, + sourceBox.y + sourceBox.height / 2 + ); + await page.waitForTimeout(200); // The step can require several instances (`instancesCount`): drop each // one at a different position. const instancesCount = trigger.instancesCount || 1; @@ -376,7 +398,7 @@ const performStepAction = async ({ // Move in several small steps so that drag events are properly emitted. await page.mouse.move( canvasBox.x + canvasBox.width * dropFractionX, - canvasBox.y + canvasBox.height / 2, + canvasBox.y + canvasBox.height * 0.3, { steps: 20 } ); await page.waitForTimeout(200); @@ -606,6 +628,34 @@ const playTutorial = async ({ page, context, tutorial, log = () => {} }) => { } const advanced = await waitForStepChange(page, state.stepIndex); + if ( + !advanced && + attemptsForCurrentStep === 1 && + (step.nextStepTrigger || {}).presenceOfElement + ) { + // The expected element may be in a virtualized list that is too short + // to contain it on small screens: search for it like a user would, + // using the bold text of the tooltip. + const textToType = extractTextToType(step); + // Prefer the search input of the open dialog, if any (there can be + // several search inputs on screen, e.g. behind the dialog). + let searchInput = page + .locator('[role="dialog"] input[placeholder*="earch" i]') + .filter({ visible: true }) + .first(); + if (!(await searchInput.isVisible().catch(() => false))) { + searchInput = page + .locator('input[placeholder*="earch" i]') + .filter({ visible: true }) + .first(); + } + if (textToType && (await searchInput.isVisible().catch(() => false))) { + log(`Searching for "${textToType}" to reveal the expected element.`); + await searchInput.fill(textToType).catch(() => {}); + await waitForStepChange(page, state.stepIndex); + continue; + } + } if (!advanced) { // An unexpected dialog (e.g. an error alert) may have opened above the // current one, blocking the step: dismiss any dialog that does not diff --git a/e2e/playwright.config.js b/e2e/playwright.config.js index f51e1c6..f99abf3 100644 --- a/e2e/playwright.config.js +++ b/e2e/playwright.config.js @@ -46,7 +46,6 @@ module.exports = defineConfig({ reporter: process.env.CI ? [['list'], ['github']] : [['list']], use: { baseURL: editorUrl, - viewport: { width: 1600, height: 900 }, // Safety net: no action should ever block until the test timeout. actionTimeout: 15 * 1000, screenshot: 'only-on-failure', @@ -55,6 +54,25 @@ module.exports = defineConfig({ // each run can be reviewed and failures can be diagnosed. video: process.env.CI ? 'on' : 'retain-on-failure', }, + // The editor has different layouts depending on the window size (see + // UI/Responsive/ResponsiveWindowMeasurer.js: mobile is width < 600 or + // height < 500, tablet/"medium" is width < 1150). Run the tutorials on + // each: select one with --project=desktop|tablet|mobile. + projects: [ + { + name: 'desktop', + use: { viewport: { width: 1600, height: 900 } }, + }, + { + name: 'tablet', + use: { viewport: { width: 1024, height: 768 }, hasTouch: true }, + }, + { + name: 'mobile', + // Landscape phone: the editor is mostly used in landscape on mobile. + use: { viewport: { width: 844, height: 390 }, hasTouch: true }, + }, + ], webServer: gdevelopRootPath ? { command: `npm start --prefix ${path.join( From 1d9d968c7193c595b41ff7a05b4546e92954e2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:13:36 +0200 Subject: [PATCH 16/20] Production wiring: GDevelop master, hourly schedule, notifications - Check out GDevelop master (the orchestrator test-mode hook is being merged there) and run on pushes to main plus an hourly schedule, which catches editor changes breaking tutorials. - Shorter artifact retention for scheduled runs (3 days vs 14). - Ping Discord on failure when the TUTORIALS_CI_DISCORD_WEBHOOK secret is set. - flingGame is fully skipped instead of expected-to-fail: there is no plan to fix it. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 42 ++++++++++++++++----- e2e/README.md | 17 ++++++--- e2e/in-app-tutorials.spec.js | 20 +++++----- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index 5d9fd88..ecfa189 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -1,15 +1,20 @@ -# Plays every in-app tutorial against the GDevelop editor (web build) and -# fails if a tutorial step cannot be completed. A video of each tutorial being -# played is uploaded as an artifact, for passing and failing runs alike. +# Plays every in-app tutorial against the GDevelop editor (web build, master +# branch) and fails if a tutorial step cannot be completed. A video of each +# tutorial being played is uploaded as an artifact, for passing and failing +# runs alike. # -# For the moment this runs on the automatic-tests branch only, against the -# automatic-tests branch of GDevelop (which contains the orchestrator test -# hook). Once merged, switch both refs and add a schedule (hourly cron). +# Runs on every push (a tutorial change should not break the tutorials) and +# every hour (a change in GDevelop master should not either). The hourly +# schedule only runs from the default branch (main). name: Test in-app tutorials on: push: - branches: [automatic-tests] + branches: [main, automatic-tests] + schedule: + # Every hour: GDevelop pushes do not trigger this workflow, the schedule + # catches editor changes that break tutorials. + - cron: '17 * * * *' workflow_dispatch: jobs: @@ -31,7 +36,7 @@ jobs: uses: actions/checkout@v4 with: repository: 4ian/GDevelop - ref: automatic-tests + ref: master path: GDevelop - uses: actions/setup-node@v4 @@ -84,7 +89,8 @@ jobs: test-results/**/*.mp4 test-results/**/*.png test-results/**/error-context.md - retention-days: 14 + # Hourly runs would accumulate a lot of videos: keep them shorter. + retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }} if-no-files-found: warn # Traces are heavy (~50MB per failed test, with a DOM snapshot of every @@ -96,5 +102,21 @@ jobs: with: name: test-traces-${{ matrix.project }} path: test-results/**/trace.zip - retention-days: 14 + retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }} if-no-files-found: ignore + + # Ping the team when tutorials are broken. Requires the + # TUTORIALS_CI_DISCORD_WEBHOOK secret (a Discord webhook URL) to be set + # on the repository; does nothing otherwise. + - name: Notify failure + if: failure() + env: + DISCORD_WEBHOOK_URL: ${{ secrets.TUTORIALS_CI_DISCORD_WEBHOOK }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + PROJECT: ${{ matrix.project }} + run: | + if [ -n "$DISCORD_WEBHOOK_URL" ]; then + curl -sS -H "Content-Type: application/json" \ + -d "{\"content\": \"❌ In-app tutorials are broken on the $PROJECT layout: $RUN_URL\"}" \ + "$DISCORD_WEBHOOK_URL" + fi diff --git a/e2e/README.md b/e2e/README.md index d3f7b2d..c299e25 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -28,8 +28,9 @@ referencing elements that no longer exist in the editor, without running it. ## CI The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static -selector check and plays every tutorial on each push (currently on the -`automatic-tests` branch, against the `automatic-tests` branch of GDevelop). +selector check and plays every tutorial against GDevelop `master`, on each +push and every hour (the hourly schedule catches GDevelop changes that break +tutorials, since GDevelop pushes do not trigger this workflow). The tutorials are played on the three editor layouts in parallel jobs — desktop (1600×900), tablet (1024×768) and mobile (844×390 landscape), matching the editor's responsive thresholds. A video of each tutorial being played @@ -41,10 +42,14 @@ with `npx playwright show-trace trace.zip`. Run a single layout locally with `--project=desktop|tablet|mobile`. -Known broken tutorials are listed in `KNOWN_BROKEN_TUTORIAL_IDS` -(`e2e/in-app-tutorials.spec.js`) and in the `--ignore` flag of the selector -check step: they are still played and recorded, but expected to fail. When -fixing one, remove it from both places. +Untested tutorials are listed in `SKIPPED_TUTORIAL_IDS` +(`e2e/in-app-tutorials.spec.js`, currently flingGame — no plan to fix it) and +in the `--ignore` flag of the selector check step. Tutorials broken on a +single layout are in `KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS`: they are still +played and recorded, but expected to fail there. + +Failures ping Discord if the `TUTORIALS_CI_DISCORD_WEBHOOK` secret (a Discord +webhook URL) is set on the repository. ## Known dev-environment behaviors diff --git a/e2e/in-app-tutorials.spec.js b/e2e/in-app-tutorials.spec.js index 1f44420..18255be 100644 --- a/e2e/in-app-tutorials.spec.js +++ b/e2e/in-app-tutorials.spec.js @@ -15,14 +15,11 @@ const { const { playTutorial } = require('./lib/tutorialPlayer'); /** - * Tutorials that are currently known to be broken (see the findings of - * `npm run check-in-app-tutorial-selectors`). They are still played (and - * recorded) but are expected to fail. Remove a tutorial from this list once it - * is fixed — the test will then fail with "unexpectedly passed" as a reminder. + * Tutorials that are not tested at all. * - flingGame: references removed editor elements (#layer-name, - * [data-default]). + * [data-default]) and there is no plan to fix it. */ -const KNOWN_BROKEN_TUTORIAL_IDS = ['flingGame']; +const SKIPPED_TUTORIAL_IDS = ['flingGame']; /** * Tutorials known to be broken on the mobile layout only. @@ -46,11 +43,14 @@ for (const tutorial of tutorials) { page, context, }, testInfo) => { + test.skip( + SKIPPED_TUTORIAL_IDS.includes(tutorial.id), + 'This tutorial is not tested.' + ); test.fail( - KNOWN_BROKEN_TUTORIAL_IDS.includes(tutorial.id) || - (testInfo.project.name === 'mobile' && - KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS.includes(tutorial.id)), - 'This tutorial is known to be broken.' + testInfo.project.name === 'mobile' && + KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS.includes(tutorial.id), + 'This tutorial is known to be broken on this layout.' ); await serveLocalTutorials(context, allTutorials); await startTutorial(page, tutorial.id); From 1a4146344dd4064e00c37681a657d4a40cd98e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:15:12 +0200 Subject: [PATCH 17/20] Schedule every 6 hours, rely on the default CI failure notification Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 26 ++++----------------- e2e/README.md | 5 +--- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index ecfa189..a9e4714 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -4,17 +4,17 @@ # runs alike. # # Runs on every push (a tutorial change should not break the tutorials) and -# every hour (a change in GDevelop master should not either). The hourly -# schedule only runs from the default branch (main). +# every 6 hours (a change in GDevelop master should not either). The schedule +# only runs from the default branch (main). name: Test in-app tutorials on: push: branches: [main, automatic-tests] schedule: - # Every hour: GDevelop pushes do not trigger this workflow, the schedule - # catches editor changes that break tutorials. - - cron: '17 * * * *' + # Every 6 hours: GDevelop pushes do not trigger this workflow, the + # schedule catches editor changes that break tutorials. + - cron: '17 */6 * * *' workflow_dispatch: jobs: @@ -104,19 +104,3 @@ jobs: path: test-results/**/trace.zip retention-days: ${{ github.event_name == 'schedule' && 3 || 14 }} if-no-files-found: ignore - - # Ping the team when tutorials are broken. Requires the - # TUTORIALS_CI_DISCORD_WEBHOOK secret (a Discord webhook URL) to be set - # on the repository; does nothing otherwise. - - name: Notify failure - if: failure() - env: - DISCORD_WEBHOOK_URL: ${{ secrets.TUTORIALS_CI_DISCORD_WEBHOOK }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - PROJECT: ${{ matrix.project }} - run: | - if [ -n "$DISCORD_WEBHOOK_URL" ]; then - curl -sS -H "Content-Type: application/json" \ - -d "{\"content\": \"❌ In-app tutorials are broken on the $PROJECT layout: $RUN_URL\"}" \ - "$DISCORD_WEBHOOK_URL" - fi diff --git a/e2e/README.md b/e2e/README.md index c299e25..918cb9b 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -29,7 +29,7 @@ referencing elements that no longer exist in the editor, without running it. The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static selector check and plays every tutorial against GDevelop `master`, on each -push and every hour (the hourly schedule catches GDevelop changes that break +push and every 6 hours (the schedule catches GDevelop changes that break tutorials, since GDevelop pushes do not trigger this workflow). The tutorials are played on the three editor layouts in parallel jobs — desktop (1600×900), tablet (1024×768) and mobile (844×390 landscape), matching @@ -48,9 +48,6 @@ in the `--ignore` flag of the selector check step. Tutorials broken on a single layout are in `KNOWN_BROKEN_ON_MOBILE_TUTORIAL_IDS`: they are still played and recorded, but expected to fail there. -Failures ping Discord if the `TUTORIALS_CI_DISCORD_WEBHOOK` secret (a Discord -webhook URL) is set on the repository. - ## Known dev-environment behaviors - When opening a template that uses leaderboards (plinkoMultiplier) with a From 30dd3156129dbc677f29dc3cf624dd9fae312973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:43:09 +0200 Subject: [PATCH 18/20] Play only the modified tutorials on pull requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On pull requests, diff against the base branch and play only the tutorials whose json changed — all of them when the test harness itself changed, none when the changes concern neither. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 41 ++++++++++++++++++--- e2e/README.md | 6 ++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index a9e4714..68dba60 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -3,14 +3,17 @@ # tutorial being played is uploaded as an artifact, for passing and failing # runs alike. # -# Runs on every push (a tutorial change should not break the tutorials) and -# every 6 hours (a change in GDevelop master should not either). The schedule -# only runs from the default branch (main). +# Runs on every push to main (a tutorial change should not break the +# tutorials) and every 6 hours (a change in GDevelop master should not +# either). The schedule only runs from the default branch (main). +# On pull requests, only the modified tutorials are played (all of them if the +# test harness itself was modified). name: Test in-app tutorials on: push: branches: [main, automatic-tests] + pull_request: schedule: # Every 6 hours: GDevelop pushes do not trigger this workflow, the # schedule catches editor changes that break tutorials. @@ -31,6 +34,9 @@ jobs: steps: - name: Checkout tutorials uses: actions/checkout@v4 + with: + # Full history, to diff against the base branch on pull requests. + fetch-depth: 0 - name: Checkout GDevelop uses: actions/checkout@v4 @@ -65,9 +71,31 @@ jobs: - name: Install Playwright browser run: npx playwright install --with-deps chromium - # Starts the editor dev server (from ./GDevelop) and plays every - # tutorial. Known broken tutorials are expected to fail. - - name: Play all in-app tutorials + # On pull requests, only play the tutorials whose json was modified — + # or all of them if the test harness itself was modified, and none if + # the changes concern neither. + - name: Determine which tutorials to play + id: scope + if: github.event_name == 'pull_request' + run: | + changed=$(git diff --name-only "origin/$GITHUB_BASE_REF"...HEAD) + echo "Changed files:"; echo "$changed" + ids=$(echo "$changed" | grep -E '^tutorials/in-app/[^/]+\.json$' | xargs -rn1 basename | sed 's/\.json$//' | paste -sd, -) + if [ -n "$ids" ]; then + echo "ids=$ids" >> "$GITHUB_OUTPUT" + echo "mode=modified-tutorials" >> "$GITHUB_OUTPUT" + elif echo "$changed" | grep -qE '^(e2e/|scripts/|package(-lock)?\.json|\.github/workflows/)'; then + echo "mode=all" >> "$GITHUB_OUTPUT" + else + echo "mode=none" >> "$GITHUB_OUTPUT" + fi + + # Starts the editor dev server (from ./GDevelop) and plays the + # tutorials. Known broken tutorials are expected to fail. + - name: Play in-app tutorials + if: steps.scope.outputs.mode != 'none' + env: + TUTORIAL_IDS: ${{ steps.scope.outputs.ids }} run: npm run test-in-app-tutorials -- --project=${{ matrix.project }} # Playwright records webm, which does not play natively on macOS: @@ -75,6 +103,7 @@ jobs: - name: Convert videos to mp4 if: always() run: | + [ -d test-results ] || exit 0 sudo apt-get update -qq && sudo apt-get install -y -qq ffmpeg find test-results -name "*.webm" -print0 | while IFS= read -r -d '' file; do ffmpeg -nostdin -y -loglevel error -i "$file" -c:v libx264 -preset veryfast -crf 28 -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -movflags +faststart "${file%.webm}.mp4" && rm "$file" diff --git a/e2e/README.md b/e2e/README.md index 918cb9b..6fc2fc7 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -29,8 +29,10 @@ referencing elements that no longer exist in the editor, without running it. The `.github/workflows/test-in-app-tutorials.yml` workflow runs the static selector check and plays every tutorial against GDevelop `master`, on each -push and every 6 hours (the schedule catches GDevelop changes that break -tutorials, since GDevelop pushes do not trigger this workflow). +push to `main` and every 6 hours (the schedule catches GDevelop changes that +break tutorials, since GDevelop pushes do not trigger this workflow). On pull +requests, only the modified tutorials are played (all of them if the test +harness itself was modified). The tutorials are played on the three editor layouts in parallel jobs — desktop (1600×900), tablet (1024×768) and mobile (844×390 landscape), matching the editor's responsive thresholds. A video of each tutorial being played From 7da5e4aee15b22dc9975c134bb88e576dab35fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:51:31 +0200 Subject: [PATCH 19/20] Cancel superseded runs, drop the temporary automatic-tests push trigger A new push cancels the still-running workflow of the same pull request or branch. The push trigger now only covers main: pull requests are covered by the pull_request event (the automatic-tests entry caused duplicate runs once the PR was opened). Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index 68dba60..d6f7093 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -12,7 +12,7 @@ name: Test in-app tutorials on: push: - branches: [main, automatic-tests] + branches: [main] pull_request: schedule: # Every 6 hours: GDevelop pushes do not trigger this workflow, the @@ -20,6 +20,12 @@ on: - cron: '17 */6 * * *' workflow_dispatch: +# A new push cancels the still-running workflow of the same pull request or +# branch (scheduled runs are grouped separately and never cancel them). +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-in-app-tutorials: # One job per editor layout (see the projects in e2e/playwright.config.js): From 98db3d67d1a310c533465203373c59901b274026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pasteau?= <4895034+ClementPasteau@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:41:16 +0200 Subject: [PATCH 20/20] Play everything on PRs that modify the test harness Harness changes take precedence over the modified-tutorials filter: a PR changing both e2e code and a tutorial was only playing that tutorial. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-in-app-tutorials.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-in-app-tutorials.yml b/.github/workflows/test-in-app-tutorials.yml index d6f7093..95a1209 100644 --- a/.github/workflows/test-in-app-tutorials.yml +++ b/.github/workflows/test-in-app-tutorials.yml @@ -87,11 +87,12 @@ jobs: changed=$(git diff --name-only "origin/$GITHUB_BASE_REF"...HEAD) echo "Changed files:"; echo "$changed" ids=$(echo "$changed" | grep -E '^tutorials/in-app/[^/]+\.json$' | xargs -rn1 basename | sed 's/\.json$//' | paste -sd, -) - if [ -n "$ids" ]; then + if echo "$changed" | grep -qE '^(e2e/|scripts/|package(-lock)?\.json|\.github/workflows/)'; then + # The harness itself changed: play everything. + echo "mode=all" >> "$GITHUB_OUTPUT" + elif [ -n "$ids" ]; then echo "ids=$ids" >> "$GITHUB_OUTPUT" echo "mode=modified-tutorials" >> "$GITHUB_OUTPUT" - elif echo "$changed" | grep -qE '^(e2e/|scripts/|package(-lock)?\.json|\.github/workflows/)'; then - echo "mode=all" >> "$GITHUB_OUTPUT" else echo "mode=none" >> "$GITHUB_OUTPUT" fi