backport: improve e2e testing approach to reduce flakiness

packages/wrangler/e2e/vitest.config.mts

@@ -9,7 +9,9 @@ export default defineConfig({
 				singleThread: true,
 			},
 		},
-		include: ["e2e/**/*.test.ts"],
+		retry: 1,
+		// eslint-disable-next-line turbo/no-undeclared-env-vars
+		include: [process.env.WRANGLER_E2E_TEST_FILE || "e2e/**/*.test.ts"],
 		// eslint-disable-next-line turbo/no-undeclared-env-vars
 		outputFile: process.env.TEST_REPORT_PATH ?? ".e2e-test-report/index.html",
 		globalSetup: path.resolve(__dirname, "./validate-environment.ts"),

packages/wrangler/turbo.json

@@ -3,73 +3,76 @@
 	"extends": ["//"],
 	"tasks": {
 		"build": {
-			"inputs": [
-				"!*/**/__tests__/**",
-				"!*/**/.wrangler/**",
-				"bin/**",
-				"src/**",
-				"scripts/**",
-				"templates/**",
-				"*.json",
-				"*.js",
-				"*.ts"
-			],
+			"inputs": ["$TURBO_DEFAULT$", "!**/__tests__/**", "!e2e/**"],
 			"outputs": [
 				"miniflare-dist/**",
 				"emitted-types/**",
 				"wrangler-dist/**",
 				"config-schema.json"
 			],
 			"env": [
-				"NODE_EXTRA_CA_CERTS",
-				"CLOUDFLARE_ACCOUNT_ID",
-				"SOURCEMAPS",
-				"NODE_ENV",
-				"SPARROW_SOURCE_KEY",
 				"ALGOLIA_APP_ID",
 				"ALGOLIA_PUBLIC_KEY",
-				"CLOUDFLARE_API_TOKEN",
-				"CLOUDFLARE_ACCOUNT_ID",
-				"WRANGLER_AUTH_DOMAIN",
-				"PATH",
-				"WRANGLER_LOG",
-				"EXPERIMENTAL_MIDDLEWARE",
-				"FORMAT_WRANGLER_ERRORS",
+				"NODE_ENV",
+				"SOURCEMAPS",
+				"SPARROW_SOURCE_KEY",
+				"SENTRY_DSN",
+				"WRANGLER_PRERELEASE_LABEL"
+			],
+			"passThroughEnv": [
+				"CF_PAGES_UPLOAD_JWT",
 				"CF_PAGES",
-				"WORKERS_CI",
 				"CI",
-				"CF_PAGES_UPLOAD_JWT",
+				"CLOUDFLARE_ACCOUNT_ID",
+				"CLOUDFLARE_API_TOKEN",
+				"CUSTOM_BUILD_VAR",
 				"EXPERIMENTAL_MIDDLEWARE",
+				"FORMAT_WRANGLER_ERRORS",
+				"http_proxy",
+				"HTTP_PROXY",
+				"https_proxy",
+				"HTTPS_PROXY",
+				"HYPERDRIVE_DATABASE_URL",
+				"WRANGLER_DOCKER_BIN",
+				"WRANGLER_DOCKER_HOST",
+				"LC_ALL",
 				"NO_D1_WARNING",
 				"NO_HYPERDRIVE_WARNING",
-				"WRANGLER",
-				"WRANGLER_IMPORT",
-				"CUSTOM_BUILD_VAR",
+				"PATH",
 				"PWD",
-				"LC_ALL",
-				"WRANGLER_SEND_METRICS",
-				"https_proxy",
-				"HTTPS_PROXY",
-				"http_proxy",
-				"HTTP_PROXY",
-				"CI_OS",
-				"SENTRY_DSN",
 				"SYSTEMROOT",
 				"TZ",
+				"WORKERS_CI",
+				"WRANGLER_API_ENVIRONMENT",
+				"WRANGLER_AUTH_DOMAIN",
+				"WRANGLER_D1_EXTRA_LOCATION_CHOICES",
 				"WRANGLER_DISABLE_EXPERIMENTAL_WARNING",
 				"WRANGLER_DISABLE_REQUEST_BODY_DRAINING",
+				"WRANGLER_LOG",
+				"WRANGLER_SEND_METRICS",
 				"WRANGLER_WORKER_REGISTRY_PORT",
-				"WRANGLER_API_ENVIRONMENT",
-				"HYPERDRIVE_DATABASE_URL"
+				"DOCKER_HOST",
+				"WRANGLER_DOCKER_HOST"
 			]
 		},
 		"test:ci": {
-			"inputs": ["!*/**/.wrangler/**", "**/__tests__/**"],
-			"dependsOn": ["build"]
+			"dependsOn": ["build"],
+			"env": ["VITEST", "NODE_DEBUG", "MINIFLARE_WORKERD_PATH"]
 		},
 		"test:e2e": {
 			"inputs": ["e2e/**"],
-			"dependsOn": ["build"]
+			"dependsOn": ["build"],
+			"env": [
+				"VITEST",
+				"NODE_DEBUG",
+				"MINIFLARE_WORKERD_PATH",
+				"WRANGLER",
+				"WRANGLER_IMPORT",
+				"MINIFLARE_IMPORT",
+				"CLOUDFLARE_ACCOUNT_ID",
+				"CLOUDFLARE_API_TOKEN",
+				"WRANGLER_E2E_TEST_FILE"
+			]
 		}
 	}
 }

tools/e2e/runIndividualE2EFiles.ts

@@ -1,32 +1,68 @@
-/**
- * Turbo only supports caching on the individual task level, but for Wrangler's
- * e2e tests we want to support caching on a more granular basis—at the file level.
- *
- * As such, we run the `test:e2e` turbo task multiple times—once per e2e test file
- * with different arguments, ensuring that each file's tests can be cached individually.
- *
- * The intended flow here is that CI will run this file, which will trigger turbo to run
- * an individual task for each Wrangler e2e test file, using `execSync`.
- */
+import assert from "assert";
 import { execSync } from "child_process";
-import { readdirSync } from "fs";
+import { statSync } from "fs";
+import path from "path";
+import { globIterateSync } from "glob";
+import type { ExecSyncOptionsWithBufferEncoding } from "child_process";
 
-// Get a list of e2e test files, each of which should have an associated script
-const e2eTests = readdirSync("packages/wrangler/e2e");
+// Turbo only supports caching on the individual task level, but for Wrangler's
+// e2e tests we want to support caching on a more granular basis - at the file level.
+//
+// As such, we run the `test:e2e` turbo task multiple times — once per e2e test file so that each file's tests can be cached individually.
+// We use the `WRANGLER_E2E_TEST_FILE` environment variable to pass the specific test file to the e2e test runner so that it reuses the cached build tasks.
+// If you use a command line argument to do this turbo will create a different cache key for the build tasks.
+//
+// The intended flow here is that CI will run this file, which will trigger turbo to run
+// an individual task for each Wrangler e2e test file, using `execSync`.
+//
+// Any params after a `--` will be passed to the Vitest runner, so you can use this to configure the test run.
+// For example to update the snapshots for all Wrangler e2e tests, you can run:
+//
+// ```bash
+// pnpm test:e2e:wrangler -- -u
+// ```
 
-const tasks = new Set<string>();
+const extraParamsIndex = process.argv.indexOf("--");
+const extraParams =
+	extraParamsIndex === -1 ? [] : process.argv.slice(extraParamsIndex);
+const command =
+	`pnpm test:e2e --log-order=stream --output-logs=new-only --summarize --filter wrangler ` +
+	extraParams.join(" ");
 
-for (const file of e2eTests) {
-	// Ignore other files in the e2e directory (the README, for instance)
-	if (file.endsWith(".test.ts")) {
-		tasks.add(
-			`pnpm test:e2e --log-order=stream --output-logs=new-only --summarize --filter wrangler --concurrency 1 -- run ./e2e/${file}`
-		);
+const failed: string[] = [];
+
+const wranglerPath = path.join(__dirname, "../../packages/wrangler");
+assert(statSync(wranglerPath).isDirectory());
+
+for (const testFile of globIterateSync("e2e/**/*.test.ts", {
+	cwd: wranglerPath,
+	// Return `/` delimited paths, even on Windows.
+	posix: true,
+})) {
+	const options: ExecSyncOptionsWithBufferEncoding = {
+		stdio: "inherit",
+		env: { ...process.env, WRANGLER_E2E_TEST_FILE: testFile },
+	};
+
+	console.log(`::group::Testing: ${testFile}`);
+
+	try {
+		execSync(command, options);
+	} catch {
+		console.error("Task failed - retrying");
+		try {
+			execSync(command, options);
+		} catch {
+			console.error("Still failed, moving on");
+			failed.push(testFile);
+		}
 	}
+	console.log("::endgroup::");
 }
 
-for (const task of tasks.values()) {
-	execSync(task, {
-		stdio: "inherit",
-	});
+if (failed.length > 0) {
+	throw new Error(
+		"At least one task failed (even on retry):" +
+			failed.map((file) => `\n - ${file}`)
+	);
 }

tools/package.json

@@ -14,6 +14,7 @@
 		"@typescript-eslint/eslint-plugin": "^6.9.0",
 		"@typescript-eslint/parser": "^6.9.0",
 		"find-up": "^6.3.0",
+		"glob": "^10.4.5",
 		"ts-dedent": "^2.2.0",
 		"undici": "catalog:default"
 	}

turbo.json

@@ -3,13 +3,16 @@
 	"remoteCache": {
 		"signature": true
 	},
-	"globalEnv": [
+	"globalEnv": ["CI_OS", "NODE_VERSION"],
+	"globalPassThroughEnv": [
+		"NODE_EXTRA_CA_CERTS",
 		"CI",
-		"CI_OS",
-		"NODE_VERSION",
-		"VITEST",
-		"NODE_DEBUG",
-		"NODE_EXTRA_CA_CERTS"
+		"VSCODE_INSPECTOR_OPTIONS",
+		"WRANGLER_LOG_PATH",
+		"TEST_REPORT_PATH",
+		"CLOUDFLARE_CONTAINER_REGISTRY",
+		"DOCKER_HOST",
+		"WRANGLER_DOCKER_HOST"
 	],
 	"tasks": {
 		"dev": {
@@ -27,24 +30,20 @@
 			"persistent": true,
 			"cache": false
 		},
-		"topological": {
-			"dependsOn": ["^topological"]
-		},
 		"check:lint": {
-			"dependsOn": ["topological"]
+			"dependsOn": ["^check:lint"]
 		},
 		"check:type": {
-			"dependsOn": ["topological"]
+			"dependsOn": ["build"]
 		},
 		"type:tests": {
-			"dependsOn": ["topological"]
+			"dependsOn": ["build"]
 		},
 		"test:ci": {
 			"dependsOn": ["build"],
 			"outputLogs": "new-only"
 		},
 		"test:e2e": {
-			"env": ["WRANGLER", "CLOUDFLARE_ACCOUNT_ID", "CLOUDFLARE_API_TOKEN"],
 			"dependsOn": ["build"],
 			"outputLogs": "new-only"
 		},