diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt index a3577df23b..a6089e694b 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt @@ -14,12 +14,15 @@ object BlackBoxUtils { private const val GENERATED_FOLDER_NAME = "generated" const val baseLocationForJavaScript = "$JS_BASE_PATH/$GENERATED_FOLDER_NAME" + const val baseLocationForPlaywright = "$JS_BASE_PATH/$GENERATED_FOLDER_NAME/playwright" const val baseLocationForPython = "$PY_BASE_PATH/$GENERATED_FOLDER_NAME" const val baseLocationForJava = "$MAVEN_BASE_PATH/src/test/java" const val baseLocationForKotlin = "$MAVEN_BASE_PATH/src/test/kotlin" fun relativePath(folderName: String) = "$GENERATED_FOLDER_NAME/$folderName" + fun relativePathPlaywright(folderName: String) = "$GENERATED_FOLDER_NAME/playwright/$folderName" + fun checkCoveredTargets(targetLabels: Collection) { targetLabels.forEach { assertTrue(CoveredTargets.isCovered(it), "Target '$it' is not covered") @@ -36,7 +39,6 @@ object BlackBoxUtils { private fun mvn() = if (isWindows()) "mvn.cmd" else "mvn" - private fun runNpmInstall() { val command = listOf(npm(), "ci") @@ -120,6 +122,20 @@ object BlackBoxUtils { runTestsCommand(command, JS_BASE_PATH, "NPM") } + fun runPlaywrightTests(folderRelativePath: String) { + runNpmInstall() + + val path = if(folderRelativePath.endsWith("/")){ + folderRelativePath + } else { + "$folderRelativePath/" + } + + val npx = if (isWindows()) "npx.cmd" else "npx" + val command = listOf(npx, "playwright", "test", path) + runTestsCommand(command, JS_BASE_PATH, "Playwright") + } + fun runPythonTests(folderRelativePath: String) { installPythonRequirements() @@ -151,4 +167,4 @@ object BlackBoxUtils { fun getOutputFilePrefixKotlin(outputFolderName: String) = "com.kotlin.$outputFolderName.EM" -} +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt index 30291759dc..ee78d106d8 100644 --- a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt @@ -90,6 +90,7 @@ abstract class SpringTestBase : GraphQLTestBase() { lambda: Consumer> ){ val baseLocation = when { + outputFormat.isPlaywright() -> BlackBoxUtils.baseLocationForPlaywright outputFormat.isJavaScript() -> BlackBoxUtils.baseLocationForJavaScript outputFormat.isPython() -> BlackBoxUtils.baseLocationForPython else -> throw IllegalArgumentException("Not supported output type $outputFormat") @@ -100,6 +101,7 @@ abstract class SpringTestBase : GraphQLTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ + outputFormat.isPlaywright() -> BlackBoxUtils.runPlaywrightTests(BlackBoxUtils.relativePathPlaywright(outputFolderName)) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) else -> throw IllegalArgumentException("Not supported output type $outputFormat") diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json index c85a0aca99..4333fffe1d 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json @@ -7,6 +7,7 @@ "name": "evomaster-client-js-e2e-tests", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", @@ -933,6 +934,21 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "dependencies": { + "playwright": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -3248,6 +3264,50 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -4567,6 +4627,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "requires": { + "playwright": "1.59.1" + } + }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -6292,6 +6361,31 @@ "find-up": "^4.0.0" } }, + "playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "requires": { + "fsevents": "2.3.2", + "playwright-core": "1.59.1" + }, + "dependencies": { + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + } + } + }, + "playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true + }, "pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json index 5c69b06726..aa42e14ae8 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json @@ -6,6 +6,7 @@ "author": "EvoMaster Team", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js new file mode 100644 index 0000000000..ce35c56d60 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js @@ -0,0 +1,6 @@ +const { defineConfig } = require('@playwright/test'); + +module.exports = defineConfig({ + testDir: './generated', + testMatch: /.*[tT]est\.js/, +}); diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt index 2ed918c6bb..e00b2bed9c 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt @@ -150,7 +150,6 @@ abstract class SpringTestBase : RestTestBase() { } } - fun runBlackBoxEM( outputFormat: OutputFormat, outputFolderName: String, @@ -160,6 +159,7 @@ abstract class SpringTestBase : RestTestBase() { lambda: Consumer> ){ val baseLocation = when { + outputFormat.isPlaywright() -> BlackBoxUtils.baseLocationForPlaywright outputFormat.isJavaScript() -> BlackBoxUtils.baseLocationForJavaScript outputFormat.isPython() -> BlackBoxUtils.baseLocationForPython outputFormat.isJava() -> BlackBoxUtils.baseLocationForJava @@ -172,6 +172,7 @@ abstract class SpringTestBase : RestTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ + outputFormat.isPlaywright() -> BlackBoxUtils.runPlaywrightTests(BlackBoxUtils.relativePathPlaywright(outputFolderName)) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isJava() -> BlackBoxUtils.runJavaTests(outputFolderName) @@ -230,4 +231,4 @@ abstract class SpringTestBase : RestTestBase() { } } -} +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml index 2fe734f2a0..caf3412ad5 100644 --- a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml @@ -35,6 +35,7 @@ org.projectlombok lombok + provided org.apache.commons @@ -133,6 +134,16 @@ org.apache.maven.plugins maven-compiler-plugin + 3.14.1 + + + + org.projectlombok + lombok + 1.18.30 + + + diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index c5aeb96248..21d4809802 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -18,6 +18,7 @@ enum class OutputFormat { KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, + JS_JEST_PLAYWRIGHT, // Testing new playwright imp //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed PYTHON_UNITTEST ; @@ -28,6 +29,12 @@ enum class OutputFormat { fun isJavaScript() = this.name.startsWith("js_", true) + /** + * Return true if the output format is Playwright. + * Playwright is currently only supported for JavaScript (or TypeScript). + */ + fun isPlaywright() = this.name.endsWith("_playwright", true) // Testing new playwright imp + fun isJavaOrKotlin() = isJava() || isKotlin() fun isJUnit5() = this.name.endsWith("junit_5", true) @@ -40,5 +47,4 @@ enum class OutputFormat { fun isCsharp() = this.name.startsWith("csharp",ignoreCase = true) fun isPython() = this.name.startsWith("python_", true) - } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index b3791fe751..378d6790a4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -50,12 +50,15 @@ object CookieWriter { when { format.isJava() -> lines.add("final Map ${cookiesName(k)} = ") format.isKotlin() -> lines.add("val ${cookiesName(k)} : Map = ") + format.isPlaywright() -> lines.add("let ${cookiesName(k)};") format.isJavaScript() -> lines.add("const ${cookiesName(k)} = ") } if (!format.isPython()) { // TODO: should we use DTO for cookie related requests? - testCaseWriter.startRequest(lines) + if (!format.isPlaywright()) { + testCaseWriter.startRequest(lines) + } lines.indent() } @@ -76,12 +79,17 @@ object CookieWriter { format.isPython() -> lines.append(".cookies") } - if(format.isJavaScript()){ + if(format.isJavaScript() && !format.isPlaywright()){ lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") lines.appendSemicolon() } + if (format.isPlaywright()) { + lines.add(".then(async (res) => { ${cookiesName(k)} = res.headers()['set-cookie']?.split('\\n')?.find(c => c.trim().length > 0)?.split(';')[0]; })") + lines.appendSemicolon() + } + if (format.isPython()) { lines.add("${cookiesName(k)} = requests.utils.dict_from_cookiejar($targetCookieVariable)") } @@ -110,7 +118,12 @@ object CookieWriter { targetVariable: String ) { - if(format.isJavaScript()) { + if (format.isPlaywright()) { + callEndpoint(lines, k, format, baseUrlOfSut) + return + } + + if(format.isJavaScript() && !format.isPlaywright()) { callEndpoint(lines, k, format, baseUrlOfSut) } @@ -122,6 +135,10 @@ object CookieWriter { if(contentType != null) { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${contentType.defaultValue}\")") + format.isPlaywright() -> { + // handled in request options 'data' or similar if needed, + // but usually Playwright sets it automatically if passed as object + } format.isJavaScript() -> lines.add(".set(\"content-type\", \"${contentType.defaultValue}\")") format.isPython() -> { lines.add("headers[\"content-type\"] = \"${contentType.defaultValue}\"") @@ -150,6 +167,9 @@ object CookieWriter { for(header in k.headers) { when { format.isJavaOrKotlin() -> lines.add(".header(\"${header.name}\", \"${header.value}\")") + format.isPlaywright() -> { + // handled in callEndpoint for Playwright + } format.isJavaScript() -> lines.add(".set(\"${header.name}\", \"${header.value}\")") format.isPython() -> { lines.add("headers[\"${header.name}\"] = \"${header.value}\"") @@ -157,7 +177,7 @@ object CookieWriter { } } - if (format.isJavaScript()){ + if (format.isJavaScript() && !format.isPlaywright()){ // disable redirections lines.add(".redirects(0)") } @@ -170,6 +190,7 @@ object CookieWriter { callEndpoint(lines, k, format, baseUrlOfSut) } + if (format.isPython()) { lines.add("$targetVariable = requests \\") lines.indent(2) @@ -189,7 +210,13 @@ object CookieWriter { baseUrlOfSut: String ) { val verb = k.verb.name.lowercase() - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("await request.$verb(") + } else { + lines.add(".$verb(") + } + if (k.externalEndpointURL != null) { lines.append("\"${k.externalEndpointURL}\"") } else { @@ -200,6 +227,33 @@ object CookieWriter { } lines.append("${k.endpoint}\"") } + + if (format.isPlaywright()) { + lines.append(", {") + lines.indented { + if (k.headers.isNotEmpty() || k.contentType != null) { + lines.add("headers: {") + lines.indented { + if (k.contentType != null) { + lines.add("'Content-Type': '${k.contentType.defaultValue}',") + } + for (header in k.headers) { + lines.add("'${header.name}': '${header.value}',") + } + } + lines.add("},") + } + if (k.payload != null) { + if (k.contentType == ContentType.JSON) { + lines.add("data: ${k.payload},") + } else { + lines.add("data: '${k.payload}',") + } + } + } + lines.add("}") + } + if (!format.isPython()) { lines.append(")") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index ec2df92b7e..81f92ef44f 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -52,11 +52,16 @@ object TokenWriter { when { format.isJava() -> lines.add("final String ${tokenName(k)} = ") format.isKotlin() -> lines.add("val ${tokenName(k)} : String = ") + format.isPlaywright() -> { + lines.add("let ${tokenName(k)};") + } format.isJavaScript() -> lines.add("let ${tokenName(k)} = ") } when{ format.isJavaOrKotlin() -> lines.append("given()") + format.isPlaywright() -> { + } format.isJavaScript() -> { lines.append("\"\"") lines.appendSemicolon() @@ -85,7 +90,10 @@ object TokenWriter { when(token.extractFrom){ TokenHandling.ExtractFrom.BODY -> { - if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;},") + lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } + } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") } } else if (format.isPython()) { @@ -100,7 +108,10 @@ object TokenWriter { } TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector - if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];},") + lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } + } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } } else if (format.isPython()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index f84839f380..61de4bd224 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -277,6 +277,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { //TODO would not this fail on recursive/nested calls??? format.isJava() -> ".body(\"${k}isEmpty()\", is(true))" format.isKotlin() -> ".body(\"${k}isEmpty()\", `is`(true))" //'is' is a keyword in Kotlin + format.isPlaywright() -> "expect(Object.keys(await $responseVariableName.json()${k}).length).toBe(0);" format.isJavaScript() -> "expect(Object.keys($responseVariableName.body${k}).length).toBe(0);" format.isCsharp() -> "Assert.True($responseVariableName${k}.ToString() == \"{}\");" format.isPython() -> "assert len($responseVariableName.json()${k}) == 0" @@ -343,9 +344,16 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { + val field = when { + format.isPlaywright() -> "(await $responseVariableName.json())" + format.isJavaScript() -> "$responseVariableName.body" + else -> "" + } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" - format.isJavaScript() -> "expect($responseVariableName.body$fieldPath).toBe(null);" + format.isPlaywright() -> "expect(($field)$fieldWithDot).toBe(null);" + format.isJavaScript() -> "expect($field$fieldPath).toBe(null);" // ($field$)fieldPath format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" else -> throw IllegalStateException("Format not supported yet: $format") @@ -395,10 +403,18 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (isSuitableToPrint(toPrint)) { if (format.isJavaScript() || format.isPython()) { + val field = when { + format.isPlaywright() -> "(await $responseVariableName.json())" + format.isJavaScript() -> "$responseVariableName.body" + else -> "" + } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" + }else if (format.isPlaywright()){ // playwright + "expect($field$fieldWithDot).toBe($toPrint);" }else { // javascript - "expect($responseVariableName.body$fieldPath).toBe($toPrint);" + "expect($field$fieldPath).toBe($toPrint);" // ($field$)fieldPath } if (flakyValue == null || flakyValue == value){ @@ -543,6 +559,9 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { + if (format.isPlaywright()) { + return "expect(await $responseVariableName.text()).toBe(\"\");" + } /* This is super ugly... but there is no clean solution for this in Jest nor SuperAgent... :( @@ -578,8 +597,15 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val path = if (fieldPath.isEmpty()) "" else "$fieldPath." ".body(\"${path}size()\", equalTo($expectedSize))" } - format.isJavaScript() -> - "expect($responseVariableName.body$fieldPath.length).toBe($expectedSize);" + format.isJavaScript() -> { + if (format.isPlaywright()) { + val field = if (fieldPath.isEmpty()) "" else if (fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" + "expect((await $responseVariableName.json())$field).toHaveLength($expectedSize);" + } else { + val field = "$responseVariableName.body" + "expect($field$fieldPath.length).toBe($expectedSize);" // ($field$)fieldPath + } + } format.isCsharp() -> "Assert.True($responseVariableName$fieldPath.Count == $expectedSize);" format.isPython() -> @@ -599,7 +625,8 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { - return "expect($responseVariableName.text).toBe(\"$content\");" + val contentCall = if (format.isPlaywright()) "await $responseVariableName.text()" else "$responseVariableName.text" + return "expect($contentCall).toBe(\"$content\");" } if (format.isCsharp()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 30962d25e7..59ce50244f 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -52,6 +52,9 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"application/json\")") + format.isPlaywright() -> { + // Handled in callEndpoint for Playwright + } format.isJavaScript() -> lines.add(".set('Content-Type','application/json')") format.isPython() -> lines.add("headers[\"content-type\"] = \"application/json\"") // format.isCsharp() -> lines.add("Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(\"application/json\"));") @@ -95,10 +98,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } override fun handleVerbEndpoint(baseUrlOfSut: String, _call: HttpWsAction, lines: Lines) { - // TODO maybe in future might want to have GET for QUERY types val verb = "post" - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("request.$verb(") + } else { + lines.add(".$verb(") + } if(config.blackBox){ /* diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index e742406143..da04e9393b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -60,7 +60,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + format.isPlaywright() -> lines.append("await request") + format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -103,6 +104,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") + format.isPlaywright() -> lines.append("const $resVarName = ") format.isJavaScript() -> lines.append("const $resVarName = ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") @@ -111,6 +113,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") + format.isPlaywright() -> {} // already handled in handleVerbEndpoint in RestTestCaseWriter format.isJavaScript() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") @@ -186,6 +189,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" + format.isPlaywright() -> "'Accept': " format.isJavaScript() -> ".set('Accept', " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " @@ -195,8 +199,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun closeAcceptHeader(openedHeader: String): String { var result = openedHeader - if (!config.outputFormat.isPython()) { - result += ")" + when { + format.isPlaywright() -> result += "," + !config.outputFormat.isPython() -> result += ")" } if (format.isCsharp()){ result = "$result;" @@ -211,7 +216,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { but that is not the case for the other libraries used for example in JS and C# */ return config.enableBasicAssertions && - (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.PYTHON_UNITTEST) + (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.JS_JEST_PLAYWRIGHT || config.outputFormat == OutputFormat.PYTHON_UNITTEST) } protected fun handleHeaders(call: HttpWsAction, lines: Lines) { @@ -227,7 +232,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val set = when { format.isJavaOrKotlin() -> "header" - format.isJavaScript() -> "set" + format.isJavaScript() && !format.isPlaywright()-> "set" + format.isPlaywright() -> "" // headers are handled in a map in the options object format.isPython() -> "headers = {}" else -> throw IllegalArgumentException("Not supported format: $format") } @@ -236,10 +242,21 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add(set) } + val bodyParam = call.parameters.find { p -> p is BodyParam } as BodyParam? + + if (format.isPlaywright() && bodyParam != null) { + val contentType = bodyParam.contentType() + if (contentType != null) { + lines.add("'Content-Type': '$contentType',") + } + } + //headers in specified auth info call.auth.headers.forEach { if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${it.value}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${it.value}', // ${call.auth.name}") } else { lines.add(".$set(\"${it.name}\", \"${it.value}\") // ${call.auth.name}") } @@ -257,6 +274,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escapedHeader = GeneUtils.applyEscapes(x, GeneUtils.EscapeMode.BODY, format) if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${escapedHeader}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${escapedHeader}',") } else { lines.add(".$set(\"${it.name}\", \"${escapedHeader}\")") @@ -270,13 +289,16 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val tokenHeader = elc.token!!.sendName if (format.isPython()) { lines.add("headers[\"$tokenHeader\"] = ${TokenWriter.authPayloadName(elc)} # ${call.auth.name}") + } else if (format.isPlaywright()) { + lines.add("'$tokenHeader': ${TokenWriter.authPayloadName(elc)}, // ${call.auth.name}") } else { lines.add(".$set(\"$tokenHeader\", ${TokenWriter.authPayloadName(elc)}) // ${call.auth.name}") } } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") - format.isJavaScript() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + format.isPlaywright() -> lines.add("'Cookie': ${CookieWriter.cookiesName(elc)},") // Python cookies are set alongside the headers and body when performing the request } } @@ -303,7 +325,19 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val code = res.getStatusCode() when { - format.isJavaScript() -> { + + format.isJavaScript() && format.isPlaywright() -> { + val statusAssert = "expect($responseVariableName.status()).toBe($code);" + if (res.getFlakyStatusCode() == null){ + lines.add(statusAssert) + }else{ + lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) + lines.addSingleCommentLine(statusAssert) + } + lines.addEmpty() + } + + format.isJavaScript() && !format.isPlaywright() -> { val statusAssert = "expect($responseVariableName.status).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) @@ -436,6 +470,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $varName = System.currentTimeMillis()") } else if(format.isPython()) { lines.addStatement("$varName = time.perf_counter() * 1000") + } else if(format.isPlaywright()) { + lines.addStatement("const $varName = performance.now()") } else if(format.isJavaScript()) { lines.addStatement("$varName = performance.now()") } @@ -455,6 +491,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $finalVarName = System.currentTimeMillis() - $varName") } else if(format.isPython()) { lines.addStatement("$finalVarName = (time.perf_counter() * 1000) - $varName") + } else if(format.isPlaywright()) { + lines.addStatement("const $finalVarName = performance.now() - $varName") } else if(format.isJavaScript()) { lines.addStatement("$finalVarName = performance.now() - $varName") } @@ -466,6 +504,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: SQL Injection vulnerability detected in this call. Expected response time (sqliInjectedSleepDurationMs) should be greater than ${config.sqliInjectedSleepDurationMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName > ${config.sqliInjectedSleepDurationMs})") + format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isPython() -> lines.addStatement("assert $finalVarName > ${config.sqliInjectedSleepDurationMs}") else -> {} @@ -474,7 +513,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: No SQL Injection vulnerability detected in this call. Expected response time (sqliBaselineMaxResponseTimeMs) should be less than ${config.sqliBaselineMaxResponseTimeMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName < ${config.sqliBaselineMaxResponseTimeMs})") - format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isJavaScript()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") format.isPython() -> lines.addStatement("assert $finalVarName < ${config.sqliBaselineMaxResponseTimeMs}") else -> {} } @@ -508,11 +548,31 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isJavaScript() -> { lines.indent(2) - //in SuperAgent, verb must be first - handleVerbEndpoint(baseUrlOfSut, call, lines) - lines.append(getAcceptHeader(call, res)) - handleHeaders(call, lines) - handleBody(call, lines) + if (format.isPlaywright()) { + handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.replaceInCurrent(Regex("\\)$"), "") + lines.append(", {") + lines.addEmpty() + lines.indented { + if (call is org.evomaster.core.problem.rest.data.RestCallAction) { + lines.add("method: \"${call.verb.name.uppercase()}\",") + } + lines.add("headers: {") + lines.indented { + lines.add(getAcceptHeader(call, res)) + handleHeaders(call, lines) + } + lines.add("},") + handleBody(call, lines, dtoVar) + } + lines.add("})") + } else { + //in SuperAgent, verb must be first + handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.append(getAcceptHeader(call, res)) + handleHeaders(call, lines) + handleBody(call, lines) + } } format.isCsharp() -> { @@ -540,7 +600,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() -> "send" + format.isJavaScript() && !format.isPlaywright() -> "send" + format.isPlaywright() -> "data" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -562,7 +623,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")") - format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isPlaywright() -> { + // handled in makeHttpCall through options object + } format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"") } @@ -580,12 +644,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val body = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.TEXT, targetFormat = format) // handle body only if it is not black - if (body.isNotBlank()){ + if (body.isNotBlank()) { if (body != "\"\"") { when { format.isCsharp() -> { lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")") } + format.isPython() -> { if (body.trim().isNullOrBlank()) { lines.add("body = \"\"") @@ -593,16 +658,29 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = $body") } } - else -> lines.add(".$send($body)") + + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: $body,") + } + + else -> { + lines.add(".$send($body)") + } } } else { when { format.isCsharp() -> { lines.append("new StringContent(\"${"""\"\""""}\", Encoding.UTF8, \"${bodyParam.contentType()}\")") } + format.isPython() -> { lines.add("body = \"\"") } + + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"${"""\"\""""}\",") + } + else -> lines.add(".$send(\"${"""\"\""""}\")") } } @@ -624,6 +702,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$body\"") } + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"$body\",") + } else -> lines.add(".$send(\"$body\")") } } else if (bodyParam.isXml()) { @@ -636,6 +717,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$escapedXml\"") } + format.isPlaywright() -> { + lines.add("data: \"$escapedXml\",") + } else -> lines.add(".$send(\"$escapedXml\")") } } else { @@ -663,7 +747,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = ${bodyLines.first()}") } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, false) + } + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) // Needs to be checked! fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, false) } } else { @@ -688,7 +775,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("${bodyLines.last()}") } } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, true) + } + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) // Needs to be checked fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, true) } } @@ -722,6 +812,20 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append(")") } + private fun writePlaywrightPayload(lines: Lines, bodyLines: List, isMultiLine: Boolean) { + lines.add("data: ${bodyLines.first()}") + if (isMultiLine) { + lines.append(" + ") + lines.indented { + (1 until bodyLines.lastIndex).forEach { i -> + lines.add("${bodyLines[i]} + ") + } + lines.add("${bodyLines.last()}") + } + } + lines.append(",") + } + /** * This is done mainly for RestAssured */ @@ -741,6 +845,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } + format.isPlaywright() -> { + // assertions for Playwright are handled in handleResponseAfterTheCall, + // as they cannot be chained directly in the request call + } + else -> throw IllegalStateException("No assertion in calls for format: $format") } @@ -757,7 +866,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handleResponseAssertions(lines: Lines, res: HttpWsCallResult, responseVariableName: String?) { - assert(responseVariableName != null || format.isJavaOrKotlin()) + assert(responseVariableName != null || format.isJavaOrKotlin() || format.isPlaywright()) /* there are 2 cases: @@ -777,8 +886,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isJavaScript() -> - "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" + format.isPlaywright() -> "expect($responseVariableName.headers()[\"allow\"]).toContain(\"$allow\")" + format.isJavaScript() -> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") } @@ -809,6 +918,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" + format.isPlaywright() -> "expect($responseVariableName.headers()[\"content-type\"]).toContain(\"$bodyTypeSimplified\")" format.isJavaScript() -> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" @@ -873,8 +983,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { - - if (format.isJavaScript()) { + if (format.isPlaywright()) { + // No extra processing for Playwright + } else if(format.isJavaScript()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... @@ -928,9 +1039,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } val jsonPath = JsonUtils.fromPointerToPath(jsonPointer) + val dictAccess = JsonUtils.fromPointerToDictionaryAccess(jsonPointer) return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" + format.isPlaywright() -> " ((await $resVarName.json())$dictAccess)?.toString()" format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 9ae514e51d..8dc94680bf 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -201,7 +201,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isJava() -> lines.add("String $name = ") format.isKotlin() -> lines.add("val $name : String? = ") - format.isJavaScript() -> lines.add("const $name = ") + format.isJavaScript() || format.isPlaywright() -> lines.add("const $name = ") format.isPython() -> {lines.add("$name = ")} // should never happen else -> throw IllegalStateException("Unsupported format $format") @@ -217,14 +217,23 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val call = _call as RestCallAction val verb = call.verb.name.lowercase() - if (format.isCsharp()) { - lines.append(".${StringUtils.capitalization(verb)}Async(") - } else { - if (verb == "trace" && format.isJavaOrKotlin()) { - //currently, RestAssured does not have a trace() method - lines.add(".request(io.restassured.http.Method.TRACE, ") - } else { - lines.add(".$verb(") + when { + format.isPlaywright() -> { + val verbToUse = call.verb.name.lowercase() + if (verbToUse.uppercase() == "OPTIONS") { + lines.add("await request.fetch(") + } else { + lines.add("await request.$verbToUse(") + } + } + format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") + else -> { + if (verb == "trace" && format.isJavaOrKotlin()) { + //currently, RestAssured does not have a trace() method + lines.add(".request(io.restassured.http.Method.TRACE, ") + } else { + lines.add(".$verb(") + } } } @@ -413,7 +422,11 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { lines.add("assertTrue(isValidURIorEmpty($location));") } format.isJavaScript() -> { - lines.add("const $location = $resVarName.header['location'];") + if (format.isPlaywright()) { + lines.add("const $location = $resVarName.headers()['location'];") + } else { + lines.add("const $location = $resVarName.header['location'];") + } val validCheck = "${TestSuiteWriter.jsImport}.isValidURIorEmpty($location)" lines.add("expect($validCheck).toBe(true);") } @@ -445,7 +458,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val extract = extractValueFromJsonResponse(resVarName, idPointer) when{ - format.isJavaScript() -> lines.add("const ") + format.isJavaScript() || format.isPlaywright() -> lines.add("const ") format.isJava() -> lines.add("String ") format.isKotlin() -> lines.add("val ") format.isPython() -> lines.add("")/* nothing to do in Python */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 7f286c394a..e12d28b0a4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -117,7 +117,8 @@ abstract class TestCaseWriter { when { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") - format.isJavaScript() -> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() && !format.isPlaywright()-> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() && format.isPlaywright() -> lines.add("test(\"${test.name}\", async ({ request }) => {") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } @@ -138,7 +139,7 @@ abstract class TestCaseWriter { lines.add("}") } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // add to code block - ok for playwright? lines.append(");") } return lines @@ -310,6 +311,12 @@ abstract class TestCaseWriter { testSuitePath: Path?, baseUrlOfSut: String ) { + val playwrightExpectException = format.isPlaywright() && shouldFailIfExceptionNotThrown(res) + val hasThrownVar = if (playwrightExpectException) "hasThrown_${counter++}" else "" + + if (playwrightExpectException) { + lines.add("let $hasThrownVar = false;") + } when { /* TODO do we need to handle differently in JS due to Promises? @@ -372,6 +379,12 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("except Exception as e:") } + if (playwrightExpectException) { + lines.indented { + lines.add("$hasThrownVar = true;") + } + } + res.getErrorMessage()?.let { lines.indented { lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}") @@ -385,6 +398,9 @@ abstract class TestCaseWriter { } else { lines.add("}") } + if (playwrightExpectException) { + lines.add("expect($hasThrownVar).toBe(true);") + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index fe08c169ec..9c09379941 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -539,7 +539,11 @@ class TestSuiteWriter { } if (format.isJavaScript()) { - lines.add("const superagent = require(\"superagent\");") + if (format.isPlaywright()) { + lines.add("const { test, expect, request } = require('@playwright/test');") + } else { + lines.add("const superagent = require(\"superagent\");") + } val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) @@ -548,7 +552,8 @@ class TestSuiteWriter { if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") } - if (config.testTimeout > 0) { + + if (config.testTimeout > 0 && !format.isPlaywright()) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") } } @@ -740,7 +745,11 @@ class TestSuiteWriter { lines.add("const $controller = new $controllerName();") lines.add("let $baseUrlOfSut;") } else { - lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + if (config.outputFormat.isPlaywright()) { + lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + } else { + lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + } } } else if (config.outputFormat.isCsharp()) { lines.add("private static readonly HttpClient Client = new HttpClient ();") @@ -791,7 +800,8 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun initClass()") } - format.isJavaScript() -> lines.add("beforeAll( async () =>") + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("(async ({ request }) =>") } lines.block { @@ -923,7 +933,7 @@ class TestSuiteWriter { testCaseWriter.addExtraInitStatement(lines) } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // End statement of blocks. Valid for playwright too lines.append(");") } } @@ -946,7 +956,8 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun tearDown()") } - format.isJavaScript() -> lines.add("afterAll( async () =>") + format.isJavaScript() && !format.isPlaywright()-> lines.add("afterAll( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.afterAll(async () =>") } if (!format.isCsharp()) { @@ -999,7 +1010,9 @@ class TestSuiteWriter { format.isKotlin() -> { lines.add("fun initTest()") } - format.isJavaScript() -> lines.add("beforeEach(async () => ") + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeEach( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeEach(async () =>") + //for C# we are actually setting up the constructor for the test class format.isCsharp() -> lines.add("public ${name.getClassName()} ($fixtureClass fixture)") } diff --git a/docs/options.md b/docs/options.md index 1eea634712..cfb45bfbb3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -42,7 +42,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_JEST_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`ratePerMinute`| __Int__. Rate limiter, of how many actions to do per minute. For example, when making HTTP calls towards an external service, might want to limit the number of calls to avoid bombarding such service (which could end up becoming equivalent to a DoS attack). A value of zero or negative means that no limiter is applied. This is needed only for black-box testing of remote services. Note that, evan without this parameter, EvoMaster will still respect the Retry-After given back in 429 responses. *Default value*: `0`.| |`header0`| __String__. In black-box testing, we still need to deal with authentication of the HTTP requests. With this parameter it is possible to specify a HTTP header that is going to be added to most requests. This should be provided in the form _name:value_. If more than 1 header is needed, use as well the other options _header1_ and _header2_. *Constraints*: `regex (.+:.+)\|(^$)`. *Default value*: `""`.|