From 05d6352617e0b5fdac503987ff610bf5a41da85a Mon Sep 17 00:00:00 2001 From: ferpasri Date: Fri, 13 Mar 2026 16:32:50 +0100 Subject: [PATCH 01/12] Update JS getIsBlockedTestar logic --- webdriver/resources/web-extension/js/testar.state.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/webdriver/resources/web-extension/js/testar.state.js b/webdriver/resources/web-extension/js/testar.state.js index aa00eab5a..078820916 100644 --- a/webdriver/resources/web-extension/js/testar.state.js +++ b/webdriver/resources/web-extension/js/testar.state.js @@ -381,8 +381,9 @@ function getIsBlockedTestar(element, xOffset, yOffset, clientRect) { return false; } - // Ignore encapsulated childs of - if (element.tagName === "A" && element.contains(elem)) { + // If hit-testing returns the element itself or one of its descendants, + // then it is not blocked (e.g. ). + if (elem === element || element.contains(elem)) { return false; } @@ -391,9 +392,8 @@ function getIsBlockedTestar(element, xOffset, yOffset, clientRect) { return false; } - // return whether obscured element has same parent node - // (will also return false if element === elem) - return elem.parentNode !== element.parentNode; + // Otherwise, treat as blocked by another element. + return true; } /* From 1b4329a8e4e931f740a6925531f98c298e0cdbef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 08:30:09 +0000 Subject: [PATCH 02/12] Bump io.appium:java-client from 10.0.0 to 10.1.0 Bumps [io.appium:java-client](https://github.com/appium/java-client) from 10.0.0 to 10.1.0. - [Release notes](https://github.com/appium/java-client/releases) - [Changelog](https://github.com/appium/java-client/blob/master/CHANGELOG.md) - [Commits](https://github.com/appium/java-client/compare/v10.0.0...v10.1.0) --- updated-dependencies: - dependency-name: io.appium:java-client dependency-version: 10.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 66af9ddf7..07914cfd5 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // https://mvnrepository.com/artifact/io.github.bonigarcia/webdrivermanager implementation group: 'io.github.bonigarcia', name: 'webdrivermanager', version: '6.3.3' // https://mvnrepository.com/artifact/io.appium/java-client - implementation group: 'io.appium', name: 'java-client', version: '10.0.0' + implementation group: 'io.appium', name: 'java-client', version: '10.1.0' // https://mvnrepository.com/artifact/com.orientechnologies/orientdb-graphdb implementation group: 'com.orientechnologies', name: 'orientdb-graphdb', version: '3.2.38' // https://mvnrepository.com/artifact/org.graalvm.sdk/graal-sdk From 124a274ee1c917311018bea21ccd8c7ca954da1f Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 11:44:28 +0100 Subject: [PATCH 03/12] Add continue, completed, invalid, LLM Oracle logic --- .../src/org/testar/monkey/alayer/Verdict.java | 2 +- .../fewshot_gemini_oracle_goal_status.json | 58 +++++++++++ .../fewshot_openai_oracle_goal_status.json | 30 ++++++ .../Protocol_03_webdriver_llm_parabank.java | 12 +++ ... parabank_login_request_invalid_loan.goal} | 2 +- .../parabank_login_request_valid_loan.goal | 5 + .../src/org/testar/oracles/llm/LlmOracle.java | 20 +++- .../org/testar/oracles/llm/LlmVerdict.java | 30 +++++- .../oracles/llm/LlmVerdictDecision.java | 58 +++++++++++ .../testar/oracles/llm/LlmVerdictParser.java | 78 +++++++++++++++ .../oracles/llm/TestLlmVerdictParser.java | 99 +++++++++++++++++++ 11 files changed, 385 insertions(+), 9 deletions(-) create mode 100644 testar/resources/prompts/fewshot_gemini_oracle_goal_status.json create mode 100644 testar/resources/prompts/fewshot_openai_oracle_goal_status.json rename testar/resources/settings/03_webdriver_llm_parabank/{parabank_login_request_loan.goal => parabank_login_request_invalid_loan.goal} (81%) create mode 100644 testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal create mode 100644 testar/src/org/testar/oracles/llm/LlmVerdictDecision.java create mode 100644 testar/src/org/testar/oracles/llm/LlmVerdictParser.java create mode 100644 testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java diff --git a/core/src/org/testar/monkey/alayer/Verdict.java b/core/src/org/testar/monkey/alayer/Verdict.java index ced9c6b77..8405d1fbe 100644 --- a/core/src/org/testar/monkey/alayer/Verdict.java +++ b/core/src/org/testar/monkey/alayer/Verdict.java @@ -31,7 +31,6 @@ package org.testar.monkey.alayer; import java.io.Serializable; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -97,6 +96,7 @@ public enum Severity { /** FAIL (0.5 - 0.899) **/ UNREPLAYABLE(0.5, "UNREPLAYABLE"), // Sequence not replayable + LLM_INVALID(0.79, "LLM_INVALID"), // LLM detected objective steps were followed but resulting state is invalid SUSPICIOUS_TAG(0.8, "SUSPICIOUS_TAG"), // Suspicious tag SUSPICIOUS_PROCESS(0.87, "SUSPICIOUS_PROCESS"), // Suspicious message in the process standard output/error SUSPICIOUS_LOG(0.89, "SUSPICIOUS_LOG"), // Suspicious message in log file or command output (LogOracle) diff --git a/testar/resources/prompts/fewshot_gemini_oracle_goal_status.json b/testar/resources/prompts/fewshot_gemini_oracle_goal_status.json new file mode 100644 index 000000000..1914785ec --- /dev/null +++ b/testar/resources/prompts/fewshot_gemini_oracle_goal_status.json @@ -0,0 +1,58 @@ +[ + { + "role": "user", + "parts": [ + { + "text": "You are a web test agent. You will receive a test objective and a list of widgets that describe the current application state. Decide one verdict only: CONTINUE (goal not met yet), COMPLETED (goal met correctly), or INVALID (the expected end result is contradicted by the current state, indicating a functional issue). Use INVALID only when there is clear evidence that the expected result should be present but the observed state indicates an incorrect behavior. If evidence is not sufficient, choose CONTINUE. Respond in JSON only with this schema: {\"status\":\"CONTINUE|COMPLETED|INVALID\",\"info\":\"short reason\"}." + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Customer Login, Widget: Username, Widget: Password, Widget: Forgot login info?. Is the test objective met in this state?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "{\"status\":\"CONTINUE\",\"info\":\"login form is still present and no successful session is visible\"}" + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Welcome John Smith, Widget: Account Services, Widget: Log Out. Is the test objective met in this state?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "{\"status\":\"COMPLETED\",\"info\":\"user is authenticated and the expected welcome message is visible\"}" + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "We are testing the \"Demo\" web application. The objective of the test is to log in with valid credentials username \"John\" and password \"Smith\" and verify successful authentication. The current state of the application contains the widgets: Widget: Login failure, Widget: Invalid Credentials. Is the test objective met in this state?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "{\"status\":\"INVALID\",\"info\":\"The objective expects successful authentication with valid credentials, but the state shows a contradictory login failure\"}" + } + ] + } +] diff --git a/testar/resources/prompts/fewshot_openai_oracle_goal_status.json b/testar/resources/prompts/fewshot_openai_oracle_goal_status.json new file mode 100644 index 000000000..614fcef01 --- /dev/null +++ b/testar/resources/prompts/fewshot_openai_oracle_goal_status.json @@ -0,0 +1,30 @@ +[ + { + "role": "system", + "content": "You are a web test agent. You will receive a test objective and a list of widgets that describe the current application state. Decide one verdict only: CONTINUE (goal not met yet), COMPLETED (goal met correctly), or INVALID (the expected end result is contradicted by the current state, indicating a functional issue). Use INVALID only when there is clear evidence that the expected result should be present but the observed state indicates an incorrect behavior. If evidence is not sufficient, choose CONTINUE. Respond in JSON only with this schema: {\"status\":\"CONTINUE|COMPLETED|INVALID\",\"info\":\"short reason\"}." + }, + { + "role": "user", + "content": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Customer Login, Widget: Username, Widget: Password, Widget: Forgot login info?. Is the test objective met in this state?" + }, + { + "role": "assistant", + "content": "{\"status\":\"CONTINUE\",\"info\":\"The login form is still present and no successful session is visible\"}" + }, + { + "role": "user", + "content": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Welcome John Smith, Widget: Account Services, Widget: Log Out. Is the test objective met in this state?" + }, + { + "role": "assistant", + "content": "{\"status\":\"COMPLETED\",\"info\":\"The John user is authenticated and the expected welcome message is visible\"}" + }, + { + "role": "user", + "content": "We are testing the \"Demo\" web application. The objective of the test is to log in with valid credentials username \"John\" and password \"Smith\" and verify successful authentication. The current state of the application contains the widgets: Widget: Login failure, Widget: Invalid Credentials. Is the test objective met in this state?" + }, + { + "role": "assistant", + "content": "{\"status\":\"INVALID\",\"info\":\"The objective expects successful authentication with valid credentials, but the state shows a contradictory login failure\"}" + } +] diff --git a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java index c34fe87c8..083207944 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java +++ b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java @@ -166,6 +166,12 @@ protected List getVerdicts(State state) { List llmVerdicts = llmOracle.getVerdicts(state); for(Verdict llmVerdict : llmVerdicts) { + if(llmVerdict.severity() == Verdict.Severity.LLM_INVALID.getValue()) { + // LLM detected an invalid final state for the current goal, terminate the test sequence. + System.out.println("LLM detected invalid behavior, stopping test sequence."); + return Collections.singletonList(llmVerdict); + } + if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { // Test goal was completed, retrieve next test goal from queue. currentTestGoal = testGoalQueue.poll(); @@ -337,6 +343,12 @@ protected boolean executeAction(SUT system, State state, Action action) { */ @Override protected boolean moreActions(State state) { + List stateVerdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); + for (Verdict verdict : stateVerdicts) { + if (verdict != null && verdict.severity() == Verdict.Severity.LLM_INVALID.getValue()) { + return false; + } + } return super.moreActions(state); } diff --git a/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_loan.goal b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_invalid_loan.goal similarity index 81% rename from testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_loan.goal rename to testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_invalid_loan.goal index 310dd67fd..8015a4d63 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_loan.goal +++ b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_invalid_loan.goal @@ -2,4 +2,4 @@ As a user, I want to log in with the credentials john/demo Then the Welcome John Smith message is shown ; As a user, I want to request a loan with an amount '999999', a down payment of '100', for the account '54321' -Then the loan is denied due to insufficient funds \ No newline at end of file +Then the loan is approved by the bank entity \ No newline at end of file diff --git a/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal new file mode 100644 index 000000000..a04edc828 --- /dev/null +++ b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal @@ -0,0 +1,5 @@ +As a user, I want to log in with the credentials john/demo +Then the Welcome John Smith message is shown +; +As a user, I want to request a loan with an amount '500', a down payment of '100', for the account '54321' +Then the loan is approved by the bank entity \ No newline at end of file diff --git a/testar/src/org/testar/oracles/llm/LlmOracle.java b/testar/src/org/testar/oracles/llm/LlmOracle.java index d3a01b395..b9b17b50e 100644 --- a/testar/src/org/testar/oracles/llm/LlmOracle.java +++ b/testar/src/org/testar/oracles/llm/LlmOracle.java @@ -66,8 +66,6 @@ import org.testar.plugin.OperatingSystems; import org.testar.settings.Settings; -import com.google.gson.Gson; - public class LlmOracle implements Oracle { protected static final Logger logger = LogManager.getLogger(); @@ -173,9 +171,21 @@ private Verdict getVerdictWithLlm(State state) { String llmResponse = getResponseFromLlm(conversationJson); - LlmVerdict llmVerdict = new Gson().fromJson(llmResponse, LlmVerdict.class); - - if(llmVerdict.match()) return new Verdict(Verdict.Severity.LLM_COMPLETE, llmVerdict.getInfo()); + LlmVerdict llmVerdict = LlmVerdictParser.parse(llmResponse); + String info = llmVerdict.getInfo() == null ? "" : llmVerdict.getInfo(); + + switch (llmVerdict.getDecision()) { + case COMPLETED: + return new Verdict(Verdict.Severity.LLM_COMPLETE, info); + case INVALID: + return new Verdict(Verdict.Severity.LLM_INVALID, info); + case CONTINUE: + return Verdict.OK; + case UNKNOWN: + default: + logger.log(Level.WARN, "LLM oracle response did not include a recognized verdict status/match."); + return Verdict.OK; + } } catch(Exception e) { logger.log(Level.ERROR, "Error obtaining the verdict with the LLM"); diff --git a/testar/src/org/testar/oracles/llm/LlmVerdict.java b/testar/src/org/testar/oracles/llm/LlmVerdict.java index 8010e7cff..694b36157 100644 --- a/testar/src/org/testar/oracles/llm/LlmVerdict.java +++ b/testar/src/org/testar/oracles/llm/LlmVerdict.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,10 +32,16 @@ public class LlmVerdict { private Boolean match; + private String status; private String info; public LlmVerdict(Boolean match, String info) { + this(match, "", info); + } + + public LlmVerdict(Boolean match, String status, String info) { this.match = match; + this.status = status; this.info = info; } @@ -43,7 +49,27 @@ public Boolean match() { return match; } + public String getStatus() { + return status; + } + public String getInfo() { return info; } + + public LlmVerdictDecision getDecision() { + LlmVerdictDecision fromStatus = LlmVerdictDecision.fromStatus(status); + // Return a (CONTINUE | COMPLETED | INVALID) detected LLM Oracle decision + if (fromStatus != LlmVerdictDecision.UNKNOWN) { + return fromStatus; + } + // Return a true/false detected LLM Oracle decision + if (Boolean.TRUE.equals(match)) { + return LlmVerdictDecision.COMPLETED; + } + if (Boolean.FALSE.equals(match)) { + return LlmVerdictDecision.CONTINUE; + } + return LlmVerdictDecision.UNKNOWN; + } } diff --git a/testar/src/org/testar/oracles/llm/LlmVerdictDecision.java b/testar/src/org/testar/oracles/llm/LlmVerdictDecision.java new file mode 100644 index 000000000..0a1890b5c --- /dev/null +++ b/testar/src/org/testar/oracles/llm/LlmVerdictDecision.java @@ -0,0 +1,58 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.e + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.oracles.llm; + +import java.util.Locale; + +public enum LlmVerdictDecision { + CONTINUE, + COMPLETED, + INVALID, + UNKNOWN; + + public static LlmVerdictDecision fromStatus(String status) { + if (status == null || status.trim().isEmpty()) { + return UNKNOWN; + } + + String normalized = status.trim().toUpperCase(Locale.ROOT); + switch (normalized) { + case "CONTINUE": + return CONTINUE; + case "COMPLETED": + return COMPLETED; + case "INVALID": + return INVALID; + default: + return UNKNOWN; + } + } +} diff --git a/testar/src/org/testar/oracles/llm/LlmVerdictParser.java b/testar/src/org/testar/oracles/llm/LlmVerdictParser.java new file mode 100644 index 000000000..72eae95d8 --- /dev/null +++ b/testar/src/org/testar/oracles/llm/LlmVerdictParser.java @@ -0,0 +1,78 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.e + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.oracles.llm; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public final class LlmVerdictParser { + + private LlmVerdictParser() {} + + public static LlmVerdict parse(String llmResponse) { + JsonObject object = JsonParser.parseString(llmResponse).getAsJsonObject(); + + String info = getStringValue(object, "info"); + String status = getStringValue(object, "status"); + Boolean match = getBooleanValue(object, "match"); + + return new LlmVerdict(match, status, info); + } + + private static String getStringValue(JsonObject object, String property) { + if (!object.has(property) || object.get(property).isJsonNull()) { + return ""; + } + JsonElement value = object.get(property); + if (value.isJsonPrimitive()) { + return value.getAsString(); + } + return value.toString(); + } + + private static Boolean getBooleanValue(JsonObject object, String property) { + if (!object.has(property) || object.get(property).isJsonNull()) { + return null; + } + + JsonElement value = object.get(property); + try { + return value.getAsBoolean(); + } catch (Exception ignored) { + try { + return Boolean.parseBoolean(value.getAsString()); + } catch (Exception ignoredAgain) { + return null; + } + } + } +} diff --git a/testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java b/testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java new file mode 100644 index 000000000..731adcdad --- /dev/null +++ b/testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java @@ -0,0 +1,99 @@ +package org.testar.oracles.llm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +public class TestLlmVerdictParser { + + @Test + public void testParseStatusCompleted() { + String response = "{\"status\":\"COMPLETED\",\"info\":\"objective reached\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.COMPLETED, verdict.getDecision()); + assertEquals("objective reached", verdict.getInfo()); + assertNull(verdict.match()); + } + + @Test + public void testParseStatusInvalid() { + String response = "{\"status\":\"INVALID\",\"info\":\"unexpected final state\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.INVALID, verdict.getDecision()); + assertEquals("unexpected final state", verdict.getInfo()); + } + + @Test + public void testParseMatchTrue() { + String response = "{\"match\":true,\"info\":\"done\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.COMPLETED, verdict.getDecision()); + assertEquals("done", verdict.getInfo()); + } + + @Test + public void testParseMatchFalse() { + String response = "{\"match\":\"false\",\"info\":\"continue\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.CONTINUE, verdict.getDecision()); + assertEquals("continue", verdict.getInfo()); + } + + @Test + public void testParseUnknownFormatResponse() { + String response = "{\"verdict\":\"completed\",\"info\":\"alias field\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("alias field", verdict.getInfo()); + } + + @Test + public void testParseUnknownStatusDecision() { + String response = "{\"status\":\"MAYBE\",\"info\":\"not recognized\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("not recognized", verdict.getInfo()); + } + + @Test + public void testParseMissingJsonFields() { + String response = "{}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("", verdict.getInfo()); + assertNull(verdict.match()); + } + + @Test + public void testParseIncorrectJsonFormat() { + String response = "{\"status\":{},\"info\":null,\"match\":{}}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("", verdict.getInfo()); + assertNull(verdict.match()); + } + + @Test(expected = NullPointerException.class) + public void testParseNullInputThrowsException() { + LlmVerdictParser.parse(null); + } + + @Test(expected = IllegalStateException.class) + public void testParseInvalidJsonThrowsException() { + LlmVerdictParser.parse("not-a-json-object"); + } + + @Test(expected = IllegalStateException.class) + public void testParseArrayJsonThrowsException() { + LlmVerdictParser.parse("[]"); + } +} From f97f67639822d7f65d042f9cbb2648e18520ebfa Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 11:55:26 +0100 Subject: [PATCH 04/12] Remove slide actions from llm parabank protocols --- .../Protocol_03_webdriver_llm_parabank.java | 5 ----- .../Protocol_webdriver_llm_state_model_evaluator.java | 5 ----- ...tocol_webdriver_llm_state_model_transition_evaluator.java | 5 ----- .../Protocol_webdriver_llm_state_widgets_evaluator.java | 5 ----- 4 files changed, 20 deletions(-) diff --git a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java index 083207944..2882964d4 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java +++ b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java @@ -51,8 +51,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_03_webdriver_llm_parabank extends WebdriverProtocol { @@ -235,9 +233,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java index 5f6373116..72428b26c 100644 --- a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java @@ -67,8 +67,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_webdriver_llm_state_model_evaluator extends WebdriverProtocol { // The LLM Action selector needs to be initialize with the settings @@ -257,9 +255,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java index ec24efdcb..c8885bc5a 100644 --- a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java @@ -67,8 +67,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_webdriver_llm_state_model_transition_evaluator extends WebdriverProtocol { // The LLM Action selector needs to be initialize with the settings @@ -262,9 +260,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java index bdf51b222..fb076daa4 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java @@ -67,8 +67,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_webdriver_llm_state_widgets_evaluator extends WebdriverProtocol { @@ -239,9 +237,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; From 5a98bac57d9d3d33e1fe635ca55864c90600fc37 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 15:20:46 +0100 Subject: [PATCH 05/12] Add WebIsDisplayed to StateCondition --- .../analysis/condition/StateCondition.java | 10 ++++++++-- .../condition/TestCheckConditionEvaluator.java | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java b/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java index 329413e5d..80f8c17fb 100644 --- a/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java +++ b/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java @@ -30,6 +30,8 @@ package org.testar.statemodel.analysis.condition; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.openqa.selenium.InvalidArgumentException; import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Tag; @@ -42,12 +44,14 @@ * Simple condition that searches for a string in all states. */ public class StateCondition extends TestCondition { + static final Logger logger = LogManager.getLogger(); + private final String field; private final String searchMessage; /** * Creates a new StateCondition. - * @param field The field in the state model to serach. Example: WebInnerHTML. + * @param field The field in the state model to search. Example: WebInnerHTML. * @param searchMessage The string to search for. * @param comparator The result of the query is compared to the threshold value using the selected operator. * @param threshold The result of the query is compared to this value. @@ -115,12 +119,14 @@ public boolean evaluate(String modelIdentifier, StateModelManager stateModelMana public boolean evaluate(State state) { for(Widget widget : state) { // For web apps check the widget is visible - if(widget.get(WdTags.WebIsFullOnScreen, true)) { + // The default value is true to avoid blocking SUT systems other than web apps + if(widget.get(WdTags.WebIsFullOnScreen, true) && widget.get(WdTags.WebIsDisplayed, true)) { for(Tag tag : widget.tags()){ if(tag.name().equals(getField()) && widget.get(tag, null) != null){ try { String tagValue = widget.get(tag).toString(); if(tagValue.contains(searchMessage)) { + logger.info("State Condition Match for Tag Value: " + tagValue); return true; } } catch (Exception e) { diff --git a/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java b/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java index 63bef253c..81cddc538 100644 --- a/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java +++ b/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java @@ -76,6 +76,21 @@ public void test_single_check_condition_unmatch_nonvisible() { Assert.isTrue(!checkEvaluator.evaluateConditions(state), "The state evaluator must unmatch for non visible widgets"); } + @Test + public void test_single_check_condition_unmatch_nondisplayed() { + StateStub state = new StateStub(); + WidgetStub widget = new WidgetStub(); + widget.set(WdTags.WebTitle, "this message is shown but the widget is non displayed"); + widget.set(WdTags.WebIsDisplayed, false); + state.addChild(widget); + widget.setParent(state); + + String testGoal = "Perform this action \n Check: this message is shown"; + CheckConditionEvaluator checkEvaluator = new CheckConditionEvaluator(WdTags.WebTitle, testGoal); + + Assert.isTrue(!checkEvaluator.evaluateConditions(state), "The state evaluator must unmatch for non displayed widgets"); + } + @Test public void test_multiple_check_conditions_loaded_messages() { String testGoal = "Perform this action \n Check: this message is shown \n Check: also a second message"; From 3955c1648d47fc1a18a041312ad72a0f290ea31e Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 15:27:09 +0100 Subject: [PATCH 06/12] Update LLM_INVALID severity --- .../src/org/testar/monkey/alayer/Verdict.java | 2 +- .../testar/monkey/VerdictProcessingTest.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/src/org/testar/monkey/alayer/Verdict.java b/core/src/org/testar/monkey/alayer/Verdict.java index 8405d1fbe..99271dfc4 100644 --- a/core/src/org/testar/monkey/alayer/Verdict.java +++ b/core/src/org/testar/monkey/alayer/Verdict.java @@ -96,7 +96,6 @@ public enum Severity { /** FAIL (0.5 - 0.899) **/ UNREPLAYABLE(0.5, "UNREPLAYABLE"), // Sequence not replayable - LLM_INVALID(0.79, "LLM_INVALID"), // LLM detected objective steps were followed but resulting state is invalid SUSPICIOUS_TAG(0.8, "SUSPICIOUS_TAG"), // Suspicious tag SUSPICIOUS_PROCESS(0.87, "SUSPICIOUS_PROCESS"), // Suspicious message in the process standard output/error SUSPICIOUS_LOG(0.89, "SUSPICIOUS_LOG"), // Suspicious message in log file or command output (LogOracle) @@ -104,6 +103,7 @@ public enum Severity { /** CRITICAL (0.9 - 1.0) **/ CRITICAL(0.9, "CRITICAL"), + LLM_INVALID(0.91, "LLM_INVALID"), // LLM detected objective steps were followed but resulting state is invalid NOT_RESPONDING(0.99999990, "NOT_RESPONDING"), // Unresponsive UNEXPECTEDCLOSE(0.99999999, "UNEXPECTEDCLOSE"), // Crash? Unexpected close? FAIL(1.0, "FAIL"); diff --git a/testar/test/org/testar/monkey/VerdictProcessingTest.java b/testar/test/org/testar/monkey/VerdictProcessingTest.java index 9e976a1ba..85a8e51fe 100644 --- a/testar/test/org/testar/monkey/VerdictProcessingTest.java +++ b/testar/test/org/testar/monkey/VerdictProcessingTest.java @@ -101,6 +101,25 @@ public void testDoesNotIgnoreDuplicateLlmComplete() throws Exception { assertEquals(Verdict.Severity.LLM_COMPLETE.getValue(), filtered.get(0).severity(), 0.0); } + @Test + public void testDoesNotIgnoreDuplicateLlmInvalid() throws Exception { + File settingsDir = tempFolder.newFolder("settings_llm_invalid"); + Settings.setSettingsPath(settingsDir.getAbsolutePath()); + + File ignoreFile = new File(settingsDir, "list_of_verdicts_with_failures.txt"); + Files.write(ignoreFile.toPath(), Collections.singletonList("goal invalid by llm"), StandardCharsets.UTF_8); + + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, true); + VerdictProcessing processing = new VerdictProcessing(settings); + + Verdict llmInvalid = new Verdict(Verdict.Severity.LLM_INVALID, "goal invalid by llm"); + List filtered = processing.filterDuplicates(Collections.singletonList(llmInvalid)); + + assertEquals(1, filtered.size()); + assertEquals(Verdict.Severity.LLM_INVALID.getValue(), filtered.get(0).severity(), 0.0); + } + @Test public void testDoesNotIgnoreDuplicateConditionComplete() throws Exception { File settingsDir = tempFolder.newFolder("settings_condition_complete"); From b074462b3ce79596a088ac0affa3c4f78055032a Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 15:28:28 +0100 Subject: [PATCH 07/12] Add LlmTestGoalOrchestrator logic --- .../Protocol_03_webdriver_llm_parabank.java | 59 +++------ ...l_webdriver_llm_state_model_evaluator.java | 47 +++----- ..._llm_state_model_transition_evaluator.java | 47 +++----- ...webdriver_llm_state_widgets_evaluator.java | 47 +++----- ...arabank_login_request_loan_conditions.goal | 2 +- .../test.settings | 4 +- testar/src/org/testar/llm/LlmTestGoal.java | 2 +- .../testar/llm/LlmTestGoalOrchestrator.java | 112 ++++++++++++++++++ .../llm/TestLlmTestGoalOrchestrator.java | 101 ++++++++++++++++ 9 files changed, 282 insertions(+), 139 deletions(-) create mode 100644 testar/src/org/testar/llm/LlmTestGoalOrchestrator.java create mode 100644 testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java diff --git a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java index 2882964d4..01f62ad52 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java +++ b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.OracleWebPromptGenerator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; @@ -56,13 +57,13 @@ public class Protocol_03_webdriver_llm_parabank extends WebdriverProtocol { // The LLM Action selector needs to be initialize with the settings private LlmActionSelector llmActionSelector; - private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; // The LLM Oracle needs to be initialize with the settings private LlmOracle llmOracle; + private List testGoals = new ArrayList<>(); + private LlmTestGoalOrchestrator testGoalOrchestrator; + /** * Called once during the life time of TESTAR * This method can be used to perform initial setup work @@ -81,6 +82,11 @@ protected void initialize(Settings settings) { // Initialize the LlmOracle using the LLM settings llmOracle = new LlmOracle(settings, new OracleWebPromptGenerator()); + + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + llmOracle.reset(goal, appendPreviousGoal); + }); } /** @@ -89,16 +95,7 @@ protected void initialize(Settings settings) { @Override protected void preSequencePreparations() { super.preSequencePreparations(); - - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); - // Reset llm oracle - llmOracle.reset(currentTestGoal, false); + testGoalOrchestrator.startSequence(); } private void setupTestGoals(List testGoalsList) { @@ -160,34 +157,12 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - // Use the LLM as an Oracle to determine if the test goal has been completed - List llmVerdicts = llmOracle.getVerdicts(state); - - for(Verdict llmVerdict : llmVerdicts) { - if(llmVerdict.severity() == Verdict.Severity.LLM_INVALID.getValue()) { - // LLM detected an invalid final state for the current goal, terminate the test sequence. - System.out.println("LLM detected invalid behavior, stopping test sequence."); - return Collections.singletonList(llmVerdict); - } + List verdicts = super.getVerdicts(state); - if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); + // Add the LLM Oracle verdicts to determine if the test goal has been completed + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(llmOracle.getVerdicts(state))); - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return Collections.singletonList(llmVerdict); - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - llmOracle.reset(currentTestGoal, true); - } - } - } - - return super.getVerdicts(state); + return verdicts; } /** @@ -338,12 +313,6 @@ protected boolean executeAction(SUT system, State state, Action action) { */ @Override protected boolean moreActions(State state) { - List stateVerdicts = state.get(Tags.OracleVerdicts, Collections.singletonList(Verdict.OK)); - for (Verdict verdict : stateVerdicts) { - if (verdict != null && verdict.severity() == Verdict.Severity.LLM_INVALID.getValue()) { - return false; - } - } return super.moreActions(state); } diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java index 72428b26c..ed89dfff0 100644 --- a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; @@ -75,8 +76,7 @@ public class Protocol_webdriver_llm_state_model_evaluator extends WebdriverProto private ConditionEvaluator conditionEvaluator; private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; + private LlmTestGoalOrchestrator testGoalOrchestrator; /** * Called once during the life time of TESTAR @@ -101,6 +101,11 @@ protected void initialize(Settings settings) { metricsManager = new MetricsManager(new LlmMetricsCollector(testGoals)); conditionEvaluator = new BasicConditionEvaluator(); + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + conditionEvaluator.clear(); + conditionEvaluator.addConditions(goal.getCompletionConditions()); + }); } private void setupTestGoals(List testGoalsList) { @@ -117,16 +122,6 @@ private void setupTestGoals(List testGoalsList) { protected void preSequencePreparations() { super.preSequencePreparations(); - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); - // Use timestamp to create a unique new state model String appName = settings.get(ConfigTags.ApplicationName, ""); String appVersion = settings.get(ConfigTags.ApplicationVersion, ""); @@ -136,6 +131,8 @@ protected void preSequencePreparations() { stateModelManager.notifyTestingEnded(); stateModelManager = StateModelManagerFactory.getStateModelManager( appName, (appVersion + "_" + timestamp), settings); + + testGoalOrchestrator.startSequence(); } /** @@ -190,26 +187,16 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - String modelIdentifier = stateModelManager.getModelIdentifier(); + List verdicts = super.getVerdicts(state); - if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - } - } + // Add the State Model Condition verdicts to determine if the test goal has been completed + String modelIdentifier = stateModelManager.getModelIdentifier(); + List stateModelConditionVerdicts = conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager) + ? Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "Current test goal conditions completed.")) + : Collections.singletonList(Verdict.OK); + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(stateModelConditionVerdicts)); - return super.getVerdicts(state); + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java index c8885bc5a..47bea1dd0 100644 --- a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; @@ -75,8 +76,7 @@ public class Protocol_webdriver_llm_state_model_transition_evaluator extends Web private ConditionEvaluator conditionEvaluator; private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; + private LlmTestGoalOrchestrator testGoalOrchestrator; /** * Called once during the life time of TESTAR @@ -101,6 +101,11 @@ protected void initialize(Settings settings) { metricsManager = new MetricsManager(new LlmMetricsCollector(testGoals)); conditionEvaluator = new BasicConditionEvaluator(); + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + conditionEvaluator.clear(); + conditionEvaluator.addConditions(goal.getCompletionConditions()); + }); } private void setupTestGoals(List testGoalsList) { @@ -122,16 +127,6 @@ private void setupTestGoals(List testGoalsList) { protected void preSequencePreparations() { super.preSequencePreparations(); - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); - // Use timestamp to create a unique new state model String appName = settings.get(ConfigTags.ApplicationName, ""); String appVersion = settings.get(ConfigTags.ApplicationVersion, ""); @@ -141,6 +136,8 @@ protected void preSequencePreparations() { stateModelManager.notifyTestingEnded(); stateModelManager = StateModelManagerFactory.getStateModelManager( appName, (appVersion + "_" + timestamp), settings); + + testGoalOrchestrator.startSequence(); } /** @@ -195,26 +192,16 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - String modelIdentifier = stateModelManager.getModelIdentifier(); + List verdicts = super.getVerdicts(state); - if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - } - } + // Add the State Model Condition verdicts to determine if the test goal has been completed + String modelIdentifier = stateModelManager.getModelIdentifier(); + List stateModelConditionVerdicts = conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager) + ? Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "Current test goal conditions completed.")) + : Collections.singletonList(Verdict.OK); + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(stateModelConditionVerdicts)); - return super.getVerdicts(state); + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java index fb076daa4..7cd031b52 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; @@ -75,8 +76,7 @@ public class Protocol_webdriver_llm_state_widgets_evaluator extends WebdriverPro private ConditionEvaluator conditionEvaluator; private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; + private LlmTestGoalOrchestrator testGoalOrchestrator; /** * Called once during the life time of TESTAR @@ -95,11 +95,16 @@ protected void initialize(Settings settings) { llmActionSelector = new LlmActionSelector(settings, new ActionWebPromptGenerator()); conditionEvaluator = new BasicConditionEvaluator(); + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + conditionEvaluator.clear(); + conditionEvaluator.addConditions(goal.getCompletionConditions()); + }); } private void setupTestGoals(List testGoalsList) { for(String testGoal : testGoalsList) { - CheckConditionEvaluator checkEvaluator = new CheckConditionEvaluator(WdTags.WebInnerHTML, testGoal); + CheckConditionEvaluator checkEvaluator = new CheckConditionEvaluator(WdTags.WebTextContent, testGoal); testGoals.add(new LlmTestGoal(testGoal, checkEvaluator.getConditions())); } } @@ -110,16 +115,7 @@ private void setupTestGoals(List testGoalsList) { @Override protected void preSequencePreparations() { super.preSequencePreparations(); - - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); + testGoalOrchestrator.startSequence(); } /** @@ -174,24 +170,15 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - // Apply state conditions to check if the test goal has been completed - if(conditionEvaluator.evaluateConditions(state)) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); - } else { - llmActionSelector.reset(currentTestGoal, true); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - System.out.println("Test goal completed, moving to next test goal: " + currentTestGoal.getTestGoal()); - } - } + List verdicts = super.getVerdicts(state); + + // Add the State Model Condition verdicts to determine if the test goal has been completed + List stateConditionVerdicts = conditionEvaluator.evaluateConditions(state) + ? Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "Current test goal conditions completed.")) + : Collections.singletonList(Verdict.OK); + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(stateConditionVerdicts)); - return super.getVerdicts(state); + return verdicts; } /** diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal index 2c7aeb3bb..6e6c8fb75 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal @@ -2,4 +2,4 @@ As a user, I want to log in with the credentials john/demo Check: John Smith ; As a user, I want to request a loan with an amount '999999' and a down payment of '100' -Check:

We cannot grant a loan in that amount with your available funds.

\ No newline at end of file +Check: We cannot grant a loan in that amount with your available funds. \ No newline at end of file diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings index 3c58ad2be..43172428c 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings @@ -126,7 +126,7 @@ MaxTime = 3.1536E7 ################################################################# ApplicationName = parabank -ApplicationVersion = login +ApplicationVersion = check_loan ################################################################# # State model inference settings @@ -185,7 +185,7 @@ LlmOracleFewshotFile = LlmTemperature = 0.2 LlmHistorySize = 5 LlmStateless = true -LlmTestGoals = Use the username 'john' and the password 'demo' to log in the application\\nCheck: Welcome John Smith +LlmTestGoals = Use the username 'john' and the password 'demo' to log in the application\\nCheck: Welcome John Smith;Apply for a loan of 999999 with a down payment of 100\\nCheck: We cannot grant a loan in that amount with your available funds ################################################################# # WebDriver features diff --git a/testar/src/org/testar/llm/LlmTestGoal.java b/testar/src/org/testar/llm/LlmTestGoal.java index 44f0099b6..006a0558b 100644 --- a/testar/src/org/testar/llm/LlmTestGoal.java +++ b/testar/src/org/testar/llm/LlmTestGoal.java @@ -36,7 +36,7 @@ /** * Contains a test goal for the llm to complete along with the conditions under which the test goal is considered - * completed. TODO: Rework GUI to support this new structure. + * completed. */ public class LlmTestGoal { private String testGoal; diff --git a/testar/src/org/testar/llm/LlmTestGoalOrchestrator.java b/testar/src/org/testar/llm/LlmTestGoalOrchestrator.java new file mode 100644 index 000000000..6d19a0d49 --- /dev/null +++ b/testar/src/org/testar/llm/LlmTestGoalOrchestrator.java @@ -0,0 +1,112 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.llm; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.testar.monkey.alayer.Verdict; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.function.BiConsumer; + +/** + * Coordinates multi-goal progression and stop conditions for LLM goal-oriented protocols. + */ +public class LlmTestGoalOrchestrator { + static final Logger logger = LogManager.getLogger(); + + private final List configuredGoals; + private final BiConsumer goalActivationHandler; + + private Queue goalQueue = new LinkedList<>(); + private LlmTestGoal currentGoal; + + public LlmTestGoalOrchestrator(List goals, BiConsumer goalActivationHandler) { + this.configuredGoals = goals == null ? Collections.emptyList() : new ArrayList<>(goals); + this.goalActivationHandler = goalActivationHandler; + } + + public void startSequence() { + goalQueue = new LinkedList<>(configuredGoals); + currentGoal = goalQueue.poll(); + activateCurrentGoal(false); + } + + public LlmTestGoal getCurrentGoal() { + return currentGoal; + } + + /** + * Processes goal evaluator verdicts (LLM or condition-based) and updates progression. + * + * @param goalVerdicts verdicts produced by a goal evaluator + * @return test goal verdicts if the sequence should stop, empty list otherwise + */ + public List processGoalVerdicts(List goalVerdicts) { + if (goalVerdicts == null || goalVerdicts.isEmpty()) { + return Collections.emptyList(); + } + + for (Verdict verdict : goalVerdicts) { + if (verdict == null) { + continue; + } + + if (verdict.severity() == Verdict.Severity.LLM_INVALID.getValue()) { + logger.info("LLM detected invalid behavior, stopping test sequence."); + return Collections.singletonList(verdict); + } + + if (verdict.isCompletion()) { + currentGoal = goalQueue.poll(); + if (currentGoal == null) { + logger.info("Test goal completed, but no more test goals."); + return Collections.singletonList(verdict); + } + logger.info("Test goal completed, moving to next test goal."); + activateCurrentGoal(true); + } + } + + return Collections.emptyList(); + } + + private void activateCurrentGoal(boolean appendPreviousGoal) { + if (currentGoal == null || goalActivationHandler == null) { + return; + } + goalActivationHandler.accept(currentGoal, appendPreviousGoal); + } +} diff --git a/testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java b/testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java new file mode 100644 index 000000000..3433e5ca9 --- /dev/null +++ b/testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java @@ -0,0 +1,101 @@ +package org.testar.llm; + +import org.junit.Test; +import org.testar.monkey.alayer.Verdict; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestLlmTestGoalOrchestrator { + + @Test + public void testGoalOrchestratorStartSequence() { + List activatedGoals = new ArrayList<>(); + AtomicBoolean appendPrevious = new AtomicBoolean(true); + + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Arrays.asList(goal("goal-1"), goal("goal-2")), + (goal, append) -> { + activatedGoals.add(goal.getTestGoal()); + appendPrevious.set(append); + } + ); + + orchestrator.startSequence(); + + assertEquals(1, activatedGoals.size()); + assertEquals("goal-1", activatedGoals.get(0)); + assertFalse(appendPrevious.get()); + } + + @Test + public void testGoalOrchestratorCompletionAdvancesToNextGoal() { + AtomicInteger activations = new AtomicInteger(0); + AtomicBoolean appendPrevious = new AtomicBoolean(false); + + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Arrays.asList(goal("goal-1"), goal("goal-2")), + (goal, append) -> { + activations.incrementAndGet(); + appendPrevious.set(append); + } + ); + orchestrator.startSequence(); + + List stopVerdicts = orchestrator.processGoalVerdicts( + Collections.singletonList(new Verdict(Verdict.Severity.LLM_COMPLETE, "done")) + ); + + assertTrue(stopVerdicts.isEmpty()); + assertEquals(2, activations.get()); + assertTrue(appendPrevious.get()); + assertEquals("goal-2", orchestrator.getCurrentGoal().getTestGoal()); + } + + @Test + public void testGoalOrchestratorCompletionFinalGoalStops() { + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Collections.singletonList(goal("goal-1")), + (goal, append) -> {} + ); + orchestrator.startSequence(); + + List stopVerdicts = orchestrator.processGoalVerdicts( + Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "all goals done")) + ); + + assertNotNull(stopVerdicts); + assertEquals(1, stopVerdicts.size()); + assertEquals(Verdict.Severity.CONDITION_COMPLETE.getValue(), stopVerdicts.get(0).severity(), 0.0); + } + + @Test + public void testGoalOrchestratorInvalidVerdict() { + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Collections.singletonList(goal("goal-1")), + (goal, append) -> {} + ); + orchestrator.startSequence(); + + List stopVerdicts = orchestrator.processGoalVerdicts( + Collections.singletonList(new Verdict(Verdict.Severity.LLM_INVALID, "invalid behavior")) + ); + + assertNotNull(stopVerdicts); + assertEquals(1, stopVerdicts.size()); + assertEquals(Verdict.Severity.LLM_INVALID.getValue(), stopVerdicts.get(0).severity(), 0.0); + } + + private static LlmTestGoal goal(String value) { + return new LlmTestGoal(value, Collections.emptyList()); + } +} From f71f5cfcc34a8a12a62ef50f11dc8bc0a923f79f Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 17:01:46 +0100 Subject: [PATCH 08/12] Update selectAction in model evaluator protocols --- ...l_webdriver_llm_state_model_evaluator.java | 23 +++++++++++-------- ..._llm_state_model_transition_evaluator.java | 23 +++++++++++-------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java index ed89dfff0..87d683940 100644 --- a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java @@ -310,19 +310,24 @@ protected boolean isTypeable(Widget widget) { */ @Override protected Action selectAction(State state, Set actions) { + // State model conditions are check one state behind, producing one extra selected action + // If the state model condition is done, avoid executing an extra step + if (conditionEvaluator.evaluateConditions(stateModelManager.getModelIdentifier(), stateModelManager)) { + return prepareActionForExecution(state, new NOP()); + } + Action toExecute = llmActionSelector.selectAction(state, actions); + return prepareActionForExecution(state, toExecute); + } - // We need to set a state to NOP actions - if(toExecute instanceof NOP) { - toExecute.set(Tags.OriginWidget, state); + private Action prepareActionForExecution(State state, Action action) { + if (action instanceof NOP) { + action.set(Tags.OriginWidget, state); } - - // We need the AbstractID for the state model - if(toExecute.get(Tags.AbstractID, null) == null) { - CodingManager.buildIDs(state, Collections.singleton(toExecute)); + if(action.get(Tags.AbstractID, null) == null) { + CodingManager.buildIDs(state, Collections.singleton(action)); } - - return toExecute; + return action; } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java index 47bea1dd0..d06b63a75 100644 --- a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java @@ -315,19 +315,24 @@ protected boolean isTypeable(Widget widget) { */ @Override protected Action selectAction(State state, Set actions) { + // State model conditions are check one state behind, producing one extra selected action + // If the state model condition is done, avoid executing an extra step + if (conditionEvaluator.evaluateConditions(stateModelManager.getModelIdentifier(), stateModelManager)) { + return prepareActionForExecution(state, new NOP()); + } + Action toExecute = llmActionSelector.selectAction(state, actions); + return prepareActionForExecution(state, toExecute); + } - // We need to set a state to NOP actions - if(toExecute instanceof NOP) { - toExecute.set(Tags.OriginWidget, state); + private Action prepareActionForExecution(State state, Action action) { + if (action instanceof NOP) { + action.set(Tags.OriginWidget, state); } - - // We need the AbstractID for the state model - if(toExecute.get(Tags.AbstractID, null) == null) { - CodingManager.buildIDs(state, Collections.singleton(toExecute)); + if(action.get(Tags.AbstractID, null) == null) { + CodingManager.buildIDs(state, Collections.singleton(action)); } - - return toExecute; + return action; } /** From eb1c0a9da7cf89e395822eca1af1c8c07b7939b3 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 17:06:19 +0100 Subject: [PATCH 09/12] Update TESTAR v2.8.5 --- CHANGELOG | 9 +++++++++ VERSION | 2 +- testar/src/org/testar/monkey/Main.java | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 9e688c3d5..d21b346f9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +#TESTAR v2.8.5 (17-Mar-2026) +- Bump io.appium:java-client from 10.0.0 to 10.1.0 +- Update JS getIsBlockedTestar logic +- Add continue, completed, invalid, LLM Oracle logic +- Add LlmTestGoalOrchestrator logic +- Add WebIsDisplayed to StateCondition +- Update selectAction in model evaluator protocols to avoid LLM select extra actions + + #TESTAR v2.8.4 (12-Mar-2026) - Remove abstract actionIds from LLM ActionHistory - Update getElementDescription prioritty and length diff --git a/VERSION b/VERSION index 0409c1638..7f04bb11e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.4 \ No newline at end of file +2.8.5 \ No newline at end of file diff --git a/testar/src/org/testar/monkey/Main.java b/testar/src/org/testar/monkey/Main.java index 072185d2a..438c7e9f5 100644 --- a/testar/src/org/testar/monkey/Main.java +++ b/testar/src/org/testar/monkey/Main.java @@ -64,7 +64,7 @@ public class Main { - public static final String TESTAR_VERSION = "v2.8.4 (12-Mar-2026)"; + public static final String TESTAR_VERSION = "v2.8.5 (17-Mar-2026)"; //public static final String TESTAR_DIR_PROPERTY = "DIRNAME"; //Use the OS environment to obtain TESTAR directory public static final String SETTINGS_FILE = "test.settings"; From 086db753adb203b320583c1642858590ad34bfaf Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 17:11:52 +0100 Subject: [PATCH 10/12] Update default parabank LlmOracleFewshotFile --- .../settings/03_webdriver_llm_parabank/test.settings | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testar/resources/settings/03_webdriver_llm_parabank/test.settings b/testar/resources/settings/03_webdriver_llm_parabank/test.settings index 2e3bba187..33bf23fe2 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/test.settings +++ b/testar/resources/settings/03_webdriver_llm_parabank/test.settings @@ -147,7 +147,7 @@ DataStoreUser = DataStorePassword = DataStoreMode = none ApplicationName = parabank -ApplicationVersion = loan_denied +ApplicationVersion = request_loan ActionSelectionAlgorithm = random StateModelStoreWidgets = false ResetDataStore = false @@ -173,11 +173,11 @@ LlmReasoning = minimal LlmHostUrl = https://api.openai.com/v1/chat/completions LlmAuthorizationHeader = Bearer %OPENAI_API% LlmActionFewshotFile = prompts/fewshot_openai_action.json -LlmOracleFewshotFile = prompts/fewshot_openai_oracle.json +LlmOracleFewshotFile = prompts/fewshot_openai_oracle_goal_status.json LlmTemperature = 0.2 LlmHistorySize = 5 LlmStateless = true -LlmTestGoals = As a user, I want to log in with the credentials john/demo\\nThen a Request Loan link is shown;As a user, I want to request a loan with an amount '999999', a down payment of '100', for the account '54321'\\nThen the loan is denied due to insufficient funds +LlmTestGoals = As a user, I want to log in with the credentials john/demo\\nThen a Request Loan link is shown;As a user, I want to request a loan with an amount '999999', a down payment of '100', for the account '54321'\\nThen the loan is approved by the bank entity ################################################################# # WebDriver features From e03b637164b28e43de03e8ce20e2ef97a5ed2d82 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 20:39:23 +0100 Subject: [PATCH 11/12] Add WebWidgetInnerText to state model abstraction --- core/src/org/testar/StateManagementTags.java | 6 +++++- .../org/testar/monkey/alayer/webdriver/enums/WdMapping.java | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/org/testar/StateManagementTags.java b/core/src/org/testar/StateManagementTags.java index 91782f0fe..1fa45d91b 100644 --- a/core/src/org/testar/StateManagementTags.java +++ b/core/src/org/testar/StateManagementTags.java @@ -308,7 +308,8 @@ public enum Group {General, ControlPattern, WebDriver} public static final Tag WebWidgetId = Tag.from("Web Widget id", String.class); public static final Tag WebWidgetName = Tag.from("Web Widget name", String.class); public static final Tag WebWidgetTagName = Tag.from("Web Widget tag name", String.class); - public static final Tag WebWidgetTextContent= Tag.from("Web Widget text content", String.class); + public static final Tag WebWidgetTextContent = Tag.from("Web Widget text content", String.class); + public static final Tag WebWidgetInnerText = Tag.from("Web Widget inner text", String.class); public static final Tag WebWidgetTitle = Tag.from("Web Widget title", String.class); public static final Tag WebWidgetHref = Tag.from("Web Widget href", String.class); public static final Tag WebWidgetValue = Tag.from("Web Widget value", String.class); @@ -402,6 +403,7 @@ public enum Group {General, ControlPattern, WebDriver} add(WebWidgetName); add(WebWidgetTagName); add(WebWidgetTextContent); + add(WebWidgetInnerText); add(WebWidgetTitle); add(WebWidgetHref); add(WebWidgetValue); @@ -501,6 +503,7 @@ public static boolean isStateManagementTag(Tag tag) { settingsMap.put(WebWidgetName, "WebWidgetName"); settingsMap.put(WebWidgetTagName, "WebWidgetTagName"); settingsMap.put(WebWidgetTextContent, "WebWidgetTextContent"); + settingsMap.put(WebWidgetInnerText, "WebWidgetInnerText"); settingsMap.put(WebWidgetTitle, "WebWidgetTitle"); settingsMap.put(WebWidgetHref, "WebWidgetHref"); settingsMap.put(WebWidgetValue, "WebWidgetValue"); @@ -590,6 +593,7 @@ public static boolean isStateManagementTag(Tag tag) { put(WebWidgetName, Group.WebDriver); put(WebWidgetTagName, Group.WebDriver); put(WebWidgetTextContent, Group.WebDriver); + put(WebWidgetInnerText, Group.WebDriver); put(WebWidgetTitle, Group.WebDriver); put(WebWidgetHref, Group.WebDriver); put(WebWidgetValue, Group.WebDriver); diff --git a/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java b/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java index f055a246c..fb51626a2 100644 --- a/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java +++ b/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java @@ -51,6 +51,7 @@ public class WdMapping { put(WebWidgetName ,WdTags.WebName); put(WebWidgetTagName, WdTags.WebTagName); put(WebWidgetTextContent, WdTags.WebTextContent); + put(WebWidgetInnerText, WdTags.WebInnerText); put(WebWidgetTitle, WdTags.WebTitle); put(WebWidgetHref, WdTags.WebHref); put(WebWidgetValue, WdTags.WebValue); From 5f576cfadb885136b5a57180cc6f858fa5062d7d Mon Sep 17 00:00:00 2001 From: ferpasri Date: Mon, 16 Mar 2026 21:44:25 +0100 Subject: [PATCH 12/12] Add webdriver mapping unit tests --- .../alayer/webdriver/TestWdTagMapping.java | 345 ++++++++++++++++++ .../TestWebdriverStateManagementTag.java | 180 +++++++-- 2 files changed, 499 insertions(+), 26 deletions(-) diff --git a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java index 2329d4eeb..2f1146741 100644 --- a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java +++ b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java @@ -1,11 +1,17 @@ package org.testar.monkey.alayer.webdriver; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import org.junit.Before; import org.junit.Test; +import org.openqa.selenium.remote.RemoteWebElement; +import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import java.util.Arrays; +import java.util.HashMap; + public class TestWdTagMapping { private WdRootElement rootWdElement; @@ -28,6 +34,289 @@ public void test_WebAriaLabel_mapping() { assertEquals("Search", childWdWidget.get(WdTags.WebAriaLabel, "")); } + @Test + public void test_WebId_mapping() { + childWdElement.id = "id-123"; + assertEquals("id-123", childWdWidget.get(WdTags.WebId, "")); + } + + @Test + public void test_WebHref_mapping() { + childWdElement.href = "https://testar.org"; + assertEquals("https://testar.org", childWdWidget.get(WdTags.WebHref, "")); + } + + @Test + public void test_WebPlaceholder_mapping() { + childWdElement.placeholder = "Type here"; + assertEquals("Type here", childWdWidget.get(WdTags.WebPlaceholder, "")); + } + + @Test + public void test_WebIsDisabled_mapping() { + childWdElement.disabled = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsDisabled, null)); + } + + @Test + public void test_WebName_mapping() { + childWdElement.name = "search-input"; + assertEquals("search-input", childWdWidget.get(WdTags.WebName, "")); + } + + @Test + public void test_WebTagName_mapping() { + childWdElement.tagName = "input"; + assertEquals("input", childWdWidget.get(WdTags.WebTagName, "")); + } + + @Test + public void test_WebTextContent_mapping() { + childWdElement.textContent = "Search text"; + assertEquals("Search text", childWdWidget.get(WdTags.WebTextContent, "")); + } + + @Test + public void test_WebInnerText_mapping() { + childWdElement.innerText = "Visible text"; + assertEquals("Visible text", childWdWidget.get(WdTags.WebInnerText, "")); + } + + @Test + public void test_WebTitle_mapping() { + childWdElement.title = "Search"; + assertEquals("Search", childWdWidget.get(WdTags.WebTitle, "")); + } + + @Test + public void test_WebValue_mapping() { + childWdElement.value = "hello"; + assertEquals("hello", childWdWidget.get(WdTags.WebValue, "")); + } + + @Test + public void test_WebType_mapping() { + childWdElement.type = "text"; + assertEquals("text", childWdWidget.get(WdTags.WebType, "")); + } + + @Test + public void test_WebSrc_mapping() { + childWdElement.src = "https://testar.org/image.png"; + assertEquals("https://testar.org/image.png", childWdWidget.get(WdTags.WebSrc, "")); + } + + @Test + public void test_WebXPath_mapping() { + childWdElement.xpath = "/html/body/div[1]"; + assertEquals("/html/body/div[1]", childWdWidget.get(WdTags.WebXPath, "")); + } + + @Test + public void test_WebCssClasses_mapping() { + childWdElement.cssClasses = Arrays.asList("btn", "primary"); + assertEquals("[btn, primary]", childWdWidget.get(WdTags.WebCssClasses, "")); + } + + @Test + public void test_WebIsEnabled_mapping() { + childWdElement.enabled = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsEnabled, null)); + } + + @Test + public void test_WebIsBlocked_mapping() { + childWdElement.blocked = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsBlocked, null)); + } + + @Test + public void test_WebIsClickable_mapping() { + childWdElement.isClickable = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsClickable, null)); + } + + @Test + public void test_WebIsShadow_mapping() { + childWdElement.isShadow = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsShadow, null)); + } + + @Test + public void test_WebHasKeyboardFocus_mapping() { + childWdElement.hasKeyboardFocus = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebHasKeyboardFocus, null)); + } + + @Test + public void test_WebIsKeyboardFocusable_mapping() { + childWdElement.isKeyboardFocusable = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsKeyboardFocusable, null)); + } + + @Test + public void test_WebIsContentElement_mapping() { + childWdElement.isContentElement = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsContentElement, null)); + } + + @Test + public void test_WebIsControlElement_mapping() { + childWdElement.isControlElement = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsControlElement, null)); + } + + @Test + public void test_WebStyle_mapping() { + childWdElement.style = "color:red;"; + assertEquals("color:red;", childWdWidget.get(WdTags.WebStyle, "")); + } + + @Test + public void test_WebStyleOpacity_mapping() { + childWdElement.styleOpacity = 0.7; + assertEquals(Double.valueOf(0.7), childWdWidget.get(WdTags.WebStyleOpacity, null)); + } + + @Test + public void test_WebStyleOverflow_mapping() { + childWdElement.styleOverflow = "auto"; + assertEquals("auto", childWdWidget.get(WdTags.WebStyleOverflow, "")); + } + + @Test + public void test_WebStyleOverflowX_mapping() { + childWdElement.styleOverflowX = "scroll"; + assertEquals("scroll", childWdWidget.get(WdTags.WebStyleOverflowX, "")); + } + + @Test + public void test_WebStyleOverflowY_mapping() { + childWdElement.styleOverflowY = "hidden"; + assertEquals("hidden", childWdWidget.get(WdTags.WebStyleOverflowY, "")); + } + + @Test + public void test_WebStylePosition_mapping() { + childWdElement.stylePosition = "relative"; + assertEquals("relative", childWdWidget.get(WdTags.WebStylePosition, "")); + } + + @Test + public void test_WebZIndex_mapping() { + childWdElement.zindex = 10.0; + assertEquals(Double.valueOf(10.0), childWdWidget.get(WdTags.WebZIndex, null)); + } + + @Test + public void test_WebBoundingRectangle_mapping() { + Rect rect = Rect.from(1, 2, 30, 40); + childWdElement.rect = rect; + assertEquals(rect, childWdWidget.get(WdTags.WebBoundingRectangle, null)); + } + + @Test + public void test_WebNaturalWidth_mapping() { + childWdElement.naturalWidth = 300; + assertEquals(Long.valueOf(300), childWdWidget.get(WdTags.WebNaturalWidth, null)); + } + + @Test + public void test_WebNaturalHeight_mapping() { + childWdElement.naturalHeight = 200; + assertEquals(Long.valueOf(200), childWdWidget.get(WdTags.WebNaturalHeight, null)); + } + + @Test + public void test_WebDisplayedWidth_mapping() { + childWdElement.displayedWidth = 120; + assertEquals(Long.valueOf(120), childWdWidget.get(WdTags.WebDisplayedWidth, null)); + } + + @Test + public void test_WebDisplayedHeight_mapping() { + childWdElement.displayedHeight = 80; + assertEquals(Long.valueOf(80), childWdWidget.get(WdTags.WebDisplayedHeight, null)); + } + + @Test + public void test_WebMaxLength_mapping() { + childWdElement.maxLength = 255; + assertEquals(Integer.valueOf(255), childWdWidget.get(WdTags.WebMaxLength, null)); + } + + @Test + public void test_WebInnerHTML_mapping() { + childWdElement.innerHTML = "hello"; + assertEquals("hello", childWdWidget.get(WdTags.WebInnerHTML, "")); + } + + @Test + public void test_WebOuterHTML_mapping() { + childWdElement.outerHTML = "
hello
"; + assertEquals("
hello
", childWdWidget.get(WdTags.WebOuterHTML, "")); + } + + @Test + public void test_WebComputedFontSize_mapping() { + childWdElement.computedFontSize = "16px"; + assertEquals("16px", childWdWidget.get(WdTags.WebComputedFontSize, "")); + } + + @Test + public void test_WebScrollPattern_mapping() { + childWdElement.scrollPattern = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebScrollPattern, null)); + } + + @Test + public void test_WebHorizontallyScrollable_mapping() { + childWdElement.hScroll = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebHorizontallyScrollable, null)); + } + + @Test + public void test_WebVerticallyScrollable_mapping() { + childWdElement.vScroll = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebVerticallyScrollable, null)); + } + + @Test + public void test_WebScrollHorizontalViewSize_mapping() { + childWdElement.hScrollViewSize = 33.3; + assertEquals(Double.valueOf(33.3), childWdWidget.get(WdTags.WebScrollHorizontalViewSize, null)); + } + + @Test + public void test_WebScrollVerticalViewSize_mapping() { + childWdElement.vScrollViewSize = 44.4; + assertEquals(Double.valueOf(44.4), childWdWidget.get(WdTags.WebScrollVerticalViewSize, null)); + } + + @Test + public void test_WebScrollHorizontalPercent_mapping() { + childWdElement.hScrollPercent = 55.5; + assertEquals(Double.valueOf(55.5), childWdWidget.get(WdTags.WebScrollHorizontalPercent, null)); + } + + @Test + public void test_WebScrollVerticalPercent_mapping() { + childWdElement.vScrollPercent = 66.6; + assertEquals(Double.valueOf(66.6), childWdWidget.get(WdTags.WebScrollVerticalPercent, null)); + } + + @Test + public void test_WebIsOffScreen_mapping() { + childWdElement.isFullVisibleOnScreen = false; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsOffScreen, null)); + } + + @Test + public void test_WebIsHidden_mapping() { + childWdElement.visibility = "hidden"; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsHidden, null)); + } + @Test public void test_WebAriaLabelledBy_mapping() { childWdElement.ariaLabelledBy = "search-label"; @@ -160,6 +449,62 @@ public void test_WebAriaModal_mapping() { assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebAriaModal, null)); } + @Test + public void test_WebTarget_mapping() { + childWdElement.target = "_blank"; + assertEquals("_blank", childWdWidget.get(WdTags.WebTarget, "")); + } + + @Test + public void test_WebAlt_mapping() { + childWdElement.alt = "banner-image"; + assertEquals("banner-image", childWdWidget.get(WdTags.WebAlt, "")); + } + + @Test + public void test_WebAccessKey_mapping() { + childWdElement.accessKey = "k"; + assertEquals("k", childWdWidget.get(WdTags.WebAccessKey, "")); + } + + @Test + public void test_WebAcceleratorKey_mapping() { + childWdElement.acceleratorKey = "Ctrl+K"; + assertEquals("Ctrl+K", childWdWidget.get(WdTags.WebAcceleratorKey, "")); + } + + @Test + public void test_WebElementSelenium_mapping() { + RemoteWebElement remoteWebElement = new RemoteWebElement(); + childWdElement.remoteWebElement = remoteWebElement; + assertSame(remoteWebElement, childWdWidget.get(WdTags.WebElementSelenium, null)); + } + + @Test + public void test_WebAttributeMap_mapping() { + childWdElement.attributeMap = new HashMap<>(); + childWdElement.attributeMap.put("aria-label", "Search"); + assertEquals(childWdElement.attributeMap, childWdWidget.get(WdTags.WebAttributeMap, null)); + } + + @Test + public void test_WebIsFullOnScreen_mapping() { + childWdElement.isFullVisibleOnScreen = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsFullOnScreen, null)); + } + + @Test + public void test_WebGenericTitle_mapping() { + childWdElement.genericTitle = "generic-title"; + assertEquals("generic-title", childWdWidget.get(WdTags.WebGenericTitle, "")); + } + + @Test + public void test_WebIsWindowModal_mapping() { + childWdElement.isModal = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsWindowModal, null)); + } + @Test public void test_NullDefaultValue_string() { childWdElement.ariaLabel = null; diff --git a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java index f187e78be..61e24f04e 100644 --- a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java +++ b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java @@ -11,50 +11,178 @@ public class TestWebdriverStateManagementTag { @Test - public void testWebdriverMapping() { - WdState wdState = new WdState(null); - wdState.set(WdTags.WebId, "UniqueId"); - wdState.set(WdTags.WebHref, "Link.org"); - - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetId), "UniqueId"); - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetHref), "Link.org"); - } - - @Test - public void testWebdriverAriaMapping() { - WdState wdState = new WdState(null); - wdState.set(WdTags.WebAriaLabel, "this is an aria label"); - wdState.set(WdTags.WebAriaLabelledBy, "and this an aria labelled by"); - - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetAriaLabel), "this is an aria label"); - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetAriaLabelledBy), "and this an aria labelled by"); - } - - @Test - public void testWebdriverCodingIDs() { + public void test_WebdriverCodingIDs_mapping() { Tag[] abstractTags = new Tag[]{StateManagementTags.WebWidgetId, StateManagementTags.WebWidgetHref}; CodingManager.setCustomTagsForAbstractId(abstractTags); WdState wdState = new WdState(null); WdWidget wdWidget = new WdWidget(wdState, wdState, null); - wdWidget.set(WdTags.WebId, "UniqueId"); - wdWidget.set(WdTags.WebHref, "Link.org"); // Build the first AbstractID and check the StateManagementTags uses the Wd values + wdWidget.set(WdTags.WebId, "UniqueId"); + wdWidget.set(WdTags.WebHref, "Link.org"); CodingManager.buildIDs(wdWidget); - Assert.assertEquals(wdWidget.get(Tags.AbstractID), "WA1r9aad8101406797296"); + String originalAbstractId = wdWidget.get(Tags.AbstractID); + Assert.assertTrue(originalAbstractId.contains("WA")); // Change WebId value to verify the AbstractID changes wdWidget.set(WdTags.WebId, "UniqueIdNEW"); wdWidget.set(WdTags.WebHref, "Link.org"); CodingManager.buildIDs(wdWidget); - Assert.assertEquals(wdWidget.get(Tags.AbstractID), "WA1vaf6dg131408350915"); + String webIdChangedAbstractId = wdWidget.get(Tags.AbstractID); + Assert.assertTrue(webIdChangedAbstractId.contains("WA")); + Assert.assertNotEquals(originalAbstractId, webIdChangedAbstractId); // Change WebHref value to verify the AbstractID changes wdWidget.set(WdTags.WebId, "UniqueIdNEW"); wdWidget.set(WdTags.WebHref, "Link.com"); CodingManager.buildIDs(wdWidget); - Assert.assertEquals(wdWidget.get(Tags.AbstractID), "WA1gxbnlz131612186923"); + String webHrefChangedAbstractId = wdWidget.get(Tags.AbstractID); + Assert.assertTrue(webHrefChangedAbstractId.contains("WA")); + Assert.assertNotEquals(originalAbstractId, webHrefChangedAbstractId); + Assert.assertNotEquals(webIdChangedAbstractId, webHrefChangedAbstractId); + } + + @Test + public void test_WebId_mapping() { + assertStringMapping(WdTags.WebId, StateManagementTags.WebWidgetId, "UniqueId"); + } + + @Test + public void test_WebName_mapping() { + assertStringMapping(WdTags.WebName, StateManagementTags.WebWidgetName, "search-input"); + } + + @Test + public void test_WebTagName_mapping() { + assertStringMapping(WdTags.WebTagName, StateManagementTags.WebWidgetTagName, "input"); + } + + @Test + public void test_WebTextContent_mapping() { + assertStringMapping(WdTags.WebTextContent, StateManagementTags.WebWidgetTextContent, "Text content"); + } + + @Test + public void test_WebInnerText_mapping() { + assertStringMapping(WdTags.WebInnerText, StateManagementTags.WebWidgetInnerText, "Visible text"); + } + + @Test + public void test_WebTitle_mapping() { + assertStringMapping(WdTags.WebTitle, StateManagementTags.WebWidgetTitle, "Widget title"); + } + + @Test + public void test_WebHref_mapping() { + assertStringMapping(WdTags.WebHref, StateManagementTags.WebWidgetHref, "Link.org"); + } + + @Test + public void test_WebValue_mapping() { + assertStringMapping(WdTags.WebValue, StateManagementTags.WebWidgetValue, "hello"); + } + + @Test + public void test_WebStyle_mapping() { + assertStringMapping(WdTags.WebStyle, StateManagementTags.WebWidgetStyle, "color:red;"); + } + + @Test + public void test_WebTarget_mapping() { + assertStringMapping(WdTags.WebTarget, StateManagementTags.WebWidgetTarget, "_blank"); + } + + @Test + public void test_WebAlt_mapping() { + assertStringMapping(WdTags.WebAlt, StateManagementTags.WebWidgetAlt, "banner-image"); + } + + @Test + public void test_WebType_mapping() { + assertStringMapping(WdTags.WebType, StateManagementTags.WebWidgetType, "text"); + } + + @Test + public void test_WebCssClasses_mapping() { + assertStringMapping(WdTags.WebCssClasses, StateManagementTags.WebWidgetCssClasses, "[btn, primary]"); + } + + @Test + public void test_WebDisplay_mapping() { + assertStringMapping(WdTags.WebDisplay, StateManagementTags.WebWidgetDisplay, "block"); + } + + @Test + public void test_WebSrc_mapping() { + assertStringMapping(WdTags.WebSrc, StateManagementTags.WebWidgetSrc, "https://testar.org/image.png"); + } + + @Test + public void test_WebPlaceholder_mapping() { + assertStringMapping(WdTags.WebPlaceholder, StateManagementTags.WebWidgetPlaceholder, "Type here"); + } + + @Test + public void test_WebIsOffScreen_mapping() { + assertBooleanMapping(WdTags.WebIsOffScreen, StateManagementTags.WebWidgetIsOffScreen, Boolean.TRUE); + } + + @Test + public void test_WebIsDisabled_mapping() { + assertBooleanMapping(WdTags.WebIsDisabled, StateManagementTags.WebWidgetIsDisabled, Boolean.TRUE); + } + + @Test + public void test_WebAriaLabel_mapping() { + assertStringMapping(WdTags.WebAriaLabel, StateManagementTags.WebWidgetAriaLabel, "aria-label"); + } + + @Test + public void test_WebAriaLabelledBy_mapping() { + assertStringMapping(WdTags.WebAriaLabelledBy, StateManagementTags.WebWidgetAriaLabelledBy, "aria labelled by"); + } + + @Test + public void test_WidgetControlType_mapping() { + assertStringMapping(WdTags.WebTagName, StateManagementTags.WidgetControlType, "button"); + } + + @Test + public void test_WidgetTitle_mapping() { + assertStringMapping(WdTags.WebGenericTitle, StateManagementTags.WidgetTitle, "Generic title"); + } + + @Test + public void test_WidgetIsEnabled_mapping() { + assertBooleanMapping(WdTags.WebIsEnabled, StateManagementTags.WidgetIsEnabled, Boolean.TRUE); + } + + @Test + public void test_WidgetPath_mapping() { + assertStringMapping(Tags.Path, StateManagementTags.WidgetPath, "/html/body/div[1]"); + } + + @Test + public void test_WidgetIsContentElement_mapping() { + assertBooleanMapping(WdTags.WebIsContentElement, StateManagementTags.WidgetIsContentElement, Boolean.TRUE); + } + + @Test + public void test_WidgetIsControlElement_mapping() { + assertBooleanMapping(WdTags.WebIsControlElement, StateManagementTags.WidgetIsControlElement, Boolean.TRUE); + } + + private void assertStringMapping(Tag wdTag, Tag stateTag, String value) { + WdState wdState = new WdState(null); + wdState.set(wdTag, value); + Assert.assertEquals(value, wdState.get(stateTag)); + } + + private void assertBooleanMapping(Tag wdTag, Tag stateTag, Boolean value) { + WdState wdState = new WdState(null); + wdState.set(wdTag, value); + Assert.assertEquals(value, wdState.get(stateTag)); } }