diff --git a/CHANGELOG b/CHANGELOG index 9e688c3d5..d21b346f9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +#TESTAR v2.8.5 (17-Mar-2026) +- Bump io.appium:java-client from 10.0.0 to 10.1.0 +- Update JS getIsBlockedTestar logic +- Add continue, completed, invalid, LLM Oracle logic +- Add LlmTestGoalOrchestrator logic +- Add WebIsDisplayed to StateCondition +- Update selectAction in model evaluator protocols to avoid LLM select extra actions + + #TESTAR v2.8.4 (12-Mar-2026) - Remove abstract actionIds from LLM ActionHistory - Update getElementDescription prioritty and length diff --git a/VERSION b/VERSION index 0409c1638..7f04bb11e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.4 \ No newline at end of file +2.8.5 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 66af9ddf7..07914cfd5 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // https://mvnrepository.com/artifact/io.github.bonigarcia/webdrivermanager implementation group: 'io.github.bonigarcia', name: 'webdrivermanager', version: '6.3.3' // https://mvnrepository.com/artifact/io.appium/java-client - implementation group: 'io.appium', name: 'java-client', version: '10.0.0' + implementation group: 'io.appium', name: 'java-client', version: '10.1.0' // https://mvnrepository.com/artifact/com.orientechnologies/orientdb-graphdb implementation group: 'com.orientechnologies', name: 'orientdb-graphdb', version: '3.2.38' // https://mvnrepository.com/artifact/org.graalvm.sdk/graal-sdk diff --git a/core/src/org/testar/StateManagementTags.java b/core/src/org/testar/StateManagementTags.java index 91782f0fe..1fa45d91b 100644 --- a/core/src/org/testar/StateManagementTags.java +++ b/core/src/org/testar/StateManagementTags.java @@ -308,7 +308,8 @@ public enum Group {General, ControlPattern, WebDriver} public static final Tag WebWidgetId = Tag.from("Web Widget id", String.class); public static final Tag WebWidgetName = Tag.from("Web Widget name", String.class); public static final Tag WebWidgetTagName = Tag.from("Web Widget tag name", String.class); - public static final Tag WebWidgetTextContent= Tag.from("Web Widget text content", String.class); + public static final Tag WebWidgetTextContent = Tag.from("Web Widget text content", String.class); + public static final Tag WebWidgetInnerText = Tag.from("Web Widget inner text", String.class); public static final Tag WebWidgetTitle = Tag.from("Web Widget title", String.class); public static final Tag WebWidgetHref = Tag.from("Web Widget href", String.class); public static final Tag WebWidgetValue = Tag.from("Web Widget value", String.class); @@ -402,6 +403,7 @@ public enum Group {General, ControlPattern, WebDriver} add(WebWidgetName); add(WebWidgetTagName); add(WebWidgetTextContent); + add(WebWidgetInnerText); add(WebWidgetTitle); add(WebWidgetHref); add(WebWidgetValue); @@ -501,6 +503,7 @@ public static boolean isStateManagementTag(Tag tag) { settingsMap.put(WebWidgetName, "WebWidgetName"); settingsMap.put(WebWidgetTagName, "WebWidgetTagName"); settingsMap.put(WebWidgetTextContent, "WebWidgetTextContent"); + settingsMap.put(WebWidgetInnerText, "WebWidgetInnerText"); settingsMap.put(WebWidgetTitle, "WebWidgetTitle"); settingsMap.put(WebWidgetHref, "WebWidgetHref"); settingsMap.put(WebWidgetValue, "WebWidgetValue"); @@ -590,6 +593,7 @@ public static boolean isStateManagementTag(Tag tag) { put(WebWidgetName, Group.WebDriver); put(WebWidgetTagName, Group.WebDriver); put(WebWidgetTextContent, Group.WebDriver); + put(WebWidgetInnerText, Group.WebDriver); put(WebWidgetTitle, Group.WebDriver); put(WebWidgetHref, Group.WebDriver); put(WebWidgetValue, Group.WebDriver); diff --git a/core/src/org/testar/monkey/alayer/Verdict.java b/core/src/org/testar/monkey/alayer/Verdict.java index ced9c6b77..99271dfc4 100644 --- a/core/src/org/testar/monkey/alayer/Verdict.java +++ b/core/src/org/testar/monkey/alayer/Verdict.java @@ -31,7 +31,6 @@ package org.testar.monkey.alayer; import java.io.Serializable; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -104,6 +103,7 @@ public enum Severity { /** CRITICAL (0.9 - 1.0) **/ CRITICAL(0.9, "CRITICAL"), + LLM_INVALID(0.91, "LLM_INVALID"), // LLM detected objective steps were followed but resulting state is invalid NOT_RESPONDING(0.99999990, "NOT_RESPONDING"), // Unresponsive UNEXPECTEDCLOSE(0.99999999, "UNEXPECTEDCLOSE"), // Crash? Unexpected close? FAIL(1.0, "FAIL"); diff --git a/testar/resources/prompts/fewshot_gemini_oracle_goal_status.json b/testar/resources/prompts/fewshot_gemini_oracle_goal_status.json new file mode 100644 index 000000000..1914785ec --- /dev/null +++ b/testar/resources/prompts/fewshot_gemini_oracle_goal_status.json @@ -0,0 +1,58 @@ +[ + { + "role": "user", + "parts": [ + { + "text": "You are a web test agent. You will receive a test objective and a list of widgets that describe the current application state. Decide one verdict only: CONTINUE (goal not met yet), COMPLETED (goal met correctly), or INVALID (the expected end result is contradicted by the current state, indicating a functional issue). Use INVALID only when there is clear evidence that the expected result should be present but the observed state indicates an incorrect behavior. If evidence is not sufficient, choose CONTINUE. Respond in JSON only with this schema: {\"status\":\"CONTINUE|COMPLETED|INVALID\",\"info\":\"short reason\"}." + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Customer Login, Widget: Username, Widget: Password, Widget: Forgot login info?. Is the test objective met in this state?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "{\"status\":\"CONTINUE\",\"info\":\"login form is still present and no successful session is visible\"}" + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Welcome John Smith, Widget: Account Services, Widget: Log Out. Is the test objective met in this state?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "{\"status\":\"COMPLETED\",\"info\":\"user is authenticated and the expected welcome message is visible\"}" + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "We are testing the \"Demo\" web application. The objective of the test is to log in with valid credentials username \"John\" and password \"Smith\" and verify successful authentication. The current state of the application contains the widgets: Widget: Login failure, Widget: Invalid Credentials. Is the test objective met in this state?" + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "{\"status\":\"INVALID\",\"info\":\"The objective expects successful authentication with valid credentials, but the state shows a contradictory login failure\"}" + } + ] + } +] diff --git a/testar/resources/prompts/fewshot_openai_oracle_goal_status.json b/testar/resources/prompts/fewshot_openai_oracle_goal_status.json new file mode 100644 index 000000000..614fcef01 --- /dev/null +++ b/testar/resources/prompts/fewshot_openai_oracle_goal_status.json @@ -0,0 +1,30 @@ +[ + { + "role": "system", + "content": "You are a web test agent. You will receive a test objective and a list of widgets that describe the current application state. Decide one verdict only: CONTINUE (goal not met yet), COMPLETED (goal met correctly), or INVALID (the expected end result is contradicted by the current state, indicating a functional issue). Use INVALID only when there is clear evidence that the expected result should be present but the observed state indicates an incorrect behavior. If evidence is not sufficient, choose CONTINUE. Respond in JSON only with this schema: {\"status\":\"CONTINUE|COMPLETED|INVALID\",\"info\":\"short reason\"}." + }, + { + "role": "user", + "content": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Customer Login, Widget: Username, Widget: Password, Widget: Forgot login info?. Is the test objective met in this state?" + }, + { + "role": "assistant", + "content": "{\"status\":\"CONTINUE\",\"info\":\"The login form is still present and no successful session is visible\"}" + }, + { + "role": "user", + "content": "We are testing the \"Demo\" web application. The objective of the test is to log in with username \"John\" and password \"Smith\". The current state of the application contains the widgets: Widget: Welcome John Smith, Widget: Account Services, Widget: Log Out. Is the test objective met in this state?" + }, + { + "role": "assistant", + "content": "{\"status\":\"COMPLETED\",\"info\":\"The John user is authenticated and the expected welcome message is visible\"}" + }, + { + "role": "user", + "content": "We are testing the \"Demo\" web application. The objective of the test is to log in with valid credentials username \"John\" and password \"Smith\" and verify successful authentication. The current state of the application contains the widgets: Widget: Login failure, Widget: Invalid Credentials. Is the test objective met in this state?" + }, + { + "role": "assistant", + "content": "{\"status\":\"INVALID\",\"info\":\"The objective expects successful authentication with valid credentials, but the state shows a contradictory login failure\"}" + } +] diff --git a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java index c34fe87c8..01f62ad52 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java +++ b/testar/resources/settings/03_webdriver_llm_parabank/Protocol_03_webdriver_llm_parabank.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.OracleWebPromptGenerator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; @@ -51,20 +52,18 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_03_webdriver_llm_parabank extends WebdriverProtocol { // The LLM Action selector needs to be initialize with the settings private LlmActionSelector llmActionSelector; - private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; // The LLM Oracle needs to be initialize with the settings private LlmOracle llmOracle; + private List testGoals = new ArrayList<>(); + private LlmTestGoalOrchestrator testGoalOrchestrator; + /** * Called once during the life time of TESTAR * This method can be used to perform initial setup work @@ -83,6 +82,11 @@ protected void initialize(Settings settings) { // Initialize the LlmOracle using the LLM settings llmOracle = new LlmOracle(settings, new OracleWebPromptGenerator()); + + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + llmOracle.reset(goal, appendPreviousGoal); + }); } /** @@ -91,16 +95,7 @@ protected void initialize(Settings settings) { @Override protected void preSequencePreparations() { super.preSequencePreparations(); - - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); - // Reset llm oracle - llmOracle.reset(currentTestGoal, false); + testGoalOrchestrator.startSequence(); } private void setupTestGoals(List testGoalsList) { @@ -162,28 +157,12 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - // Use the LLM as an Oracle to determine if the test goal has been completed - List llmVerdicts = llmOracle.getVerdicts(state); - - for(Verdict llmVerdict : llmVerdicts) { - if(llmVerdict.severity() == Verdict.Severity.LLM_COMPLETE.getValue()) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return Collections.singletonList(llmVerdict); - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - llmOracle.reset(currentTestGoal, true); - } - } - } + List verdicts = super.getVerdicts(state); + + // Add the LLM Oracle verdicts to determine if the test goal has been completed + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(llmOracle.getVerdicts(state))); - return super.getVerdicts(state); + return verdicts; } /** @@ -229,9 +208,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; diff --git a/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_loan.goal b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_invalid_loan.goal similarity index 81% rename from testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_loan.goal rename to testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_invalid_loan.goal index 310dd67fd..8015a4d63 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_loan.goal +++ b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_invalid_loan.goal @@ -2,4 +2,4 @@ As a user, I want to log in with the credentials john/demo Then the Welcome John Smith message is shown ; As a user, I want to request a loan with an amount '999999', a down payment of '100', for the account '54321' -Then the loan is denied due to insufficient funds \ No newline at end of file +Then the loan is approved by the bank entity \ No newline at end of file diff --git a/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal new file mode 100644 index 000000000..a04edc828 --- /dev/null +++ b/testar/resources/settings/03_webdriver_llm_parabank/parabank_login_request_valid_loan.goal @@ -0,0 +1,5 @@ +As a user, I want to log in with the credentials john/demo +Then the Welcome John Smith message is shown +; +As a user, I want to request a loan with an amount '500', a down payment of '100', for the account '54321' +Then the loan is approved by the bank entity \ No newline at end of file diff --git a/testar/resources/settings/03_webdriver_llm_parabank/test.settings b/testar/resources/settings/03_webdriver_llm_parabank/test.settings index 2e3bba187..33bf23fe2 100644 --- a/testar/resources/settings/03_webdriver_llm_parabank/test.settings +++ b/testar/resources/settings/03_webdriver_llm_parabank/test.settings @@ -147,7 +147,7 @@ DataStoreUser = DataStorePassword = DataStoreMode = none ApplicationName = parabank -ApplicationVersion = loan_denied +ApplicationVersion = request_loan ActionSelectionAlgorithm = random StateModelStoreWidgets = false ResetDataStore = false @@ -173,11 +173,11 @@ LlmReasoning = minimal LlmHostUrl = https://api.openai.com/v1/chat/completions LlmAuthorizationHeader = Bearer %OPENAI_API% LlmActionFewshotFile = prompts/fewshot_openai_action.json -LlmOracleFewshotFile = prompts/fewshot_openai_oracle.json +LlmOracleFewshotFile = prompts/fewshot_openai_oracle_goal_status.json LlmTemperature = 0.2 LlmHistorySize = 5 LlmStateless = true -LlmTestGoals = As a user, I want to log in with the credentials john/demo\\nThen a Request Loan link is shown;As a user, I want to request a loan with an amount '999999', a down payment of '100', for the account '54321'\\nThen the loan is denied due to insufficient funds +LlmTestGoals = As a user, I want to log in with the credentials john/demo\\nThen a Request Loan link is shown;As a user, I want to request a loan with an amount '999999', a down payment of '100', for the account '54321'\\nThen the loan is approved by the bank entity ################################################################# # WebDriver features diff --git a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java index 5f6373116..87d683940 100644 --- a/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_evaluator/Protocol_webdriver_llm_state_model_evaluator.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; @@ -67,8 +68,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_webdriver_llm_state_model_evaluator extends WebdriverProtocol { // The LLM Action selector needs to be initialize with the settings @@ -77,8 +76,7 @@ public class Protocol_webdriver_llm_state_model_evaluator extends WebdriverProto private ConditionEvaluator conditionEvaluator; private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; + private LlmTestGoalOrchestrator testGoalOrchestrator; /** * Called once during the life time of TESTAR @@ -103,6 +101,11 @@ protected void initialize(Settings settings) { metricsManager = new MetricsManager(new LlmMetricsCollector(testGoals)); conditionEvaluator = new BasicConditionEvaluator(); + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + conditionEvaluator.clear(); + conditionEvaluator.addConditions(goal.getCompletionConditions()); + }); } private void setupTestGoals(List testGoalsList) { @@ -119,16 +122,6 @@ private void setupTestGoals(List testGoalsList) { protected void preSequencePreparations() { super.preSequencePreparations(); - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); - // Use timestamp to create a unique new state model String appName = settings.get(ConfigTags.ApplicationName, ""); String appVersion = settings.get(ConfigTags.ApplicationVersion, ""); @@ -138,6 +131,8 @@ protected void preSequencePreparations() { stateModelManager.notifyTestingEnded(); stateModelManager = StateModelManagerFactory.getStateModelManager( appName, (appVersion + "_" + timestamp), settings); + + testGoalOrchestrator.startSequence(); } /** @@ -192,26 +187,16 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - String modelIdentifier = stateModelManager.getModelIdentifier(); + List verdicts = super.getVerdicts(state); - if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - } - } + // Add the State Model Condition verdicts to determine if the test goal has been completed + String modelIdentifier = stateModelManager.getModelIdentifier(); + List stateModelConditionVerdicts = conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager) + ? Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "Current test goal conditions completed.")) + : Collections.singletonList(Verdict.OK); + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(stateModelConditionVerdicts)); - return super.getVerdicts(state); + return verdicts; } /** @@ -257,9 +242,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; @@ -328,19 +310,24 @@ protected boolean isTypeable(Widget widget) { */ @Override protected Action selectAction(State state, Set actions) { + // State model conditions are check one state behind, producing one extra selected action + // If the state model condition is done, avoid executing an extra step + if (conditionEvaluator.evaluateConditions(stateModelManager.getModelIdentifier(), stateModelManager)) { + return prepareActionForExecution(state, new NOP()); + } + Action toExecute = llmActionSelector.selectAction(state, actions); + return prepareActionForExecution(state, toExecute); + } - // We need to set a state to NOP actions - if(toExecute instanceof NOP) { - toExecute.set(Tags.OriginWidget, state); + private Action prepareActionForExecution(State state, Action action) { + if (action instanceof NOP) { + action.set(Tags.OriginWidget, state); } - - // We need the AbstractID for the state model - if(toExecute.get(Tags.AbstractID, null) == null) { - CodingManager.buildIDs(state, Collections.singleton(toExecute)); + if(action.get(Tags.AbstractID, null) == null) { + CodingManager.buildIDs(state, Collections.singleton(action)); } - - return toExecute; + return action; } /** diff --git a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java index ec24efdcb..d06b63a75 100644 --- a/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_model_transition_evaluator/Protocol_webdriver_llm_state_model_transition_evaluator.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; @@ -67,8 +68,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_webdriver_llm_state_model_transition_evaluator extends WebdriverProtocol { // The LLM Action selector needs to be initialize with the settings @@ -77,8 +76,7 @@ public class Protocol_webdriver_llm_state_model_transition_evaluator extends Web private ConditionEvaluator conditionEvaluator; private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; + private LlmTestGoalOrchestrator testGoalOrchestrator; /** * Called once during the life time of TESTAR @@ -103,6 +101,11 @@ protected void initialize(Settings settings) { metricsManager = new MetricsManager(new LlmMetricsCollector(testGoals)); conditionEvaluator = new BasicConditionEvaluator(); + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + conditionEvaluator.clear(); + conditionEvaluator.addConditions(goal.getCompletionConditions()); + }); } private void setupTestGoals(List testGoalsList) { @@ -124,16 +127,6 @@ private void setupTestGoals(List testGoalsList) { protected void preSequencePreparations() { super.preSequencePreparations(); - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); - // Use timestamp to create a unique new state model String appName = settings.get(ConfigTags.ApplicationName, ""); String appVersion = settings.get(ConfigTags.ApplicationVersion, ""); @@ -143,6 +136,8 @@ protected void preSequencePreparations() { stateModelManager.notifyTestingEnded(); stateModelManager = StateModelManagerFactory.getStateModelManager( appName, (appVersion + "_" + timestamp), settings); + + testGoalOrchestrator.startSequence(); } /** @@ -197,26 +192,16 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - String modelIdentifier = stateModelManager.getModelIdentifier(); + List verdicts = super.getVerdicts(state); - if(conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager)) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - System.out.println("Test goal completed, but no more test goals."); - return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); - } else { - System.out.println("Test goal completed, moving to next test goal."); - llmActionSelector.reset(currentTestGoal, true); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - } - } + // Add the State Model Condition verdicts to determine if the test goal has been completed + String modelIdentifier = stateModelManager.getModelIdentifier(); + List stateModelConditionVerdicts = conditionEvaluator.evaluateConditions(modelIdentifier, stateModelManager) + ? Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "Current test goal conditions completed.")) + : Collections.singletonList(Verdict.OK); + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(stateModelConditionVerdicts)); - return super.getVerdicts(state); + return verdicts; } /** @@ -262,9 +247,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; @@ -333,19 +315,24 @@ protected boolean isTypeable(Widget widget) { */ @Override protected Action selectAction(State state, Set actions) { + // State model conditions are check one state behind, producing one extra selected action + // If the state model condition is done, avoid executing an extra step + if (conditionEvaluator.evaluateConditions(stateModelManager.getModelIdentifier(), stateModelManager)) { + return prepareActionForExecution(state, new NOP()); + } + Action toExecute = llmActionSelector.selectAction(state, actions); + return prepareActionForExecution(state, toExecute); + } - // We need to set a state to NOP actions - if(toExecute instanceof NOP) { - toExecute.set(Tags.OriginWidget, state); + private Action prepareActionForExecution(State state, Action action) { + if (action instanceof NOP) { + action.set(Tags.OriginWidget, state); } - - // We need the AbstractID for the state model - if(toExecute.get(Tags.AbstractID, null) == null) { - CodingManager.buildIDs(state, Collections.singleton(toExecute)); + if(action.get(Tags.AbstractID, null) == null) { + CodingManager.buildIDs(state, Collections.singleton(action)); } - - return toExecute; + return action; } /** diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java index bdf51b222..7cd031b52 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/Protocol_webdriver_llm_state_widgets_evaluator.java @@ -32,6 +32,7 @@ import org.testar.SutVisualization; import org.testar.action.priorization.llm.LlmActionSelector; import org.testar.llm.LlmTestGoal; +import org.testar.llm.LlmTestGoalOrchestrator; import org.testar.llm.prompt.ActionWebPromptGenerator; import org.testar.managers.InputDataManager; import org.testar.monkey.alayer.*; @@ -67,8 +68,6 @@ import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; -import static org.testar.monkey.alayer.webdriver.Constants.scrollArrowSize; -import static org.testar.monkey.alayer.webdriver.Constants.scrollThick; public class Protocol_webdriver_llm_state_widgets_evaluator extends WebdriverProtocol { @@ -77,8 +76,7 @@ public class Protocol_webdriver_llm_state_widgets_evaluator extends WebdriverPro private ConditionEvaluator conditionEvaluator; private List testGoals = new ArrayList<>(); - private Queue testGoalQueue; - private LlmTestGoal currentTestGoal; + private LlmTestGoalOrchestrator testGoalOrchestrator; /** * Called once during the life time of TESTAR @@ -97,11 +95,16 @@ protected void initialize(Settings settings) { llmActionSelector = new LlmActionSelector(settings, new ActionWebPromptGenerator()); conditionEvaluator = new BasicConditionEvaluator(); + testGoalOrchestrator = new LlmTestGoalOrchestrator(testGoals, (goal, appendPreviousGoal) -> { + llmActionSelector.reset(goal, appendPreviousGoal); + conditionEvaluator.clear(); + conditionEvaluator.addConditions(goal.getCompletionConditions()); + }); } private void setupTestGoals(List testGoalsList) { for(String testGoal : testGoalsList) { - CheckConditionEvaluator checkEvaluator = new CheckConditionEvaluator(WdTags.WebInnerHTML, testGoal); + CheckConditionEvaluator checkEvaluator = new CheckConditionEvaluator(WdTags.WebTextContent, testGoal); testGoals.add(new LlmTestGoal(testGoal, checkEvaluator.getConditions())); } } @@ -112,16 +115,7 @@ private void setupTestGoals(List testGoalsList) { @Override protected void preSequencePreparations() { super.preSequencePreparations(); - - // Setup test goal queue - testGoalQueue = new LinkedList<>(); - testGoalQueue.addAll(testGoals); - currentTestGoal = testGoalQueue.poll(); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - - // Reset llm action selector - llmActionSelector.reset(currentTestGoal, false); + testGoalOrchestrator.startSequence(); } /** @@ -176,24 +170,15 @@ protected State getState(SUT system) throws StateBuildException { */ @Override protected List getVerdicts(State state) { - // Apply state conditions to check if the test goal has been completed - if(conditionEvaluator.evaluateConditions(state)) { - // Test goal was completed, retrieve next test goal from queue. - currentTestGoal = testGoalQueue.poll(); - - // Poll returns null if there are no more items remaining in the queue. - if(currentTestGoal == null) { - // No more test goals remaining, terminate sequence. - return Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "All test goal conditions completed.")); - } else { - llmActionSelector.reset(currentTestGoal, true); - conditionEvaluator.clear(); - conditionEvaluator.addConditions(currentTestGoal.getCompletionConditions()); - System.out.println("Test goal completed, moving to next test goal: " + currentTestGoal.getTestGoal()); - } - } + List verdicts = super.getVerdicts(state); + + // Add the State Model Condition verdicts to determine if the test goal has been completed + List stateConditionVerdicts = conditionEvaluator.evaluateConditions(state) + ? Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "Current test goal conditions completed.")) + : Collections.singletonList(Verdict.OK); + verdicts.addAll(testGoalOrchestrator.processGoalVerdicts(stateConditionVerdicts)); - return super.getVerdicts(state); + return verdicts; } /** @@ -239,9 +224,6 @@ protected Set deriveActions(SUT system, State state) throws ActionBuildE continue; } - // slides can happen, even though the widget might be blocked - //addSlidingActions(actions, ac, scrollArrowSize, scrollThick, widget); - // If the element is blocked, Testar can't click on or type in the widget if (widget.get(Blocked, false) && !widget.get(WdTags.WebIsShadow, false)) { continue; diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal index 2c7aeb3bb..6e6c8fb75 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/parabank_login_request_loan_conditions.goal @@ -2,4 +2,4 @@ As a user, I want to log in with the credentials john/demo Check: John Smith ; As a user, I want to request a loan with an amount '999999' and a down payment of '100' -Check:

We cannot grant a loan in that amount with your available funds.

\ No newline at end of file +Check: We cannot grant a loan in that amount with your available funds. \ No newline at end of file diff --git a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings index 3c58ad2be..43172428c 100644 --- a/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings +++ b/testar/resources/settings/webdriver_llm_state_widgets_evaluator/test.settings @@ -126,7 +126,7 @@ MaxTime = 3.1536E7 ################################################################# ApplicationName = parabank -ApplicationVersion = login +ApplicationVersion = check_loan ################################################################# # State model inference settings @@ -185,7 +185,7 @@ LlmOracleFewshotFile = LlmTemperature = 0.2 LlmHistorySize = 5 LlmStateless = true -LlmTestGoals = Use the username 'john' and the password 'demo' to log in the application\\nCheck: Welcome John Smith +LlmTestGoals = Use the username 'john' and the password 'demo' to log in the application\\nCheck: Welcome John Smith;Apply for a loan of 999999 with a down payment of 100\\nCheck: We cannot grant a loan in that amount with your available funds ################################################################# # WebDriver features diff --git a/testar/src/org/testar/llm/LlmTestGoal.java b/testar/src/org/testar/llm/LlmTestGoal.java index 44f0099b6..006a0558b 100644 --- a/testar/src/org/testar/llm/LlmTestGoal.java +++ b/testar/src/org/testar/llm/LlmTestGoal.java @@ -36,7 +36,7 @@ /** * Contains a test goal for the llm to complete along with the conditions under which the test goal is considered - * completed. TODO: Rework GUI to support this new structure. + * completed. */ public class LlmTestGoal { private String testGoal; diff --git a/testar/src/org/testar/llm/LlmTestGoalOrchestrator.java b/testar/src/org/testar/llm/LlmTestGoalOrchestrator.java new file mode 100644 index 000000000..6d19a0d49 --- /dev/null +++ b/testar/src/org/testar/llm/LlmTestGoalOrchestrator.java @@ -0,0 +1,112 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.llm; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.testar.monkey.alayer.Verdict; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.function.BiConsumer; + +/** + * Coordinates multi-goal progression and stop conditions for LLM goal-oriented protocols. + */ +public class LlmTestGoalOrchestrator { + static final Logger logger = LogManager.getLogger(); + + private final List configuredGoals; + private final BiConsumer goalActivationHandler; + + private Queue goalQueue = new LinkedList<>(); + private LlmTestGoal currentGoal; + + public LlmTestGoalOrchestrator(List goals, BiConsumer goalActivationHandler) { + this.configuredGoals = goals == null ? Collections.emptyList() : new ArrayList<>(goals); + this.goalActivationHandler = goalActivationHandler; + } + + public void startSequence() { + goalQueue = new LinkedList<>(configuredGoals); + currentGoal = goalQueue.poll(); + activateCurrentGoal(false); + } + + public LlmTestGoal getCurrentGoal() { + return currentGoal; + } + + /** + * Processes goal evaluator verdicts (LLM or condition-based) and updates progression. + * + * @param goalVerdicts verdicts produced by a goal evaluator + * @return test goal verdicts if the sequence should stop, empty list otherwise + */ + public List processGoalVerdicts(List goalVerdicts) { + if (goalVerdicts == null || goalVerdicts.isEmpty()) { + return Collections.emptyList(); + } + + for (Verdict verdict : goalVerdicts) { + if (verdict == null) { + continue; + } + + if (verdict.severity() == Verdict.Severity.LLM_INVALID.getValue()) { + logger.info("LLM detected invalid behavior, stopping test sequence."); + return Collections.singletonList(verdict); + } + + if (verdict.isCompletion()) { + currentGoal = goalQueue.poll(); + if (currentGoal == null) { + logger.info("Test goal completed, but no more test goals."); + return Collections.singletonList(verdict); + } + logger.info("Test goal completed, moving to next test goal."); + activateCurrentGoal(true); + } + } + + return Collections.emptyList(); + } + + private void activateCurrentGoal(boolean appendPreviousGoal) { + if (currentGoal == null || goalActivationHandler == null) { + return; + } + goalActivationHandler.accept(currentGoal, appendPreviousGoal); + } +} diff --git a/testar/src/org/testar/monkey/Main.java b/testar/src/org/testar/monkey/Main.java index 072185d2a..438c7e9f5 100644 --- a/testar/src/org/testar/monkey/Main.java +++ b/testar/src/org/testar/monkey/Main.java @@ -64,7 +64,7 @@ public class Main { - public static final String TESTAR_VERSION = "v2.8.4 (12-Mar-2026)"; + public static final String TESTAR_VERSION = "v2.8.5 (17-Mar-2026)"; //public static final String TESTAR_DIR_PROPERTY = "DIRNAME"; //Use the OS environment to obtain TESTAR directory public static final String SETTINGS_FILE = "test.settings"; diff --git a/testar/src/org/testar/oracles/llm/LlmOracle.java b/testar/src/org/testar/oracles/llm/LlmOracle.java index d3a01b395..b9b17b50e 100644 --- a/testar/src/org/testar/oracles/llm/LlmOracle.java +++ b/testar/src/org/testar/oracles/llm/LlmOracle.java @@ -66,8 +66,6 @@ import org.testar.plugin.OperatingSystems; import org.testar.settings.Settings; -import com.google.gson.Gson; - public class LlmOracle implements Oracle { protected static final Logger logger = LogManager.getLogger(); @@ -173,9 +171,21 @@ private Verdict getVerdictWithLlm(State state) { String llmResponse = getResponseFromLlm(conversationJson); - LlmVerdict llmVerdict = new Gson().fromJson(llmResponse, LlmVerdict.class); - - if(llmVerdict.match()) return new Verdict(Verdict.Severity.LLM_COMPLETE, llmVerdict.getInfo()); + LlmVerdict llmVerdict = LlmVerdictParser.parse(llmResponse); + String info = llmVerdict.getInfo() == null ? "" : llmVerdict.getInfo(); + + switch (llmVerdict.getDecision()) { + case COMPLETED: + return new Verdict(Verdict.Severity.LLM_COMPLETE, info); + case INVALID: + return new Verdict(Verdict.Severity.LLM_INVALID, info); + case CONTINUE: + return Verdict.OK; + case UNKNOWN: + default: + logger.log(Level.WARN, "LLM oracle response did not include a recognized verdict status/match."); + return Verdict.OK; + } } catch(Exception e) { logger.log(Level.ERROR, "Error obtaining the verdict with the LLM"); diff --git a/testar/src/org/testar/oracles/llm/LlmVerdict.java b/testar/src/org/testar/oracles/llm/LlmVerdict.java index 8010e7cff..694b36157 100644 --- a/testar/src/org/testar/oracles/llm/LlmVerdict.java +++ b/testar/src/org/testar/oracles/llm/LlmVerdict.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,10 +32,16 @@ public class LlmVerdict { private Boolean match; + private String status; private String info; public LlmVerdict(Boolean match, String info) { + this(match, "", info); + } + + public LlmVerdict(Boolean match, String status, String info) { this.match = match; + this.status = status; this.info = info; } @@ -43,7 +49,27 @@ public Boolean match() { return match; } + public String getStatus() { + return status; + } + public String getInfo() { return info; } + + public LlmVerdictDecision getDecision() { + LlmVerdictDecision fromStatus = LlmVerdictDecision.fromStatus(status); + // Return a (CONTINUE | COMPLETED | INVALID) detected LLM Oracle decision + if (fromStatus != LlmVerdictDecision.UNKNOWN) { + return fromStatus; + } + // Return a true/false detected LLM Oracle decision + if (Boolean.TRUE.equals(match)) { + return LlmVerdictDecision.COMPLETED; + } + if (Boolean.FALSE.equals(match)) { + return LlmVerdictDecision.CONTINUE; + } + return LlmVerdictDecision.UNKNOWN; + } } diff --git a/testar/src/org/testar/oracles/llm/LlmVerdictDecision.java b/testar/src/org/testar/oracles/llm/LlmVerdictDecision.java new file mode 100644 index 000000000..0a1890b5c --- /dev/null +++ b/testar/src/org/testar/oracles/llm/LlmVerdictDecision.java @@ -0,0 +1,58 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.e + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.oracles.llm; + +import java.util.Locale; + +public enum LlmVerdictDecision { + CONTINUE, + COMPLETED, + INVALID, + UNKNOWN; + + public static LlmVerdictDecision fromStatus(String status) { + if (status == null || status.trim().isEmpty()) { + return UNKNOWN; + } + + String normalized = status.trim().toUpperCase(Locale.ROOT); + switch (normalized) { + case "CONTINUE": + return CONTINUE; + case "COMPLETED": + return COMPLETED; + case "INVALID": + return INVALID; + default: + return UNKNOWN; + } + } +} diff --git a/testar/src/org/testar/oracles/llm/LlmVerdictParser.java b/testar/src/org/testar/oracles/llm/LlmVerdictParser.java new file mode 100644 index 000000000..72eae95d8 --- /dev/null +++ b/testar/src/org/testar/oracles/llm/LlmVerdictParser.java @@ -0,0 +1,78 @@ +/*************************************************************************************************** + * + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.e + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + *******************************************************************************************************/ + +package org.testar.oracles.llm; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public final class LlmVerdictParser { + + private LlmVerdictParser() {} + + public static LlmVerdict parse(String llmResponse) { + JsonObject object = JsonParser.parseString(llmResponse).getAsJsonObject(); + + String info = getStringValue(object, "info"); + String status = getStringValue(object, "status"); + Boolean match = getBooleanValue(object, "match"); + + return new LlmVerdict(match, status, info); + } + + private static String getStringValue(JsonObject object, String property) { + if (!object.has(property) || object.get(property).isJsonNull()) { + return ""; + } + JsonElement value = object.get(property); + if (value.isJsonPrimitive()) { + return value.getAsString(); + } + return value.toString(); + } + + private static Boolean getBooleanValue(JsonObject object, String property) { + if (!object.has(property) || object.get(property).isJsonNull()) { + return null; + } + + JsonElement value = object.get(property); + try { + return value.getAsBoolean(); + } catch (Exception ignored) { + try { + return Boolean.parseBoolean(value.getAsString()); + } catch (Exception ignoredAgain) { + return null; + } + } + } +} diff --git a/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java b/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java index 329413e5d..80f8c17fb 100644 --- a/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java +++ b/testar/src/org/testar/statemodel/analysis/condition/StateCondition.java @@ -30,6 +30,8 @@ package org.testar.statemodel.analysis.condition; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.openqa.selenium.InvalidArgumentException; import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Tag; @@ -42,12 +44,14 @@ * Simple condition that searches for a string in all states. */ public class StateCondition extends TestCondition { + static final Logger logger = LogManager.getLogger(); + private final String field; private final String searchMessage; /** * Creates a new StateCondition. - * @param field The field in the state model to serach. Example: WebInnerHTML. + * @param field The field in the state model to search. Example: WebInnerHTML. * @param searchMessage The string to search for. * @param comparator The result of the query is compared to the threshold value using the selected operator. * @param threshold The result of the query is compared to this value. @@ -115,12 +119,14 @@ public boolean evaluate(String modelIdentifier, StateModelManager stateModelMana public boolean evaluate(State state) { for(Widget widget : state) { // For web apps check the widget is visible - if(widget.get(WdTags.WebIsFullOnScreen, true)) { + // The default value is true to avoid blocking SUT systems other than web apps + if(widget.get(WdTags.WebIsFullOnScreen, true) && widget.get(WdTags.WebIsDisplayed, true)) { for(Tag tag : widget.tags()){ if(tag.name().equals(getField()) && widget.get(tag, null) != null){ try { String tagValue = widget.get(tag).toString(); if(tagValue.contains(searchMessage)) { + logger.info("State Condition Match for Tag Value: " + tagValue); return true; } } catch (Exception e) { diff --git a/testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java b/testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java new file mode 100644 index 000000000..3433e5ca9 --- /dev/null +++ b/testar/test/org/testar/llm/TestLlmTestGoalOrchestrator.java @@ -0,0 +1,101 @@ +package org.testar.llm; + +import org.junit.Test; +import org.testar.monkey.alayer.Verdict; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestLlmTestGoalOrchestrator { + + @Test + public void testGoalOrchestratorStartSequence() { + List activatedGoals = new ArrayList<>(); + AtomicBoolean appendPrevious = new AtomicBoolean(true); + + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Arrays.asList(goal("goal-1"), goal("goal-2")), + (goal, append) -> { + activatedGoals.add(goal.getTestGoal()); + appendPrevious.set(append); + } + ); + + orchestrator.startSequence(); + + assertEquals(1, activatedGoals.size()); + assertEquals("goal-1", activatedGoals.get(0)); + assertFalse(appendPrevious.get()); + } + + @Test + public void testGoalOrchestratorCompletionAdvancesToNextGoal() { + AtomicInteger activations = new AtomicInteger(0); + AtomicBoolean appendPrevious = new AtomicBoolean(false); + + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Arrays.asList(goal("goal-1"), goal("goal-2")), + (goal, append) -> { + activations.incrementAndGet(); + appendPrevious.set(append); + } + ); + orchestrator.startSequence(); + + List stopVerdicts = orchestrator.processGoalVerdicts( + Collections.singletonList(new Verdict(Verdict.Severity.LLM_COMPLETE, "done")) + ); + + assertTrue(stopVerdicts.isEmpty()); + assertEquals(2, activations.get()); + assertTrue(appendPrevious.get()); + assertEquals("goal-2", orchestrator.getCurrentGoal().getTestGoal()); + } + + @Test + public void testGoalOrchestratorCompletionFinalGoalStops() { + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Collections.singletonList(goal("goal-1")), + (goal, append) -> {} + ); + orchestrator.startSequence(); + + List stopVerdicts = orchestrator.processGoalVerdicts( + Collections.singletonList(new Verdict(Verdict.Severity.CONDITION_COMPLETE, "all goals done")) + ); + + assertNotNull(stopVerdicts); + assertEquals(1, stopVerdicts.size()); + assertEquals(Verdict.Severity.CONDITION_COMPLETE.getValue(), stopVerdicts.get(0).severity(), 0.0); + } + + @Test + public void testGoalOrchestratorInvalidVerdict() { + LlmTestGoalOrchestrator orchestrator = new LlmTestGoalOrchestrator( + Collections.singletonList(goal("goal-1")), + (goal, append) -> {} + ); + orchestrator.startSequence(); + + List stopVerdicts = orchestrator.processGoalVerdicts( + Collections.singletonList(new Verdict(Verdict.Severity.LLM_INVALID, "invalid behavior")) + ); + + assertNotNull(stopVerdicts); + assertEquals(1, stopVerdicts.size()); + assertEquals(Verdict.Severity.LLM_INVALID.getValue(), stopVerdicts.get(0).severity(), 0.0); + } + + private static LlmTestGoal goal(String value) { + return new LlmTestGoal(value, Collections.emptyList()); + } +} diff --git a/testar/test/org/testar/monkey/VerdictProcessingTest.java b/testar/test/org/testar/monkey/VerdictProcessingTest.java index 9e976a1ba..85a8e51fe 100644 --- a/testar/test/org/testar/monkey/VerdictProcessingTest.java +++ b/testar/test/org/testar/monkey/VerdictProcessingTest.java @@ -101,6 +101,25 @@ public void testDoesNotIgnoreDuplicateLlmComplete() throws Exception { assertEquals(Verdict.Severity.LLM_COMPLETE.getValue(), filtered.get(0).severity(), 0.0); } + @Test + public void testDoesNotIgnoreDuplicateLlmInvalid() throws Exception { + File settingsDir = tempFolder.newFolder("settings_llm_invalid"); + Settings.setSettingsPath(settingsDir.getAbsolutePath()); + + File ignoreFile = new File(settingsDir, "list_of_verdicts_with_failures.txt"); + Files.write(ignoreFile.toPath(), Collections.singletonList("goal invalid by llm"), StandardCharsets.UTF_8); + + Settings settings = new Settings(); + settings.set(ConfigTags.IgnoreDuplicatedVerdicts, true); + VerdictProcessing processing = new VerdictProcessing(settings); + + Verdict llmInvalid = new Verdict(Verdict.Severity.LLM_INVALID, "goal invalid by llm"); + List filtered = processing.filterDuplicates(Collections.singletonList(llmInvalid)); + + assertEquals(1, filtered.size()); + assertEquals(Verdict.Severity.LLM_INVALID.getValue(), filtered.get(0).severity(), 0.0); + } + @Test public void testDoesNotIgnoreDuplicateConditionComplete() throws Exception { File settingsDir = tempFolder.newFolder("settings_condition_complete"); diff --git a/testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java b/testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java new file mode 100644 index 000000000..731adcdad --- /dev/null +++ b/testar/test/org/testar/oracles/llm/TestLlmVerdictParser.java @@ -0,0 +1,99 @@ +package org.testar.oracles.llm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +public class TestLlmVerdictParser { + + @Test + public void testParseStatusCompleted() { + String response = "{\"status\":\"COMPLETED\",\"info\":\"objective reached\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.COMPLETED, verdict.getDecision()); + assertEquals("objective reached", verdict.getInfo()); + assertNull(verdict.match()); + } + + @Test + public void testParseStatusInvalid() { + String response = "{\"status\":\"INVALID\",\"info\":\"unexpected final state\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.INVALID, verdict.getDecision()); + assertEquals("unexpected final state", verdict.getInfo()); + } + + @Test + public void testParseMatchTrue() { + String response = "{\"match\":true,\"info\":\"done\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.COMPLETED, verdict.getDecision()); + assertEquals("done", verdict.getInfo()); + } + + @Test + public void testParseMatchFalse() { + String response = "{\"match\":\"false\",\"info\":\"continue\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.CONTINUE, verdict.getDecision()); + assertEquals("continue", verdict.getInfo()); + } + + @Test + public void testParseUnknownFormatResponse() { + String response = "{\"verdict\":\"completed\",\"info\":\"alias field\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("alias field", verdict.getInfo()); + } + + @Test + public void testParseUnknownStatusDecision() { + String response = "{\"status\":\"MAYBE\",\"info\":\"not recognized\"}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("not recognized", verdict.getInfo()); + } + + @Test + public void testParseMissingJsonFields() { + String response = "{}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("", verdict.getInfo()); + assertNull(verdict.match()); + } + + @Test + public void testParseIncorrectJsonFormat() { + String response = "{\"status\":{},\"info\":null,\"match\":{}}"; + LlmVerdict verdict = LlmVerdictParser.parse(response); + + assertEquals(LlmVerdictDecision.UNKNOWN, verdict.getDecision()); + assertEquals("", verdict.getInfo()); + assertNull(verdict.match()); + } + + @Test(expected = NullPointerException.class) + public void testParseNullInputThrowsException() { + LlmVerdictParser.parse(null); + } + + @Test(expected = IllegalStateException.class) + public void testParseInvalidJsonThrowsException() { + LlmVerdictParser.parse("not-a-json-object"); + } + + @Test(expected = IllegalStateException.class) + public void testParseArrayJsonThrowsException() { + LlmVerdictParser.parse("[]"); + } +} diff --git a/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java b/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java index 63bef253c..81cddc538 100644 --- a/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java +++ b/testar/test/org/testar/statemodel/analysis/condition/TestCheckConditionEvaluator.java @@ -76,6 +76,21 @@ public void test_single_check_condition_unmatch_nonvisible() { Assert.isTrue(!checkEvaluator.evaluateConditions(state), "The state evaluator must unmatch for non visible widgets"); } + @Test + public void test_single_check_condition_unmatch_nondisplayed() { + StateStub state = new StateStub(); + WidgetStub widget = new WidgetStub(); + widget.set(WdTags.WebTitle, "this message is shown but the widget is non displayed"); + widget.set(WdTags.WebIsDisplayed, false); + state.addChild(widget); + widget.setParent(state); + + String testGoal = "Perform this action \n Check: this message is shown"; + CheckConditionEvaluator checkEvaluator = new CheckConditionEvaluator(WdTags.WebTitle, testGoal); + + Assert.isTrue(!checkEvaluator.evaluateConditions(state), "The state evaluator must unmatch for non displayed widgets"); + } + @Test public void test_multiple_check_conditions_loaded_messages() { String testGoal = "Perform this action \n Check: this message is shown \n Check: also a second message"; diff --git a/webdriver/resources/web-extension/js/testar.state.js b/webdriver/resources/web-extension/js/testar.state.js index aa00eab5a..078820916 100644 --- a/webdriver/resources/web-extension/js/testar.state.js +++ b/webdriver/resources/web-extension/js/testar.state.js @@ -381,8 +381,9 @@ function getIsBlockedTestar(element, xOffset, yOffset, clientRect) { return false; } - // Ignore encapsulated childs of - if (element.tagName === "A" && element.contains(elem)) { + // If hit-testing returns the element itself or one of its descendants, + // then it is not blocked (e.g. ). + if (elem === element || element.contains(elem)) { return false; } @@ -391,9 +392,8 @@ function getIsBlockedTestar(element, xOffset, yOffset, clientRect) { return false; } - // return whether obscured element has same parent node - // (will also return false if element === elem) - return elem.parentNode !== element.parentNode; + // Otherwise, treat as blocked by another element. + return true; } /* diff --git a/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java b/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java index f055a246c..fb51626a2 100644 --- a/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java +++ b/webdriver/src/org/testar/monkey/alayer/webdriver/enums/WdMapping.java @@ -51,6 +51,7 @@ public class WdMapping { put(WebWidgetName ,WdTags.WebName); put(WebWidgetTagName, WdTags.WebTagName); put(WebWidgetTextContent, WdTags.WebTextContent); + put(WebWidgetInnerText, WdTags.WebInnerText); put(WebWidgetTitle, WdTags.WebTitle); put(WebWidgetHref, WdTags.WebHref); put(WebWidgetValue, WdTags.WebValue); diff --git a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java index 2329d4eeb..2f1146741 100644 --- a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java +++ b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWdTagMapping.java @@ -1,11 +1,17 @@ package org.testar.monkey.alayer.webdriver; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import org.junit.Before; import org.junit.Test; +import org.openqa.selenium.remote.RemoteWebElement; +import org.testar.monkey.alayer.Rect; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import java.util.Arrays; +import java.util.HashMap; + public class TestWdTagMapping { private WdRootElement rootWdElement; @@ -28,6 +34,289 @@ public void test_WebAriaLabel_mapping() { assertEquals("Search", childWdWidget.get(WdTags.WebAriaLabel, "")); } + @Test + public void test_WebId_mapping() { + childWdElement.id = "id-123"; + assertEquals("id-123", childWdWidget.get(WdTags.WebId, "")); + } + + @Test + public void test_WebHref_mapping() { + childWdElement.href = "https://testar.org"; + assertEquals("https://testar.org", childWdWidget.get(WdTags.WebHref, "")); + } + + @Test + public void test_WebPlaceholder_mapping() { + childWdElement.placeholder = "Type here"; + assertEquals("Type here", childWdWidget.get(WdTags.WebPlaceholder, "")); + } + + @Test + public void test_WebIsDisabled_mapping() { + childWdElement.disabled = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsDisabled, null)); + } + + @Test + public void test_WebName_mapping() { + childWdElement.name = "search-input"; + assertEquals("search-input", childWdWidget.get(WdTags.WebName, "")); + } + + @Test + public void test_WebTagName_mapping() { + childWdElement.tagName = "input"; + assertEquals("input", childWdWidget.get(WdTags.WebTagName, "")); + } + + @Test + public void test_WebTextContent_mapping() { + childWdElement.textContent = "Search text"; + assertEquals("Search text", childWdWidget.get(WdTags.WebTextContent, "")); + } + + @Test + public void test_WebInnerText_mapping() { + childWdElement.innerText = "Visible text"; + assertEquals("Visible text", childWdWidget.get(WdTags.WebInnerText, "")); + } + + @Test + public void test_WebTitle_mapping() { + childWdElement.title = "Search"; + assertEquals("Search", childWdWidget.get(WdTags.WebTitle, "")); + } + + @Test + public void test_WebValue_mapping() { + childWdElement.value = "hello"; + assertEquals("hello", childWdWidget.get(WdTags.WebValue, "")); + } + + @Test + public void test_WebType_mapping() { + childWdElement.type = "text"; + assertEquals("text", childWdWidget.get(WdTags.WebType, "")); + } + + @Test + public void test_WebSrc_mapping() { + childWdElement.src = "https://testar.org/image.png"; + assertEquals("https://testar.org/image.png", childWdWidget.get(WdTags.WebSrc, "")); + } + + @Test + public void test_WebXPath_mapping() { + childWdElement.xpath = "/html/body/div[1]"; + assertEquals("/html/body/div[1]", childWdWidget.get(WdTags.WebXPath, "")); + } + + @Test + public void test_WebCssClasses_mapping() { + childWdElement.cssClasses = Arrays.asList("btn", "primary"); + assertEquals("[btn, primary]", childWdWidget.get(WdTags.WebCssClasses, "")); + } + + @Test + public void test_WebIsEnabled_mapping() { + childWdElement.enabled = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsEnabled, null)); + } + + @Test + public void test_WebIsBlocked_mapping() { + childWdElement.blocked = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsBlocked, null)); + } + + @Test + public void test_WebIsClickable_mapping() { + childWdElement.isClickable = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsClickable, null)); + } + + @Test + public void test_WebIsShadow_mapping() { + childWdElement.isShadow = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsShadow, null)); + } + + @Test + public void test_WebHasKeyboardFocus_mapping() { + childWdElement.hasKeyboardFocus = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebHasKeyboardFocus, null)); + } + + @Test + public void test_WebIsKeyboardFocusable_mapping() { + childWdElement.isKeyboardFocusable = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsKeyboardFocusable, null)); + } + + @Test + public void test_WebIsContentElement_mapping() { + childWdElement.isContentElement = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsContentElement, null)); + } + + @Test + public void test_WebIsControlElement_mapping() { + childWdElement.isControlElement = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsControlElement, null)); + } + + @Test + public void test_WebStyle_mapping() { + childWdElement.style = "color:red;"; + assertEquals("color:red;", childWdWidget.get(WdTags.WebStyle, "")); + } + + @Test + public void test_WebStyleOpacity_mapping() { + childWdElement.styleOpacity = 0.7; + assertEquals(Double.valueOf(0.7), childWdWidget.get(WdTags.WebStyleOpacity, null)); + } + + @Test + public void test_WebStyleOverflow_mapping() { + childWdElement.styleOverflow = "auto"; + assertEquals("auto", childWdWidget.get(WdTags.WebStyleOverflow, "")); + } + + @Test + public void test_WebStyleOverflowX_mapping() { + childWdElement.styleOverflowX = "scroll"; + assertEquals("scroll", childWdWidget.get(WdTags.WebStyleOverflowX, "")); + } + + @Test + public void test_WebStyleOverflowY_mapping() { + childWdElement.styleOverflowY = "hidden"; + assertEquals("hidden", childWdWidget.get(WdTags.WebStyleOverflowY, "")); + } + + @Test + public void test_WebStylePosition_mapping() { + childWdElement.stylePosition = "relative"; + assertEquals("relative", childWdWidget.get(WdTags.WebStylePosition, "")); + } + + @Test + public void test_WebZIndex_mapping() { + childWdElement.zindex = 10.0; + assertEquals(Double.valueOf(10.0), childWdWidget.get(WdTags.WebZIndex, null)); + } + + @Test + public void test_WebBoundingRectangle_mapping() { + Rect rect = Rect.from(1, 2, 30, 40); + childWdElement.rect = rect; + assertEquals(rect, childWdWidget.get(WdTags.WebBoundingRectangle, null)); + } + + @Test + public void test_WebNaturalWidth_mapping() { + childWdElement.naturalWidth = 300; + assertEquals(Long.valueOf(300), childWdWidget.get(WdTags.WebNaturalWidth, null)); + } + + @Test + public void test_WebNaturalHeight_mapping() { + childWdElement.naturalHeight = 200; + assertEquals(Long.valueOf(200), childWdWidget.get(WdTags.WebNaturalHeight, null)); + } + + @Test + public void test_WebDisplayedWidth_mapping() { + childWdElement.displayedWidth = 120; + assertEquals(Long.valueOf(120), childWdWidget.get(WdTags.WebDisplayedWidth, null)); + } + + @Test + public void test_WebDisplayedHeight_mapping() { + childWdElement.displayedHeight = 80; + assertEquals(Long.valueOf(80), childWdWidget.get(WdTags.WebDisplayedHeight, null)); + } + + @Test + public void test_WebMaxLength_mapping() { + childWdElement.maxLength = 255; + assertEquals(Integer.valueOf(255), childWdWidget.get(WdTags.WebMaxLength, null)); + } + + @Test + public void test_WebInnerHTML_mapping() { + childWdElement.innerHTML = "hello"; + assertEquals("hello", childWdWidget.get(WdTags.WebInnerHTML, "")); + } + + @Test + public void test_WebOuterHTML_mapping() { + childWdElement.outerHTML = "
hello
"; + assertEquals("
hello
", childWdWidget.get(WdTags.WebOuterHTML, "")); + } + + @Test + public void test_WebComputedFontSize_mapping() { + childWdElement.computedFontSize = "16px"; + assertEquals("16px", childWdWidget.get(WdTags.WebComputedFontSize, "")); + } + + @Test + public void test_WebScrollPattern_mapping() { + childWdElement.scrollPattern = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebScrollPattern, null)); + } + + @Test + public void test_WebHorizontallyScrollable_mapping() { + childWdElement.hScroll = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebHorizontallyScrollable, null)); + } + + @Test + public void test_WebVerticallyScrollable_mapping() { + childWdElement.vScroll = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebVerticallyScrollable, null)); + } + + @Test + public void test_WebScrollHorizontalViewSize_mapping() { + childWdElement.hScrollViewSize = 33.3; + assertEquals(Double.valueOf(33.3), childWdWidget.get(WdTags.WebScrollHorizontalViewSize, null)); + } + + @Test + public void test_WebScrollVerticalViewSize_mapping() { + childWdElement.vScrollViewSize = 44.4; + assertEquals(Double.valueOf(44.4), childWdWidget.get(WdTags.WebScrollVerticalViewSize, null)); + } + + @Test + public void test_WebScrollHorizontalPercent_mapping() { + childWdElement.hScrollPercent = 55.5; + assertEquals(Double.valueOf(55.5), childWdWidget.get(WdTags.WebScrollHorizontalPercent, null)); + } + + @Test + public void test_WebScrollVerticalPercent_mapping() { + childWdElement.vScrollPercent = 66.6; + assertEquals(Double.valueOf(66.6), childWdWidget.get(WdTags.WebScrollVerticalPercent, null)); + } + + @Test + public void test_WebIsOffScreen_mapping() { + childWdElement.isFullVisibleOnScreen = false; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsOffScreen, null)); + } + + @Test + public void test_WebIsHidden_mapping() { + childWdElement.visibility = "hidden"; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsHidden, null)); + } + @Test public void test_WebAriaLabelledBy_mapping() { childWdElement.ariaLabelledBy = "search-label"; @@ -160,6 +449,62 @@ public void test_WebAriaModal_mapping() { assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebAriaModal, null)); } + @Test + public void test_WebTarget_mapping() { + childWdElement.target = "_blank"; + assertEquals("_blank", childWdWidget.get(WdTags.WebTarget, "")); + } + + @Test + public void test_WebAlt_mapping() { + childWdElement.alt = "banner-image"; + assertEquals("banner-image", childWdWidget.get(WdTags.WebAlt, "")); + } + + @Test + public void test_WebAccessKey_mapping() { + childWdElement.accessKey = "k"; + assertEquals("k", childWdWidget.get(WdTags.WebAccessKey, "")); + } + + @Test + public void test_WebAcceleratorKey_mapping() { + childWdElement.acceleratorKey = "Ctrl+K"; + assertEquals("Ctrl+K", childWdWidget.get(WdTags.WebAcceleratorKey, "")); + } + + @Test + public void test_WebElementSelenium_mapping() { + RemoteWebElement remoteWebElement = new RemoteWebElement(); + childWdElement.remoteWebElement = remoteWebElement; + assertSame(remoteWebElement, childWdWidget.get(WdTags.WebElementSelenium, null)); + } + + @Test + public void test_WebAttributeMap_mapping() { + childWdElement.attributeMap = new HashMap<>(); + childWdElement.attributeMap.put("aria-label", "Search"); + assertEquals(childWdElement.attributeMap, childWdWidget.get(WdTags.WebAttributeMap, null)); + } + + @Test + public void test_WebIsFullOnScreen_mapping() { + childWdElement.isFullVisibleOnScreen = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsFullOnScreen, null)); + } + + @Test + public void test_WebGenericTitle_mapping() { + childWdElement.genericTitle = "generic-title"; + assertEquals("generic-title", childWdWidget.get(WdTags.WebGenericTitle, "")); + } + + @Test + public void test_WebIsWindowModal_mapping() { + childWdElement.isModal = true; + assertEquals(Boolean.TRUE, childWdWidget.get(WdTags.WebIsWindowModal, null)); + } + @Test public void test_NullDefaultValue_string() { childWdElement.ariaLabel = null; diff --git a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java index f187e78be..61e24f04e 100644 --- a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java +++ b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverStateManagementTag.java @@ -11,50 +11,178 @@ public class TestWebdriverStateManagementTag { @Test - public void testWebdriverMapping() { - WdState wdState = new WdState(null); - wdState.set(WdTags.WebId, "UniqueId"); - wdState.set(WdTags.WebHref, "Link.org"); - - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetId), "UniqueId"); - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetHref), "Link.org"); - } - - @Test - public void testWebdriverAriaMapping() { - WdState wdState = new WdState(null); - wdState.set(WdTags.WebAriaLabel, "this is an aria label"); - wdState.set(WdTags.WebAriaLabelledBy, "and this an aria labelled by"); - - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetAriaLabel), "this is an aria label"); - Assert.assertEquals(wdState.get(StateManagementTags.WebWidgetAriaLabelledBy), "and this an aria labelled by"); - } - - @Test - public void testWebdriverCodingIDs() { + public void test_WebdriverCodingIDs_mapping() { Tag[] abstractTags = new Tag[]{StateManagementTags.WebWidgetId, StateManagementTags.WebWidgetHref}; CodingManager.setCustomTagsForAbstractId(abstractTags); WdState wdState = new WdState(null); WdWidget wdWidget = new WdWidget(wdState, wdState, null); - wdWidget.set(WdTags.WebId, "UniqueId"); - wdWidget.set(WdTags.WebHref, "Link.org"); // Build the first AbstractID and check the StateManagementTags uses the Wd values + wdWidget.set(WdTags.WebId, "UniqueId"); + wdWidget.set(WdTags.WebHref, "Link.org"); CodingManager.buildIDs(wdWidget); - Assert.assertEquals(wdWidget.get(Tags.AbstractID), "WA1r9aad8101406797296"); + String originalAbstractId = wdWidget.get(Tags.AbstractID); + Assert.assertTrue(originalAbstractId.contains("WA")); // Change WebId value to verify the AbstractID changes wdWidget.set(WdTags.WebId, "UniqueIdNEW"); wdWidget.set(WdTags.WebHref, "Link.org"); CodingManager.buildIDs(wdWidget); - Assert.assertEquals(wdWidget.get(Tags.AbstractID), "WA1vaf6dg131408350915"); + String webIdChangedAbstractId = wdWidget.get(Tags.AbstractID); + Assert.assertTrue(webIdChangedAbstractId.contains("WA")); + Assert.assertNotEquals(originalAbstractId, webIdChangedAbstractId); // Change WebHref value to verify the AbstractID changes wdWidget.set(WdTags.WebId, "UniqueIdNEW"); wdWidget.set(WdTags.WebHref, "Link.com"); CodingManager.buildIDs(wdWidget); - Assert.assertEquals(wdWidget.get(Tags.AbstractID), "WA1gxbnlz131612186923"); + String webHrefChangedAbstractId = wdWidget.get(Tags.AbstractID); + Assert.assertTrue(webHrefChangedAbstractId.contains("WA")); + Assert.assertNotEquals(originalAbstractId, webHrefChangedAbstractId); + Assert.assertNotEquals(webIdChangedAbstractId, webHrefChangedAbstractId); + } + + @Test + public void test_WebId_mapping() { + assertStringMapping(WdTags.WebId, StateManagementTags.WebWidgetId, "UniqueId"); + } + + @Test + public void test_WebName_mapping() { + assertStringMapping(WdTags.WebName, StateManagementTags.WebWidgetName, "search-input"); + } + + @Test + public void test_WebTagName_mapping() { + assertStringMapping(WdTags.WebTagName, StateManagementTags.WebWidgetTagName, "input"); + } + + @Test + public void test_WebTextContent_mapping() { + assertStringMapping(WdTags.WebTextContent, StateManagementTags.WebWidgetTextContent, "Text content"); + } + + @Test + public void test_WebInnerText_mapping() { + assertStringMapping(WdTags.WebInnerText, StateManagementTags.WebWidgetInnerText, "Visible text"); + } + + @Test + public void test_WebTitle_mapping() { + assertStringMapping(WdTags.WebTitle, StateManagementTags.WebWidgetTitle, "Widget title"); + } + + @Test + public void test_WebHref_mapping() { + assertStringMapping(WdTags.WebHref, StateManagementTags.WebWidgetHref, "Link.org"); + } + + @Test + public void test_WebValue_mapping() { + assertStringMapping(WdTags.WebValue, StateManagementTags.WebWidgetValue, "hello"); + } + + @Test + public void test_WebStyle_mapping() { + assertStringMapping(WdTags.WebStyle, StateManagementTags.WebWidgetStyle, "color:red;"); + } + + @Test + public void test_WebTarget_mapping() { + assertStringMapping(WdTags.WebTarget, StateManagementTags.WebWidgetTarget, "_blank"); + } + + @Test + public void test_WebAlt_mapping() { + assertStringMapping(WdTags.WebAlt, StateManagementTags.WebWidgetAlt, "banner-image"); + } + + @Test + public void test_WebType_mapping() { + assertStringMapping(WdTags.WebType, StateManagementTags.WebWidgetType, "text"); + } + + @Test + public void test_WebCssClasses_mapping() { + assertStringMapping(WdTags.WebCssClasses, StateManagementTags.WebWidgetCssClasses, "[btn, primary]"); + } + + @Test + public void test_WebDisplay_mapping() { + assertStringMapping(WdTags.WebDisplay, StateManagementTags.WebWidgetDisplay, "block"); + } + + @Test + public void test_WebSrc_mapping() { + assertStringMapping(WdTags.WebSrc, StateManagementTags.WebWidgetSrc, "https://testar.org/image.png"); + } + + @Test + public void test_WebPlaceholder_mapping() { + assertStringMapping(WdTags.WebPlaceholder, StateManagementTags.WebWidgetPlaceholder, "Type here"); + } + + @Test + public void test_WebIsOffScreen_mapping() { + assertBooleanMapping(WdTags.WebIsOffScreen, StateManagementTags.WebWidgetIsOffScreen, Boolean.TRUE); + } + + @Test + public void test_WebIsDisabled_mapping() { + assertBooleanMapping(WdTags.WebIsDisabled, StateManagementTags.WebWidgetIsDisabled, Boolean.TRUE); + } + + @Test + public void test_WebAriaLabel_mapping() { + assertStringMapping(WdTags.WebAriaLabel, StateManagementTags.WebWidgetAriaLabel, "aria-label"); + } + + @Test + public void test_WebAriaLabelledBy_mapping() { + assertStringMapping(WdTags.WebAriaLabelledBy, StateManagementTags.WebWidgetAriaLabelledBy, "aria labelled by"); + } + + @Test + public void test_WidgetControlType_mapping() { + assertStringMapping(WdTags.WebTagName, StateManagementTags.WidgetControlType, "button"); + } + + @Test + public void test_WidgetTitle_mapping() { + assertStringMapping(WdTags.WebGenericTitle, StateManagementTags.WidgetTitle, "Generic title"); + } + + @Test + public void test_WidgetIsEnabled_mapping() { + assertBooleanMapping(WdTags.WebIsEnabled, StateManagementTags.WidgetIsEnabled, Boolean.TRUE); + } + + @Test + public void test_WidgetPath_mapping() { + assertStringMapping(Tags.Path, StateManagementTags.WidgetPath, "/html/body/div[1]"); + } + + @Test + public void test_WidgetIsContentElement_mapping() { + assertBooleanMapping(WdTags.WebIsContentElement, StateManagementTags.WidgetIsContentElement, Boolean.TRUE); + } + + @Test + public void test_WidgetIsControlElement_mapping() { + assertBooleanMapping(WdTags.WebIsControlElement, StateManagementTags.WidgetIsControlElement, Boolean.TRUE); + } + + private void assertStringMapping(Tag wdTag, Tag stateTag, String value) { + WdState wdState = new WdState(null); + wdState.set(wdTag, value); + Assert.assertEquals(value, wdState.get(stateTag)); + } + + private void assertBooleanMapping(Tag wdTag, Tag stateTag, Boolean value) { + WdState wdState = new WdState(null); + wdState.set(wdTag, value); + Assert.assertEquals(value, wdState.get(stateTag)); } }