From 92ed274f71a4bfa1bc284aecd077daf61015de0b Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 17 Jun 2026 10:43:52 +0200 Subject: [PATCH 1/5] fix: removed several unused RPCs from PilotManager --- .../Agent/PilotStatusAgent.py | 7 +- .../DB/PilotAgentsDB.py | 71 +++-------- .../DB/PilotAgentsDB.sql | 10 -- .../Service/PilotManagerHandler.py | 115 ------------------ .../Test_PilotsClient.py | 32 ----- 5 files changed, 19 insertions(+), 216 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py index f4e3b9a7d91..2a3b2ecb7b8 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py @@ -1,4 +1,4 @@ -""" The Pilot Status Agent updates the status of the pilot jobs in the +"""The Pilot Status Agent updates the status of the pilot jobs in the PilotAgents database. .. literalinclude:: ../ConfigTemplate.cfg @@ -7,6 +7,7 @@ :dedent: 2 :caption: PilotStatusAgent options """ + import datetime from DIRAC import S_OK @@ -16,7 +17,6 @@ from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Utilities import TimeUtilities from DIRAC.WorkloadManagementSystem.Client import PilotStatus -from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB from DIRAC.WorkloadManagementSystem.Service.WMSUtilities import killPilotsInQueues @@ -52,7 +52,6 @@ def initialize(self): self.jobDB = JobDB() self.clearPilotsDelay = self.am_getOption("ClearPilotsDelay", 30) self.clearAbortedDelay = self.am_getOption("ClearAbortedPilotsDelay", 7) - self.pilots = PilotManagerClient() return S_OK() @@ -70,7 +69,7 @@ def execute(self): # Now handle pilots not updated in the last N days and declare them Deleted. result = self.handleOldPilots(connection) - result = self.pilots.clearPilots(self.clearPilotsDelay, self.clearAbortedDelay) + result = self.pilotDB.clearPilots(self.clearPilotsDelay, self.clearAbortedDelay) if not result["OK"]: self.log.warn("Failed to clear old pilots in the PilotAgentsDB") diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py index 6cce6240d89..1c525bf0b49 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py @@ -1,22 +1,21 @@ -""" PilotAgentsDB class is a front-end to the Pilot Agent Database. - This database keeps track of all the submitted grid pilot jobs. - It also registers the mapping of the DIRAC jobs to the pilot - agents. - - Available methods are: - - addPilotReferences() - setPilotStatus() - deletePilot() - clearPilots() - setPilotDestinationSite() - storePilotOutput() - getPilotOutput() - setJobForPilot() - getPilotsSummary() - getGroupedPilotSummary() +"""PilotAgentsDB class is a front-end to the Pilot Agent Database. +This database keeps track of all the submitted grid pilot jobs. +It also registers the mapping of the DIRAC jobs to the pilot +agents. + +Available methods are: + +addPilotReferences() +setPilotStatus() +deletePilot() +clearPilots() +setPilotDestinationSite() +setJobForPilot() +getPilotsSummary() +getGroupedPilotSummary() """ + import datetime import decimal import threading @@ -403,44 +402,6 @@ def setAccountingFlag(self, pilotRef, mark="True"): ) return self._update(req, args=args) - ########################################################################################## - def storePilotOutput(self, pilotRef, output, error): - """Store standard output and error for a pilot with pilotRef""" - pilotID = self.__getPilotID(pilotRef) - if not pilotID: - return S_ERROR(f"Pilot reference not found {pilotRef}") - - req = "INSERT INTO PilotOutput (PilotID,StdOutput,StdError) VALUES (%s, %s, %s)" - args = ( - pilotID, - output, - error, - ) - return self._update(req, args=args) - - ########################################################################################## - def getPilotOutput(self, pilotRef): - """Retrieve standard output and error for pilot with pilotRef""" - - req = "SELECT StdOutput, StdError FROM PilotOutput,PilotAgents WHERE " - req += "PilotOutput.PilotID = PilotAgents.PilotID AND PilotAgents.PilotJobReference=%s" - result = self._query(req, args=(pilotRef,)) - if not result["OK"]: - return result - if not result["Value"]: - return S_ERROR(f"PilotJobReference {pilotRef} not found") - try: - stdout = result["Value"][0][0].decode() # account for the use of BLOBs - error = result["Value"][0][1].decode() - except AttributeError: - stdout = result["Value"][0][0] - error = result["Value"][0][1] - if stdout == '""': - stdout = "" - if error == '""': - error = "" - return S_OK({"StdOut": stdout, "StdErr": error}) - ########################################################################################## def __getPilotID(self, pilotRef): """Get Pilot ID for the given pilot reference or a list of references""" diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql index 24cbe5a82b8..fa11604afcc 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql @@ -1,5 +1,3 @@ --- $Header: /tmp/libdirac/tmp.stZoy15380/dirac/DIRAC3/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql,v 1.20 2009/08/26 09:39:53 rgracian Exp $ - -- ------------------------------------------------------------------------------ -- -- Schema definition for the PilotAgentsDB database - containing the Pilots status @@ -58,11 +56,3 @@ CREATE TABLE `JobToPilotMapping` ( KEY `JobID` (`JobID`), KEY `PilotID` (`PilotID`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -DROP TABLE IF EXISTS `PilotOutput`; -CREATE TABLE `PilotOutput` ( - `PilotID` INT(11) UNSIGNED NOT NULL, - `StdOutput` MEDIUMTEXT, - `StdError` MEDIUMTEXT, - PRIMARY KEY (`PilotID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py index 964b2012328..5ab2f28b5c8 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py @@ -27,38 +27,6 @@ def initializeHandler(cls, serviceInfoDict): return S_OK() - ############################################################################## - types_getCurrentPilotCounters = [dict] - - @classmethod - def export_getCurrentPilotCounters(cls, attrDict={}): - """Get pilot counters per Status with attrDict selection. Final statuses are given for - the last day. - """ - - result = cls.pilotAgentsDB.getCounters("PilotAgents", ["Status"], attrDict, timeStamp="LastUpdateTime") - if not result["OK"]: - return result - last_update = datetime.datetime.utcnow() - TimeUtilities.day - resultDay = cls.pilotAgentsDB.getCounters( - "PilotAgents", ["Status"], attrDict, newer=last_update, timeStamp="LastUpdateTime" - ) - if not resultDay["OK"]: - return resultDay - - resultDict = {} - for statusDict, count in result["Value"]: - status = statusDict["Status"] - resultDict[status] = count - if status in PilotStatus.PILOT_FINAL_STATES: - resultDict[status] = 0 - for statusDayDict, ccount in resultDay["Value"]: - if status == statusDayDict["Status"]: - resultDict[status] = ccount - break - - return S_OK(resultDict) - ########################################################################################## types_addPilotReferences = [list, str] @@ -101,20 +69,6 @@ def export_getPilotSummary(cls, startdate="", enddate=""): return cls.pilotAgentsDB.getPilotSummary(startdate, enddate) - ############################################################################## - types_getGroupedPilotSummary = [list] - - @classmethod - def export_getGroupedPilotSummary(cls, columnList): - """ - Get pilot summary showing grouped by columns in columnList, all pilot states - and pilot efficiencies in a single row. - - :param columnList: a list of columns to GROUP BY (less status column) - :return: a dictionary containing column names and data records - """ - return cls.pilotAgentsDB.getGroupedPilotSummary(columnList) - ############################################################################## types_getPilots = [[str, int]] @@ -128,40 +82,6 @@ def export_getPilots(cls, jobID): return cls.pilotAgentsDB.getPilotInfo(pilotID=result["Value"]) ############################################################################## - types_setJobForPilot = [[str, int], str] - - @classmethod - def export_setJobForPilot(cls, jobID, pilotRef, destination=None): - """Report the DIRAC job ID which is executed by the given pilot job""" - - result = cls.pilotAgentsDB.setJobForPilot(int(jobID), pilotRef) - if not result["OK"]: - return result - result = cls.pilotAgentsDB.setCurrentJobID(pilotRef, int(jobID)) - if not result["OK"]: - return result - if destination: - result = cls.pilotAgentsDB.setPilotDestinationSite(pilotRef, destination) - - return result - - ########################################################################################## - types_setPilotBenchmark = [str, float] - - @classmethod - def export_setPilotBenchmark(cls, pilotRef, mark): - """Set the pilot agent benchmark""" - return cls.pilotAgentsDB.setPilotBenchmark(pilotRef, mark) - - ########################################################################################## - types_setAccountingFlag = [str] - - @classmethod - def export_setAccountingFlag(cls, pilotRef, mark="True"): - """Set the pilot AccountingSent flag""" - return cls.pilotAgentsDB.setAccountingFlag(pilotRef, mark) - - ########################################################################################## types_setPilotStatus = [str, str] @classmethod @@ -171,38 +91,3 @@ def export_setPilotStatus(cls, pilotRef, status, destination=None, reason=None, return cls.pilotAgentsDB.setPilotStatus( pilotRef, status, destination=destination, statusReason=reason, gridSite=gridSite, queue=queue ) - - ########################################################################################## - types_countPilots = [dict] - - @classmethod - def export_countPilots(cls, condDict, older=None, newer=None, timeStamp="SubmissionTime"): - """Set the pilot agent status""" - - return cls.pilotAgentsDB.countPilots(condDict, older, newer, timeStamp) - - ########################################################################################## - types_deletePilots = [[list, str, int]] - - @classmethod - def export_deletePilots(cls, pilotIDs): - if isinstance(pilotIDs, str): - return cls.pilotAgentsDB.deletePilot(pilotIDs) - - if isinstance(pilotIDs, int): - pilotIDs = [ - pilotIDs, - ] - - result = cls.pilotAgentsDB.deletePilots(pilotIDs) - if not result["OK"]: - return result - - return S_OK() - - ############################################################################## - types_clearPilots = [int, int] - - @classmethod - def export_clearPilots(cls, interval=30, aborted_interval=7): - return cls.pilotAgentsDB.clearPilots(interval, aborted_interval) diff --git a/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py b/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py index b9c293b1dc2..0649b405e98 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py +++ b/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py @@ -25,31 +25,15 @@ def test_PilotsDB(): pilots = PilotManagerClient() webapp = WebAppClient() - # This will allow you to run the test again if necessary - for jobID in ["aPilot", "anotherPilot"]: - pilots.deletePilots(jobID) - res = pilots.addPilotReferences(["aPilot"], "VO") assert res["OK"], res["Message"] - res = pilots.getCurrentPilotCounters({}) - assert res["OK"], res["Message"] - assert "Submitted" in res["Value"] - res = pilots.deletePilots("aPilot") - assert res["OK"], res["Message"] - res = pilots.getCurrentPilotCounters({}) - assert res["OK"], res["Message"] - res = pilots.addPilotReferences(["anotherPilot"], "VO") assert res["OK"], res["Message"] - res = pilots.storePilotOutput("anotherPilot", "This is an output", "this is an error") - assert res["OK"], res["Message"] res = pilots.getPilotInfo("anotherPilot") assert res["OK"], res["Message"] assert res["Value"]["anotherPilot"]["AccountingSent"] == "False" assert res["Value"]["anotherPilot"]["PilotJobReference"] == "anotherPilot" - res = pilots.selectPilots({}) - assert res["OK"], res["Message"] res = pilots.getPilotSummary("", "") assert res["OK"], res["Message"] assert res["Value"]["Total"]["Submitted"] >= 1 @@ -62,23 +46,7 @@ def test_PilotsDB(): assert res["OK"], res["Message"] assert res["Value"]["TotalRecords"] >= 1 - res = pilots.setAccountingFlag("anotherPilot", "True") - assert res["OK"], res["Message"] res = pilots.setPilotStatus("anotherPilot", "Running") assert res["OK"], res["Message"] res = pilots.getPilotInfo("anotherPilot") assert res["OK"], res["Message"] - assert res["Value"]["anotherPilot"]["AccountingSent"] == "True" - assert res["Value"]["anotherPilot"]["Status"] == "Running" - - res = pilots.setJobForPilot(123, "anotherPilot") - assert res["OK"], res["Message"] - res = pilots.setPilotBenchmark("anotherPilot", 12.3) - assert res["OK"], res["Message"] - res = pilots.countPilots({}) - assert res["OK"], res["Message"] - - res = pilots.deletePilots("anotherPilot") - assert res["OK"], res["Message"] - res = pilots.getCurrentPilotCounters({}) - assert res["OK"], res["Message"] From 618425a462bf13718b7f1dfa048ebb4b55482d80 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 17 Jun 2026 11:00:43 +0200 Subject: [PATCH 2/5] fix: removed unused DB calls --- .../DB/PilotAgentsDB.py | 106 ++++++------------ .../DB/PilotAgentsDB.sql | 10 ++ 2 files changed, 45 insertions(+), 71 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py index 1c525bf0b49..519f5dd6cff 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py @@ -1,19 +1,6 @@ """PilotAgentsDB class is a front-end to the Pilot Agent Database. This database keeps track of all the submitted grid pilot jobs. -It also registers the mapping of the DIRAC jobs to the pilot -agents. - -Available methods are: - -addPilotReferences() -setPilotStatus() -deletePilot() -clearPilots() -setPilotDestinationSite() -setJobForPilot() -getPilotsSummary() -getGroupedPilotSummary() - +It also registers the mapping of the DIRAC jobs to the pilot agents. """ import datetime @@ -151,25 +138,6 @@ def setPilotStatus( args.append(pilotRef) return self._update(req, args=args, conn=conn) - # ########################################################################################### - # FIXME: this can't work ATM because of how the DB table is made. Maybe it would be useful later. - # def setPilotStatusBulk(self, pilotRefsStatusDict=None, statusReason=None, - # conn=False): - # """ Set pilot job status in a bulk - # """ - # if not pilotRefsStatusDict: - # return S_OK() - - # # Building the request with "ON DUPLICATE KEY UPDATE" - # reqBase = "INSERT INTO PilotAgents (PilotJobReference, Status, StatusReason) VALUES " - - # for pilotJobReference, status in pilotRefsStatusDict.items(): - # req = reqBase + ','.join("('%s', '%s', '%s')" % (pilotJobReference, status, statusReason)) - # req += " ON DUPLICATE KEY UPDATE Status=VALUES(Status),StatusReason=VALUES(StatusReason)" - - # return self._update(req, conn=conn) - - ########################################################################################## def selectPilots( self, condDict, older=None, newer=None, timeStamp="SubmissionTime", orderAttribute=None, limit=None ): @@ -364,43 +332,53 @@ def getPilotInfo(self, pilotRef=False, conn=False, paramNames=[], pilotID=False) return S_OK(resDict) ########################################################################################## - def setPilotDestinationSite(self, pilotRef, destination, conn=False): - """Set the pilot agent destination site""" - - gridSite = "Unknown" - res = getCESiteMapping(destination) - if res["OK"] and res["Value"]: - gridSite = res["Value"][destination] + def setAccountingFlag(self, pilotRef, mark="True"): + """Set the pilot AccountingSent flag""" - req = "UPDATE PilotAgents SET DestinationSite=%s, GridSite=%s WHERE PilotJobReference=%s" + req = "UPDATE PilotAgents SET AccountingSent=%s WHERE PilotJobReference=%s" args = ( - destination, - gridSite, + mark, pilotRef, ) - return self._update(req, args=args, conn=conn) + return self._update(req, args=args) ########################################################################################## - def setPilotBenchmark(self, pilotRef, mark): - """Set the pilot agent benchmark""" + def storePilotOutput(self, pilotRef, output, error): + """Store standard output and error for a pilot with pilotRef""" + pilotID = self.__getPilotID(pilotRef) + if not pilotID: + return S_ERROR(f"Pilot reference not found {pilotRef}") - req = "UPDATE PilotAgents SET BenchMark=%s WHERE PilotJobReference=%s" + req = "INSERT INTO PilotOutput (PilotID,StdOutput,StdError) VALUES (%s, %s, %s)" args = ( - f"{mark:f}", - pilotRef, + pilotID, + output, + error, ) return self._update(req, args=args) ########################################################################################## - def setAccountingFlag(self, pilotRef, mark="True"): - """Set the pilot AccountingSent flag""" + def getPilotOutput(self, pilotRef): + """Retrieve standard output and error for pilot with pilotRef""" - req = "UPDATE PilotAgents SET AccountingSent=%s WHERE PilotJobReference=%s" - args = ( - mark, - pilotRef, - ) - return self._update(req, args=args) + req = "SELECT StdOutput, StdError FROM PilotOutput,PilotAgents WHERE " + req += "PilotOutput.PilotID = PilotAgents.PilotID AND PilotAgents.PilotJobReference=%s" + result = self._query(req, args=(pilotRef,)) + if not result["OK"]: + return result + if not result["Value"]: + return S_ERROR(f"PilotJobReference {pilotRef} not found") + try: + stdout = result["Value"][0][0].decode() # account for the use of BLOBs + error = result["Value"][0][1].decode() + except AttributeError: + stdout = result["Value"][0][0] + error = result["Value"][0][1] + if stdout == '""': + stdout = "" + if error == '""': + error = "" + return S_OK({"StdOut": stdout, "StdErr": error}) ########################################################################################## def __getPilotID(self, pilotRef): @@ -501,20 +479,6 @@ def getPilotsForJobID(self, jobID): return S_OK([]) ########################################################################################## - def getPilotCurrentJob(self, pilotRef): - """The job ID currently executed by the pilot""" - req = "SELECT CurrentJobID FROM PilotAgents WHERE PilotJobReference=%s" - result = self._query(req, args=(pilotRef,)) - if not result["OK"]: - return result - if result["Value"]: - jobID = int(result["Value"][0][0]) - return S_OK(jobID) - self.log.warn(f"Current job ID for pilot {pilotRef} is not known: pilot did not match jobs yet?") - return S_OK() - - ########################################################################################## - # FIXME: investigate it getPilotSummaryShort can replace this method def getPilotSummary(self, startdate="", enddate=""): """Get summary of the pilot jobs status by site""" summary_dict = {} diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql index fa11604afcc..24cbe5a82b8 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql @@ -1,3 +1,5 @@ +-- $Header: /tmp/libdirac/tmp.stZoy15380/dirac/DIRAC3/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql,v 1.20 2009/08/26 09:39:53 rgracian Exp $ + -- ------------------------------------------------------------------------------ -- -- Schema definition for the PilotAgentsDB database - containing the Pilots status @@ -56,3 +58,11 @@ CREATE TABLE `JobToPilotMapping` ( KEY `JobID` (`JobID`), KEY `PilotID` (`PilotID`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +DROP TABLE IF EXISTS `PilotOutput`; +CREATE TABLE `PilotOutput` ( + `PilotID` INT(11) UNSIGNED NOT NULL, + `StdOutput` MEDIUMTEXT, + `StdError` MEDIUMTEXT, + PRIMARY KEY (`PilotID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From 150085755fc5edab8aff0b2a7ca7c620879f3fba Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 17 Jun 2026 11:12:47 +0200 Subject: [PATCH 3/5] fix: use directly the DBs instead of the services --- .../ResourceStatusSystem/Command/PilotCommand.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py b/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py index cfbc8cf4e0d..4796c4583eb 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py @@ -1,14 +1,15 @@ -""" PilotCommand +"""PilotCommand - The PilotCommand class is a command class to know about present pilots - efficiency. +The PilotCommand class is a command class to know about present pilots +efficiency. """ + from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping, getSites from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient from DIRAC.ResourceStatusSystem.Command.Command import Command -from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient +from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB class PilotCommand(Command): @@ -19,11 +20,7 @@ class PilotCommand(Command): def __init__(self, args=None, clients=None): super().__init__(args, clients) - if "Pilots" in self.apis: - self.pilots = self.apis["Pilots"] - else: - self.pilots = PilotManagerClient() - + self.pilots = PilotAgentsDB() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: From df7d7a5ce7a73046627e7a63aeb67a6602687774 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 18 Jun 2026 17:09:58 +0200 Subject: [PATCH 4/5] test: add also DN of adminusername --- tests/Jenkins/utilities.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index c9bdb9929d2..9c4362eac1a 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -412,6 +412,11 @@ diracUserAndGroup() { exit 1 fi + if ! dirac-admin-add-user -N adminusername -D /C=ch/O=DIRAC/OU=DIRAC\ CI/CN=adminusername -M lhcb-dirac-ci@cern.ch -G dirac_user -o /DIRAC/Security/UseServerCertificate=True "${DEBUG}"; then + echo 'ERROR: dirac-admin-add-user failed ' >&2 + exit 1 + fi + if ! dirac-admin-add-user -N pilot -D /C=ch/O=DIRAC/OU=DIRAC\ CI/CN=pilot -M lhcb-dirac-ci@cern.ch -G dirac_user -o /DIRAC/Security/UseServerCertificate=True "${DEBUG}"; then echo 'ERROR: dirac-admin-add-user failed' >&2 exit 1 From 674746952ea46e24ed374b682d40e349882cc102 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 22 Jun 2026 16:32:48 +0200 Subject: [PATCH 5/5] test: run all integration tests --- tests/CI/install_client.sh | 48 +++++++++---------- tests/CI/run_pilot.sh | 6 --- .../all_integration_client_tests.sh | 10 ++-- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/tests/CI/install_client.sh b/tests/CI/install_client.sh index a1fa7081746..9637c8340b2 100755 --- a/tests/CI/install_client.sh +++ b/tests/CI/install_client.sh @@ -58,35 +58,33 @@ echo -e "*** $(date -u) **** Client INSTALLATION START ****\n" installDIRAC -if [[ -z "${INSTALLATION_BRANCH}" ]]; then - echo 'Generate a pilot proxy, to be used by the pilot' - dirac-proxy-init -g pilot -C /ca/certs/pilot.pem -K /ca/certs/pilot.key "${DEBUG}" - mv /tmp/x509up_u$UID /ca/certs/pilot_proxy +echo 'Generate a pilot proxy, to be used by the pilot' +dirac-proxy-init -g pilot -C /ca/certs/pilot.pem -K /ca/certs/pilot.key "${DEBUG}" +mv /tmp/x509up_u$UID /ca/certs/pilot_proxy - echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt - dirac-proxy-init "${DEBUG}" |& tee -a clientTestOutputs.txt +echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt +dirac-proxy-init "${DEBUG}" |& tee -a clientTestOutputs.txt - #-------------------------------------------------------------------------------# - echo -e "*** $(date -u) **** Submit a job ****\n" +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Submit a job ****\n" - echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";' > test.jdl - echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test.jdl - echo "]" >> test.jdl - dirac-wms-job-submit test.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt +echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";' > test.jdl +echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test.jdl +echo "]" >> test.jdl +dirac-wms-job-submit test.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt - #-------------------------------------------------------------------------------# - echo -e "*** $(date -u) **** add a file ****\n" +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** add a file ****\n" - echo "${CLIENT_UPLOAD_BASE64}" > b64_lfn - base64 b64_lfn --decode > "${CLIENT_UPLOAD_FILE}" - dirac-dms-add-file "${CLIENT_UPLOAD_LFN}" "${CLIENT_UPLOAD_FILE}" S3-DIRECT - echo $? +echo "${CLIENT_UPLOAD_BASE64}" > b64_lfn +base64 b64_lfn --decode > "${CLIENT_UPLOAD_FILE}" +dirac-dms-add-file "${CLIENT_UPLOAD_LFN}" "${CLIENT_UPLOAD_FILE}" S3-DIRECT +echo $? - #-------------------------------------------------------------------------------# - echo -e "*** $(date -u) **** Submit a job with an input ****\n" +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Submit a job with an input ****\n" - echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";\n InputData = "/vo/test_lfn.txt";' > test_dl.jdl - echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test_dl.jdl - echo "]" >> test_dl.jdl - dirac-wms-job-submit test_dl.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt -fi +echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";\n InputData = "/vo/test_lfn.txt";' > test_dl.jdl +echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test_dl.jdl +echo "]" >> test_dl.jdl +dirac-wms-job-submit test_dl.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt diff --git a/tests/CI/run_pilot.sh b/tests/CI/run_pilot.sh index a94ecce1b41..31b23e7539d 100755 --- a/tests/CI/run_pilot.sh +++ b/tests/CI/run_pilot.sh @@ -11,12 +11,6 @@ set -x echo "Starting run_pilot.sh" source CONFIG -if [[ -n "${INSTALLATION_BRANCH}" ]]; then - # Do not run this - echo "Not running the DIRAC Pilot" - exit -fi - # Creating "the worker node" mkdir -p /home/dirac/etc/grid-security/certificates mkdir -p /home/dirac/etc/grid-security/vomsdir diff --git a/tests/Integration/all_integration_client_tests.sh b/tests/Integration/all_integration_client_tests.sh index e6a296de44a..bfc0078dcba 100644 --- a/tests/Integration/all_integration_client_tests.sh +++ b/tests/Integration/all_integration_client_tests.sh @@ -56,10 +56,8 @@ pytest --no-check-dirac-environment "${THIS_DIR}/ResourceStatusSystem/Test_Email echo -e "*** $(date -u) **** WMS TESTS ****\n" pytest --no-check-dirac-environment "${THIS_DIR}/WorkloadManagementSystem/Test_SandboxStoreClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -if [[ -z "${INSTALLATION_BRANCH}" ]]; then - pytest --no-check-dirac-environment "${THIS_DIR}/WorkloadManagementSystem/Test_PilotsClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) - pytest --no-check-dirac-environment "${THIS_DIR}/WorkloadManagementSystem/Test_Client_WMS.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -fi +pytest --no-check-dirac-environment "${THIS_DIR}/WorkloadManagementSystem/Test_PilotsClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest --no-check-dirac-environment "${THIS_DIR}/WorkloadManagementSystem/Test_Client_WMS.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) # Make sure we have the prod role for these tests to get the VmRpcOperator permission dirac-proxy-init -g prod "${DEBUG}" |& tee -a clientTestOutputs.txt @@ -72,9 +70,7 @@ python "${THIS_DIR}/WorkloadManagementSystem/createJobXMLDescriptions.py" |& tee #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** MONITORING TESTS ****\n" pytest --no-check-dirac-environment "${THIS_DIR}/Monitoring/Test_MonitoringSystem.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -if [[ -z "${INSTALLATION_BRANCH}" ]]; then - pytest --no-check-dirac-environment "${THIS_DIR}/Monitoring/Test_WebAppClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -fi +pytest --no-check-dirac-environment "${THIS_DIR}/Monitoring/Test_WebAppClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** TS TESTS ****\n"