Skip to content

Commit e2764d6

Browse files
fstagnifstagni
authored
Merge pull request #279 from fstagni/newMaster20260225
New master
2 parents 10330ec + 7feb97a commit e2764d6

12 files changed

Lines changed: 403 additions & 355 deletions

‎.github/workflows/basic.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ jobs:
2424
fail-fast: false
2525
matrix:
2626
python:
27-
- 2.7.18
2827
- 3.6.15
2928
- 3.9.17
3029

@@ -55,7 +54,6 @@ jobs:
5554
fail-fast: false
5655
matrix:
5756
python:
58-
- 2.7.18
5957
- 3.6.15
6058
- 3.9.17
6159

@@ -82,7 +80,6 @@ jobs:
8280
fail-fast: false
8381
matrix:
8482
python:
85-
- 2.7.18
8683
- 3.6.15
8784
- 3.9.17
8885

‎.github/workflows/integration.yml‎

Lines changed: 11 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
pip install DIRAC
4444
pilot_proxy_file=$(mktemp)
4545
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
46-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file
46+
dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file
4747
cd ${GITHUB_WORKSPACE}/Pilot
4848
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
4949
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -59,17 +59,14 @@ jobs:
5959
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
6060
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_branch }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug
6161
62-
release_prod_pre-cvmfs_CEs:
62+
release_cvmfs_CEs:
6363
runs-on: ubuntu-latest
6464

6565
strategy:
6666
matrix:
6767
pilot_schema:
6868
- pilot_oldSchema.json
6969
- pilot_newSchema.json
70-
dirac_version:
71-
- prod
72-
- old
7370
ce:
7471
- jenkins.cern.ch
7572
- jenkins-singularity.cern.ch
@@ -98,15 +95,11 @@ jobs:
9895
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
9996
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
10097
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
101-
if [ "${{ matrix.dirac_version }}" == "prod" ]; then
102-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
103-
else
104-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1)
105-
fi
98+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
10699
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
107100
pilot_proxy_file=$(mktemp)
108101
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
109-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
102+
dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
110103
cd ${GITHUB_WORKSPACE}/Pilot
111104
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
112105
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -156,7 +149,7 @@ jobs:
156149
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
157150
pilot_proxy_file=$(mktemp)
158151
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
159-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
152+
dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
160153
cd ${GITHUB_WORKSPACE}/Pilot
161154
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
162155
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -172,18 +165,14 @@ jobs:
172165
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
173166
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
174167
175-
release_prod_pre-cvmfs_matching:
168+
release_cvmfs_matching:
176169
runs-on: ubuntu-latest
177170

178171
strategy:
179172
matrix:
180173
VO:
181174
- dteam
182175
- gridpp
183-
dirac_version:
184-
- prod
185-
- old
186-
187176
steps:
188177
- uses: actions/checkout@v4
189178
- uses: cvmfs-contrib/github-action-cvmfs@v3
@@ -206,15 +195,11 @@ jobs:
206195
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
207196
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
208197
cd ${GITHUB_WORKSPACE}/Pilot
209-
if [ "${{ matrix.dirac_version }}" == "prod" ]; then
210-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
211-
else
212-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1)
213-
fi
198+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
214199
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
215200
pilot_proxy_file=$(mktemp)
216201
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
217-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
202+
dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
218203
cd ${GITHUB_WORKSPACE}/Pilot
219204
cp ../tests/CI/pilot_newSchema.json pilot.json
220205
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -267,7 +252,7 @@ jobs:
267252
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
268253
pilot_proxy_file=$(mktemp)
269254
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
270-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
255+
dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
271256
cd ${GITHUB_WORKSPACE}/Pilot
272257
cp ../tests/CI/pilot_newSchema.json pilot.json
273258
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
@@ -327,7 +312,7 @@ jobs:
327312
source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc
328313
pilot_proxy_file=$(mktemp)
329314
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
330-
X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
315+
X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
331316
cd ${GITHUB_WORKSPACE}/Pilot
332317
export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot
333318
curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py
@@ -382,7 +367,7 @@ jobs:
382367
source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc
383368
pilot_proxy_file=$(mktemp)
384369
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
385-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
370+
dirac-admin-get-proxy fstagni dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
386371
cd ${GITHUB_WORKSPACE}/Pilot
387372
curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py
388373
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json

‎.pylintrc‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,3 @@ dummy-variables=_
1818
disable=
1919
invalid-name,
2020
line-too-long, # would be nice to remove this one
21-
consider-using-f-string, # python2/3 support
22-
unspecified-encoding, # python2/3 support
23-
super-with-arguments, # python2/3 support
24-
redefined-builtin, # python2/3 support

‎Pilot/__init__.py‎

Whitespace-only changes.

‎Pilot/dirac-pilot.py‎

Lines changed: 8 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,36 +19,19 @@
1919
But, as said, all the actions are actually configurable.
2020
"""
2121

22-
from __future__ import absolute_import, division, print_function
23-
2422
import os
2523
import sys
2624
import time
25+
from io import StringIO
2726

28-
############################
29-
# python 2 -> 3 "hacks"
30-
31-
try:
32-
from cStringIO import StringIO
33-
except ImportError:
34-
from io import StringIO
27+
from pilotTools import (
28+
Logger,
29+
PilotParams,
30+
RemoteLogger,
31+
getCommand,
32+
pythonPathCheck,
33+
)
3534

36-
try:
37-
from Pilot.pilotTools import (
38-
Logger,
39-
PilotParams,
40-
RemoteLogger,
41-
getCommand,
42-
pythonPathCheck,
43-
)
44-
except ImportError:
45-
from pilotTools import (
46-
Logger,
47-
PilotParams,
48-
RemoteLogger,
49-
getCommand,
50-
pythonPathCheck,
51-
)
5235
############################
5336

5437
if __name__ == "__main__":

‎Pilot/pilotCommands.py‎

Lines changed: 25 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@ def __init__(self, pilotParams):
1717
execution.
1818
"""
1919

20-
from __future__ import absolute_import, division, print_function
21-
2220
import filecmp
2321
import os
2422
import platform
@@ -28,39 +26,18 @@ def __init__(self, pilotParams):
2826
import sys
2927
import time
3028
import traceback
31-
import subprocess
3229
from collections import Counter
30+
from http.client import HTTPSConnection
31+
from shlex import quote
32+
33+
from pilotTools import (
34+
CommandBase,
35+
getSubmitterInfo,
36+
retrieveUrlTimeout,
37+
safe_listdir,
38+
sendMessage,
39+
)
3340

34-
############################
35-
# python 2 -> 3 "hacks"
36-
try:
37-
# For Python 3.0 and later
38-
from http.client import HTTPSConnection
39-
except ImportError:
40-
# Fall back to Python 2
41-
from httplib import HTTPSConnection
42-
43-
try:
44-
from shlex import quote
45-
except ImportError:
46-
from pipes import quote
47-
48-
try:
49-
from Pilot.pilotTools import (
50-
CommandBase,
51-
getSubmitterInfo,
52-
retrieveUrlTimeout,
53-
safe_listdir,
54-
sendMessage,
55-
)
56-
except ImportError:
57-
from pilotTools import (
58-
CommandBase,
59-
getSubmitterInfo,
60-
retrieveUrlTimeout,
61-
safe_listdir,
62-
sendMessage,
63-
)
6441
############################
6542

6643

@@ -138,15 +115,19 @@ def execute(self):
138115
self.log.info("Host FQDN = %s" % socket.getfqdn())
139116
self.log.info("WorkingDir = %s" % self.pp.workingDir) # this could be different than rootPath
140117

141-
fileName = "/etc/redhat-release"
142-
if os.path.exists(fileName):
143-
with open(fileName, "r") as f:
144-
self.log.info("RedHat Release = %s" % f.read().strip())
145-
146-
fileName = "/etc/lsb-release"
147-
if os.path.isfile(fileName):
148-
with open(fileName, "r") as f:
149-
self.log.info("Linux release:\n%s" % f.read().strip())
118+
for fileName in ["/etc/os-release", "/usr/lib/os-release"]:
119+
if os.path.isfile(fileName):
120+
try:
121+
with open(fileName, "r") as f:
122+
for line in f:
123+
line = line.strip()
124+
if line.startswith(("NAME=", "VERSION=", "PRETTY_NAME=")):
125+
self.log.info(
126+
"OS Release = %s" % line.split("=", 1)[1].strip('"')
127+
)
128+
break
129+
except (OSError, IOError):
130+
self.log.debug("Could not read %s" % fileName)
150131

151132
fileName = "/proc/cpuinfo"
152133
if os.path.exists(fileName):
@@ -283,7 +264,7 @@ def _getPreinstalledEnvScript(self):
283264
self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript
284265

285266
def _localInstallDIRAC(self):
286-
"""Install python3 version of DIRAC client"""
267+
"""Install DIRAC client"""
287268

288269
self.log.info("Installing DIRAC locally")
289270

@@ -296,10 +277,7 @@ def _localInstallDIRAC(self):
296277

297278
# 1. Get the DIRACOS installer name
298279
# curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh
299-
try:
300-
machine = os.uname().machine # py3
301-
except AttributeError:
302-
machine = os.uname()[4] # py2
280+
machine = os.uname().machine
303281

304282
installerName = "DIRACOS-Linux-%s.sh" % machine
305283

0 commit comments

Comments
 (0)