diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a72ddef4..b57a2f84 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,21 +8,23 @@ jobs: strategy: matrix: - python: - - 2.7.18 - - 3.6.15 - - 3.9.17 dirac_version: - rel-v8r0 - integration pilot_schema: - pilot_oldSchema.json - pilot_newSchema.json - - container: python:${{ matrix.python }}-slim + VO: + - dteam + - gridpp steps: - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + - name: Retrieve the secret and decode it to a file env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} @@ -45,7 +47,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug integration-cvmfs: @@ -93,7 +98,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --pilotUUID="${pilotUUID}" --debug integration-cvmfs_no_env: @@ -128,7 +136,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug @@ -137,18 +148,17 @@ jobs: strategy: matrix: - python: - - 2.7.18 - - 3.6.15 - - 3.9.17 pilot_schema: - pilot_oldSchema.json - pilot_newSchema.json - container: python:${{ matrix.python }}-slim - steps: - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + - name: Retrieve the secret and decode it to a file env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} @@ -171,7 +181,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --pilotUUID="${pilotUUID}" --debug integration-cvmfs_matching: @@ -215,7 +228,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug ################################## @@ -272,7 +288,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o lbRunOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o lbRunOnly --pilotUUID="${pilotUUID}" --debug ext-lhcb_integration_dirac_installer_no_env: runs-on: ubuntu-latest @@ -316,7 +335,10 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug ext-lhcb_integration_no_env: runs-on: ubuntu-latest @@ -360,4 +382,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug + g_job="testintegrationworkflow${GITHUB_JOB//-/}" + pilotUUID="${g_job//_/}""${GITHUB_RUN_ID}" + pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 66f0c282..c79b4dff 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -206,6 +206,7 @@ def execute(self): ) self.exitWithError(1) + class InstallDIRAC(CommandBase): """ Source from CVMFS, or install locally """ @@ -498,7 +499,6 @@ def execute(self): # environment variables to add? if self.pp.userEnvVariables: # User-requested environment variables (comma-separated, name and value separated by ":::") - # The case of dirac-install is handled in vanilla DIRAC Pilot newEnvVars = dict( name.split(":::", 1) for name in self.pp.userEnvVariables.replace(" ", "").split(",") ) @@ -526,7 +526,7 @@ class ConfigureBasics(CommandBase): It calls dirac-configure to: - * download, by default, the CAs + * (maybe) download the CAs * creates a standard or custom (defined by self.pp.localConfigFile) cfg file (by default 'pilot.cfg') to be used where all the pilot configuration is to be set, e.g.: * adds to it basic info like the version @@ -605,37 +605,6 @@ def _getBasicsCFG(self): if self.pp.wnVO: self.cfg.append('-o "/Resources/Computing/CEDefaults/VirtualOrganization=%s"' % self.pp.wnVO) - def __checkSecurityDir(self, envName, dirName): - - if envName in os.environ and safe_listdir(os.environ[envName]): - self.log.debug( - "%s is set in the host environment as %s, aligning installEnv to it" - % (envName, os.environ[envName]) - ) - self.pp.installEnv[envName] = os.environ[envName] - else: - self.log.debug("%s is not set in the host environment" % envName) - # try and find it - for candidate in self.pp.CVMFS_locations: - candidateDir = os.path.join(candidate, - 'etc/grid-security', - dirName) - self.log.debug( - "Candidate directory for %s is %s" - % (envName, candidateDir) - ) - if safe_listdir(candidateDir): - - self.log.debug("Setting %s=%s" % (envName, candidateDir)) - self.pp.installEnv[envName] = candidateDir - os.environ[envName] = candidateDir - break - self.log.debug("%s not found or not a directory" % candidateDir) - - if envName not in self.pp.installEnv: - self.log.error("Could not find/set %s" % envName) - sys.exit(1) - def _getSecurityCFG(self): """ Sets security-related env variables, if needed """ @@ -645,17 +614,8 @@ def _getSecurityCFG(self): self.cfg.append("-o /DIRAC/Security/CertFile=%s/hostcert.pem" % self.pp.certsLocation) self.cfg.append("-o /DIRAC/Security/KeyFile=%s/hostkey.pem" % self.pp.certsLocation) - # If DIRAC (or its extension) is installed in CVMFS: + # If DIRAC (or its extension) is installed in CVMFS do not download VOMS and CAs if self.pp.preinstalledEnv: - - self.__checkSecurityDir("X509_CERT_DIR", "certificates") - self.__checkSecurityDir("X509_VOMS_DIR", "vomsdir") - self.__checkSecurityDir("X509_VOMSES", "vomses") - # This is needed for the integration tests - self.pp.installEnv["DIRAC_VOMSES"] = self.pp.installEnv["X509_VOMSES"] - os.environ["DIRAC_VOMSES"] = os.environ["X509_VOMSES"] - - # In any case do not download VOMS and CAs self.cfg.append("-DMH") diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8da986da..6f9a0f30 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -8,7 +8,6 @@ import imp import json import os -import pickle import re import select import signal @@ -17,7 +16,6 @@ import sys import threading from datetime import datetime -from distutils.version import LooseVersion from functools import partial, wraps from threading import RLock @@ -53,6 +51,11 @@ except NameError: FileNotFoundError = OSError +try: + IsADirectoryError # pylint: disable=used-before-assignment +except NameError: + IsADirectoryError = OSError + # Timer 2.7 and < 3.3 versions issue where Timer is a function if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: from threading import _Timer as Timer # pylint: disable=no-name-in-module @@ -89,15 +92,6 @@ def parseVersion(releaseVersion, useLegacyStyle): return version -def printVersion(log): - log.info("Running %s" % " ".join(sys.argv)) - try: - with open("%s.run" % sys.argv[0], "w") as fd: - pickle.dump(sys.argv[1:], fd) - except OSError: - pass - - def pythonPathCheck(): try: os.umask(18) # 022 @@ -621,18 +615,29 @@ def sendMessage(url, pilotUUID, method, rawMessage): :param str rawMessage: a message to be sent, in JSON format :return: None. """ - message = json.dumps((json.dumps(rawMessage), pilotUUID)) - major, minor, micro, _, _ = sys.version_info - if major >= 3: - data = urlencode({"method": method, "args": message}).encode("utf-8") # encode to bytes ! for python3 - else: - data = urlencode({"method": method, "args": message}) caPath = os.getenv("X509_CERT_DIR") cert = os.getenv("X509_USER_PROXY") context = ssl.create_default_context() context.load_verify_locations(capath=caPath) - context.load_cert_chain(cert) + + message = json.dumps((json.dumps(rawMessage), pilotUUID)) + + try: + context.load_cert_chain(cert) # this is a proxy + raw_data = {"method": method, "args": message} + except IsADirectoryError: # assuming it'a dir containing cert and key + context.load_cert_chain( + os.path.join(cert, "hostcert.pem"), + os.path.join(cert, "hostkey.pem") + ) + raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} + + if sys.version_info[0] == 3: + data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 + else: + data = urlencode(raw_data) + res = urlopen(url, data, context=context) res.close() @@ -850,7 +855,7 @@ def __init__(self): self.pilotCFGFile = "pilot.json" self.pilotLogging = False self.loggerURL = None - self.loggerTimerInterval = 600 + self.loggerTimerInterval = 0 self.pilotUUID = "unknown" self.modules = "" # see dirac-install "-m" option documentation self.userEnvVariables = "" # see dirac-install "--userEnvVariables" option documentation @@ -940,6 +945,48 @@ def __init__(self): # Command line can override options from JSON self.__initCommandLine2() + self.__checkSecurityDir("X509_CERT_DIR", "certificates") + self.__checkSecurityDir("X509_VOMS_DIR", "vomsdir") + self.__checkSecurityDir("X509_VOMSES", "vomses") + # This is needed for the integration tests + self.installEnv["DIRAC_VOMSES"] = self.installEnv["X509_VOMSES"] + os.environ["DIRAC_VOMSES"] = os.environ["X509_VOMSES"] + + if self.useServerCertificate: + self.installEnv["X509_USER_PROXY"] = self.certsLocation + os.environ["X509_USER_PROXY"] = self.certsLocation + + def __checkSecurityDir(self, envName, dirName): + + if envName in os.environ and safe_listdir(os.environ[envName]): + self.log.debug( + "%s is set in the host environment as %s, aligning installEnv to it" + % (envName, os.environ[envName]) + ) + self.installEnv[envName] = os.environ[envName] + else: + self.log.debug("%s is not set in the host environment" % envName) + # try and find it + for candidate in self.CVMFS_locations: + candidateDir = os.path.join(candidate, + 'etc/grid-security', + dirName) + self.log.debug( + "Candidate directory for %s is %s" + % (envName, candidateDir) + ) + if safe_listdir(candidateDir): + self.log.debug("Setting %s=%s" % (envName, candidateDir)) + self.installEnv[envName] = candidateDir + os.environ[envName] = candidateDir + break + self.log.debug("%s not found or not a directory" % candidateDir) + + if envName not in self.installEnv: + self.log.error("Could not find/set %s" % envName) + sys.exit(1) + + def __initCommandLine1(self): """Parses and interpret options on the command line: first pass (essential things)""" @@ -1148,6 +1195,10 @@ def __initJSON2(self): self.releaseProject = pilotOptions.get("Project", self.releaseProject) # default from the code. self.log.debug("Release project: %s" % self.releaseProject) + self.CVMFS_locations = pilotOptions.get("CVMFS_locations", self.CVMFS_locations) # default from the code. + self.log.debug("CVMFS locations: %s" % self.CVMFS_locations) + + def getPilotOptionsDict(self): """ Get pilot option dictionary by searching paths in a certain order (commands, logging etc.). @@ -1185,7 +1236,7 @@ def __getVO(self): except IOError as err: self.log.error("Could not read a proxy, setting vo to 'unknown': ", os.strerror(err.errno)) else: - self.log.error("Could not locate a proxy via X509_USER_PROXY, setting vo to 'unknown' ") + self.log.error("Could not locate a proxy via X509_USER_PROXY") # is there a token, and can we get a VO from the token? # TBD diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index adf69e82..1f5c5986 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -3,18 +3,18 @@ from __future__ import absolute_import, division, print_function -# pylint: disable=protected-access, missing-docstring, invalid-name, line-too-long - -# imports -import unittest import json -import stat -import sys import os import shutil +import stat +import sys + +# pylint: disable=protected-access, missing-docstring, invalid-name, line-too-long +# imports +import unittest -from Pilot.pilotTools import PilotParams from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from Pilot.pilotTools import PilotParams class PilotTestCase(unittest.TestCase): @@ -39,6 +39,10 @@ def setUp(self): }, fp, ) + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() def tearDown(self): for fileProd in [ diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 3cc53209..7a8688cb 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,12 +1,10 @@ -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function -import subprocess -import shlex -import sys import os +import shlex import shutil +import subprocess +import sys import unittest ############################ @@ -49,7 +47,7 @@ def test_getVOPopenFails(self, popenMock): data = fp.read() os.remove(cert) with self.assertRaises(Exception) as exc: - vo = getVO(data) + getVO(data) self.assertEqual(str(exc.exception), msg) @@ -58,7 +56,7 @@ def test_getVOPopenFails(self, popenMock): msg = "command not found: openssl" popenMock.side_effect = OSError(msg) with self.assertRaises(OSError) as exc: - res = getVO(data) + getVO(data) self.assertEqual(str(exc.exception), msg) @patch("Pilot.proxyTools.Popen") @@ -70,7 +68,7 @@ def test_parseASN1Fails(self, popenMock): msg = "Error when invoking openssl asn1parse" with self.assertRaises(Exception) as exc: - res = parseASN1("Anything") + parseASN1("Anything") self.assertEqual(str(exc.exception), msg) @@ -80,7 +78,7 @@ def test_parseASN1Fails(self, popenMock): msg = "command not found: openssl" popenMock.side_effect = OSError(msg) with self.assertRaises(OSError) as exc: - res = parseASN1("Anything") + parseASN1("Anything") self.assertEqual(str(exc.exception), msg) def test_createFakeProxy(self): diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 701c1efc..9c176c82 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -34,6 +34,10 @@ def test_pilotParamsInit(self, argvmock): "tests/pilot.json", ] + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() pp = PilotParams() argvmock.__getitem__.assert_called() @@ -89,13 +93,17 @@ def test_pilotOptions(self, argvmock, mockPaths): "/%s/Pilot" % vo ] mockPaths.return_value = paths + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() pp = PilotParams() lTESTcommands = "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements" pp.gridCEType = "TEST" res = pp.getPilotOptionsDict() - logURL = "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging" + logURL = "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging" self.assertEqual(res.get("RemoteLoggerURL"), logURL) self.assertEqual(pp.loggerURL, logURL) self.assertEqual(res.get("RemoteLogging"), "False") @@ -112,6 +120,9 @@ def setUp(self): # These temporary files, opened in a binary mode, will act as standard stream pipes for `Popen` self.stdout_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False) self.stderr_mock = tempfile.NamedTemporaryFile(mode="rb+", delete=False) + os.environ["X509_CERT_DIR"] = '/some/thing/' + os.environ["X509_USER_PROXY"] = '/some/thing' + def tearDown(self): # At the end of the test, we'll close and remove the created files @@ -122,7 +133,10 @@ def tearDown(self): @patch(("sys.argv")) @patch("subprocess.Popen") def test_executeAndGetOutput(self, popenMock, argvmock): - + os.environ["X509_CERT_DIR"] = os.getcwd() + os.environ["X509_VOMS_DIR"] = os.getcwd() + os.environ["X509_VOMSES"] = os.getcwd() + os.environ["X509_USER_PROXY"] = os.getcwd() argvmock.__getitem__.return_value = [ "-d", "-g", diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 38b2b34a..8caa6f5a 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -101,43 +101,48 @@ "Version": "VAR_DIRAC_VERSION", "Extensions": "None", "CheckVersion": "False", - "pilotFileServer": "lbvobox70.cern.ch:8443", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "pilotRepoBranch": "does_not_matter", "pilotRepo": "https://github.com/does_not_matter/Pilot.git", "GenericPilotGroup": "gridpp_pilot", "GenericPilotDN": "VAR_USERDN_GRIDPP", "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "UploadSE": "UKI-LT2-IC-HEP-disk", "UploadPath": "/gridpp/pilotlogs/", + "RemoteLoggerTimerInterval": 0, "LoggingShifterName": "GridPPLogManager", - "PilotLogLevel": "DEBUG" + "PilotLogLevel": "DEBUG", + "CVMFS_locations": ["/cvmfs/grid.cern.ch", "/cvmfs/dirac.egi.eu"] } }, "dteam": { "Pilot": { "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "True", - "pilotFileServer": "lbvobox70.cern.ch:8443", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "pilotRepoBranch": "should_not_matter", "pilotRepo": "https://github.com/should_not_matter/Pilot.git", "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "PilotLogLevel": "DEBUG" + "RemoteLogging": "True", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": ["/cvmfs/grid.cern.ch", "/cvmfs/dirac.egi.eu"] } }, "LHCb": { "Pilot": { "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "True", - "pilotFileServer": "lbvobox70.cern.ch:8443", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbvobox70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "PilotLogLevel": "DEBUG" + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": ["/cvmfs/lhcb.cern.ch/", "$VO_LHCB_SW_DIR"] } }, "ConfigurationServers": [