diff --git a/Dockerfile b/Dockerfile index 5bfccae2..b0529616 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ RUN mkdir /var/idds RUN mkdir /var/idds/wsgisocks RUN chown atlpan -R /opt/idds # RUN chown atlpan -R /opt/idds_source -RUN chown atlpan /var/log/idds +RUN chown atlpan -R /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ # setup conda virtual env @@ -65,8 +65,23 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client -RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG + +WORKDIR /tmp/src +COPY . . + +RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ + if [[ -z "$TAG" ]] ; then \ + python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \ + else \ + python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ + fi + +WORKDIR /tmp +RUN rm -rf /tmp/src + +RUN chmod 777 /opt/idds/monitor/data +RUN chmod 777 /opt/idds/monitor/data/conf.js RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds # RUN mkdir /opt/idds/config_default @@ -93,12 +108,32 @@ RUN ln -fs /opt/idds/config/idds/auth.cfg /opt/idds/etc/idds/auth/auth.cfg RUN ln -fs /opt/idds/config/idds/gacl /opt/idds/etc/idds/rest/gacl RUN ln -fs /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf +# update http config +RUN sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf +RUN sed -i 's/Listen\ 80/#\ Listen\ 80/g' /etc/httpd/conf/httpd.conf +RUN sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf + # for idds daemons RUN ln -fs /opt/idds/config/idds/supervisord_idds.ini /etc/supervisord.d/idds.ini +RUN chmod -R 777 /opt/idds/config +RUN chmod -R 777 /var/log/idds +RUN chmod 777 /etc/grid-security +RUN chmod 777 /etc/httpd/conf.d +RUN chmod 777 /etc/httpd/conf/httpd.conf +RUN chmod 777 /etc/httpd/conf +RUN chmod 777 /run/httpd +RUN chmod 777 /var/log/supervisor/ +RUN chmod 777 /var/run/supervisor +RUN chmod 777 /var/run +RUN chmod 777 /etc/httpd/logs + ENV PATH /opt/idds/bin/:$PATH ADD start-daemon.sh /opt/idds/bin/ +RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back +# ADD ssl.conf /etc/httpd/conf.d/ssl.conf +RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf VOLUME /var/log/idds VOLUME /opt/idds/config diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 0384e5e0..7c2e8610 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py index bcd505bb..07fdf512 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -117,6 +117,23 @@ def load_panda_urls(self): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if panda_config.has_option('panda', 'pandacache_url'): + self.pandacache_url = panda_config.get('panda', 'pandacache_url') + os.environ['PANDACACHE_URL'] = self.pandacache_url + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if panda_config.has_option('panda', 'panda_verify_host'): + self.panda_verify_host = panda_config.get('panda', 'panda_verify_host') + os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) + if panda_config.has_option('panda', 'panda_auth'): + self.panda_auth = panda_config.get('panda', 'panda_auth') + os.environ['PANDA_AUTH'] = self.panda_auth + if panda_config.has_option('panda', 'panda_auth_vo'): + self.panda_auth_vo = panda_config.get('panda', 'panda_auth_vo') + os.environ['PANDA_AUTH_VO'] = self.panda_auth_vo + if panda_config.has_option('panda', 'panda_config_root'): + self.panda_config_root = panda_config.get('panda', 'panda_config_root') + os.environ['PANDA_CONFIG_ROOT'] = self.panda_config_root if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']: self.panda_monitor = os.environ['PANDA_MONITOR_URL'] @@ -127,6 +144,18 @@ def load_panda_urls(self): if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']: + self.pandacache_url = os.environ['PANDACACHE_URL'] + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']: + self.panda_verify_host = os.environ['PANDA_VERIFY_HOST'] + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) + if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']: + self.panda_auth = os.environ['PANDA_AUTH'] + if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']: + self.panda_auth_vo = os.environ['PANDA_AUTH_VO'] + if not self.panda_config_root and 'PANDA_CONFIG_ROOT' in os.environ and os.environ['PANDA_CONFIG_ROOT']: + self.panda_config_root = os.environ['PANDA_CONFIG_ROOT'] def set_agent_attributes(self, attrs, req_attributes=None): if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 077379cd..fdfef10d 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.0 - - idds-workflow==0.11.0 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/client/lib/idds/client/base.py b/client/lib/idds/client/base.py index 1a4efffd..2f24f1e8 100644 --- a/client/lib/idds/client/base.py +++ b/client/lib/idds/client/base.py @@ -26,7 +26,7 @@ from idds.common import exceptions from idds.common.constants import HTTP_STATUS_CODE -from idds.common.utils import json_dumps, json_loads +from idds.common.utils import json_dumps, json_loads, get_proxy_path from idds.common.authentication import OIDCAuthenticationUtils @@ -68,6 +68,20 @@ def __init__(self, host=None, auth=None, timeout=None, client_proxy=None): self.check_auth() + def get_user_proxy(sellf): + """ + Get the user proxy. + + :returns: the path of the user proxy. + """ + + client_proxy = get_proxy_path() + + if not client_proxy or not os.path.exists(client_proxy): + raise exceptions.RestException("Cannot find a valid x509 proxy.") + + return client_proxy + def check_auth(self): """ To check whether the auth type is supported and the input for the auth is available. @@ -77,6 +91,8 @@ def check_auth(self): self.auth_type = 'x509_proxy' if self.auth_type in ['x509_proxy']: + if not self.client_proxy: + self.client_proxy = self.get_user_proxy() if not self.client_proxy or not os.path.exists(self.client_proxy): raise exceptions.RestException("Cannot find a valid x509 proxy.") elif self.auth_type in ['oidc']: diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 0384e5e0..7c2e8610 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 51b4c3f8..45756d68 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.0 - - idds-workflow==0.11.0 + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/common/lib/idds/common/authentication.py b/common/lib/idds/common/authentication.py index d6336f36..1ed0a549 100644 --- a/common/lib/idds/common/authentication.py +++ b/common/lib/idds/common/authentication.py @@ -45,6 +45,12 @@ def decode_value(val): return int.from_bytes(decoded, 'big') +def should_verify(): + if os.environ.get('IDDS_AUTH_NO_VERIFY', None): + return False + return True + + class BaseAuthentication(object): def __init__(self, timeout=None): self.timeout = timeout @@ -98,7 +104,7 @@ def get_auth_config(self, vo): def get_http_content(self, url): try: - r = requests.get(url, allow_redirects=True) + r = requests.get(url, allow_redirects=True, verify=should_verify()) return r.content except Exception as error: return False, 'Failed to get http content for %s: %s' (str(url), str(error)) @@ -128,6 +134,7 @@ def get_oidc_sign_url(self, vo): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: @@ -172,6 +179,7 @@ def get_id_token(self, vo, device_code, interval=5, expires_in=60): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: if result.status_code == HTTP_STATUS_CODE.OK and result.text: @@ -203,6 +211,7 @@ def refresh_id_token(self, vo, refresh_token): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 0384e5e0..7c2e8610 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 8e5196ff..a9a272c5 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index e8bc1fbf..9805f961 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -89,7 +89,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.prodSourceLabel = prodSourceLabel self.task_type = task_type self.maxWalltime = maxwalltime - self.maxAttempt = maxattempt + self.maxAttempt = maxattempt if maxattempt else 5 self.core_count = core_count self.task_log = task_log @@ -150,6 +150,14 @@ def load_panda_urls(self): self.panda_url = panda_config.get('panda', 'panda_url') os.environ['PANDA_URL'] = self.panda_url # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if panda_config.has_option('panda', 'pandacache_url'): + self.pandacache_url = panda_config.get('panda', 'pandacache_url') + os.environ['PANDACACHE_URL'] = self.pandacache_url + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if panda_config.has_option('panda', 'panda_verify_host'): + self.panda_verify_host = panda_config.get('panda', 'panda_verify_host') + os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) if panda_config.has_option('panda', 'panda_url_ssl'): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl @@ -173,6 +181,12 @@ def load_panda_urls(self): if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']: + self.pandacache_url = os.environ['PANDACACHE_URL'] + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']: + self.panda_verify_host = os.environ['PANDA_VERIFY_HOST'] + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']: self.panda_auth = os.environ['PANDA_AUTH'] if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']: diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 61559a5d..4847bdb4 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.0 - - idds-workflow==0.11.0 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/main/config_default/httpd-idds-443-py39-cc7.conf b/main/config_default/httpd-idds-443-py39-cc7.conf index db09cc6b..ffb5b3d1 100644 --- a/main/config_default/httpd-idds-443-py39-cc7.conf +++ b/main/config_default/httpd-idds-443-py39-cc7.conf @@ -34,7 +34,7 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages WSGIApplicationGroup %GLOBAL WSGIScriptAlias /idds /opt/idds/bin/idds.wsgi # WSGIScriptAliasMatch ^/idds/(.+)$ /opt/idds/etc/idds/rest/test.wsgi - WSGISocketPrefix /var/log/idds/wsgisocks/wsgi + WSGISocketPrefix /var/idds/wsgisocks/wsgi WSGIPassAuthorization On @@ -87,7 +87,7 @@ Alias "/monitor" "/opt/idds/monitor/data" GridSiteDNlists /etc/grid-security/dn-lists/ GridSiteGSIProxyLimit 16 GridSiteEnvs on - # GridSiteACLPath /opt/idds/etc/idds/rest/gacl + GridSiteACLPath /opt/idds/etc/idds/rest/gacl # GridSiteMethods GET diff --git a/main/config_default/idds.cfg b/main/config_default/idds.cfg index abc2370e..2a5b3d8f 100755 --- a/main/config_default/idds.cfg +++ b/main/config_default/idds.cfg @@ -26,15 +26,15 @@ agents = clerk, transformer, carrier, conductor [clerk] num_threads = 4 -poll_time_period = 120 -poll_operation_time_period = 120 -retrieve_bulk_size = 1 +poll_time_period = 60 +poll_operation_time_period = 60 +retrieve_bulk_size = 4 pending_time = 4 [transformer] num_threads = 8 poll_time_period = 60 -retrieve_bulk_size = 1 +retrieve_bulk_size = 4 poll_operation_time_period = 240 message_bulk_size = 1000 @@ -45,7 +45,7 @@ domapandawork.num_retries = 0 num_threads = 8 poll_time_period = 60 poll_operation_time_period = 240 -retrieve_bulk_size = 3 +retrieve_bulk_size = 4 message_bulk_size = 1000 diff --git a/main/config_default/supervisord_idds.ini b/main/config_default/supervisord_idds.ini index 3a4a75a4..30d7e2f4 100644 --- a/main/config_default/supervisord_idds.ini +++ b/main/config_default/supervisord_idds.ini @@ -7,7 +7,7 @@ environment = ;command=/opt/idds/bin/run-idds command=bash -c "source /etc/profile.d/conda.sh && conda activate /opt/idds && /opt/idds/bin/run-idds" process_name=%(process_num)02d -user=atlpan +# user=atlpan childlogdir=/var/log/idds stdout_logfile=/var/log/idds/%(program_name)s-stdout.log stderr_logfile=/var/log/idds/%(program_name)s-stderr.log diff --git a/main/etc/idds/rest/ssl.conf b/main/etc/idds/rest/ssl.conf new file mode 100644 index 00000000..e713647c --- /dev/null +++ b/main/etc/idds/rest/ssl.conf @@ -0,0 +1,55 @@ +# +# When we also provide SSL we have to listen to the +# the HTTPS port in addition. +# +# Listen 443 https + +## +## SSL Global Context +## +## All SSL configuration in this context applies both to +## the main server and all SSL-enabled virtual hosts. +## + +# Pass Phrase Dialog: +# Configure the pass phrase gathering process. +# The filtering dialog program (`builtin' is a internal +# terminal dialog) has to provide the pass phrase on stdout. +SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +# Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism +# to use and second the expiring timeout (in seconds). +SSLSessionCache shmcb:/run/httpd/sslcache(512000) +SSLSessionCacheTimeout 300 + +# Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the +# SSL library. The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +# +# Use "SSLCryptoDevice" to enable any supported hardware +# accelerators. Use "openssl engine -v" to list supported +# engine names. NOTE: If you enable an accelerator and the +# server does not start, consult the error logs and ensure +# your accelerator is functioning properly. +# +SSLCryptoDevice builtin +#SSLCryptoDevice ubsec + +## +## SSL Virtual Host Context +## + diff --git a/main/etc/rucio.cfg.default b/main/etc/rucio.cfg.default new file mode 100644 index 00000000..d7045960 --- /dev/null +++ b/main/etc/rucio.cfg.default @@ -0,0 +1,20 @@ +# Copyright European Organization for Nuclear Research (CERN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Vincent Garonne, , 2013 + +[common] + +[client] +rucio_host = https://voatlasrucio-server-prod.cern.ch:443 +auth_host = https://voatlasrucio-auth-prod.cern.ch:443 +ca_cert = $RUCIO_HOME/etc/ca.crt +client_cert = ~/.globus/usercert.pem +client_key = ~/.globus/userkey.pem +client_x509_proxy = $X509_USER_PROXY +auth_type = x509_proxy +request_retries = 3 diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql new file mode 100644 index 00000000..b6f5e9c5 --- /dev/null +++ b/main/etc/sql/postgresql.sql @@ -0,0 +1,6 @@ +CREATE USER doma_idds_r WITH PASSWORD 'Tiaroa4dr_idds'; +GRANT CONNECT ON DATABASE doma_idds TO doma_idds_r; +GRANT USAGE ON SCHEMA doma_idds TO doma_idds_r; +GRANT SELECT ON ALL TABLES IN SCHEMA doma_idds TO doma_idds_r; +ALTER DEFAULT PRIVILEGES IN SCHEMA doma_idds GRANT SELECT ON TABLES TO doma_idds_r; + diff --git a/main/lib/idds/agents/conductor/plugins/messaging.py b/main/lib/idds/agents/conductor/plugins/messaging.py index 1652f84d..cce41a02 100644 --- a/main/lib/idds/agents/conductor/plugins/messaging.py +++ b/main/lib/idds/agents/conductor/plugins/messaging.py @@ -104,7 +104,6 @@ def connect_to_messaging_brokers(self): vhost=self.vhost, keepalive=True, timeout=self.broker_timeout) - conn.set_listener('message-sender', MessagingListener(conn.transport._Transport__host_and_ports[0])) self.conns.append(conn) def send_message(self, msg): diff --git a/main/lib/idds/core/processings.py b/main/lib/idds/core/processings.py index 218652cd..4b39c738 100644 --- a/main/lib/idds/core/processings.py +++ b/main/lib/idds/core/processings.py @@ -138,7 +138,8 @@ def get_processings_with_messaging(locking=False, bulk_size=None, session=None): @transactional_session -def get_processings_by_status(status, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, with_messaging=False, session=None): +def get_processings_by_status(status, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, + with_messaging=False, for_poller=False, session=None): """ Get processing or raise a NoObject exception. @@ -163,12 +164,14 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size # then select with locking. proc_ids = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size * 2, to_json=False, locking_for_update=False, - by_substatus=by_substatus, only_return_id=True, session=session) + by_substatus=by_substatus, only_return_id=True, + for_poller=for_poller, session=session) if proc_ids: processing2s = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, processing_ids=proc_ids, bulk_size=None, to_json=to_json, locking_for_update=True, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, + for_poller=for_poller, session=session) if processing2s: # reqs = req2s[:bulk_size] # order requests @@ -188,7 +191,7 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size else: processings = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, locking_for_update=locking, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, for_poller=for_poller, session=session) parameters = {'locking': ProcessingLocking.Locking} for processing in processings: @@ -196,7 +199,7 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size else: processings = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, for_poller=for_poller, session=session) return processings diff --git a/main/lib/idds/orm/base/models.py b/main/lib/idds/orm/base/models.py index 0c9f957d..f35e557b 100644 --- a/main/lib/idds/orm/base/models.py +++ b/main/lib/idds/orm/base/models.py @@ -356,6 +356,7 @@ class Processing(BASE, ModelBase): created_at = Column("created_at", DateTime, default=datetime.datetime.utcnow) updated_at = Column("updated_at", DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) next_poll_at = Column("next_poll_at", DateTime, default=datetime.datetime.utcnow) + poller_updated_at = Column("poller_updated_at", DateTime, default=datetime.datetime.utcnow) submitted_at = Column("submitted_at", DateTime) finished_at = Column("finished_at", DateTime) expired_at = Column("expired_at", DateTime) @@ -561,7 +562,8 @@ def register_models(engine): models = (Request, Transform, Processing, Collection, Content, Health, Message) for model in models: - model.metadata.create_all(engine) # pylint: disable=maybe-no-member + if not engine.has_table(model.__tablename__, model.metadata.schema): + model.metadata.create_all(engine) # pylint: disable=maybe-no-member def unregister_models(engine): diff --git a/main/lib/idds/orm/base/utils.py b/main/lib/idds/orm/base/utils.py index 6391e4f7..d3ffe4a8 100644 --- a/main/lib/idds/orm/base/utils.py +++ b/main/lib/idds/orm/base/utils.py @@ -34,7 +34,7 @@ def build_database(echo=True, tests=False): if config_has_option('database', 'schema'): schema = config_get('database', 'schema') - if schema: + if schema and not engine.dialect.has_schema(engine, schema): print('Schema set in config, trying to create schema:', schema) try: engine.execute(CreateSchema(schema)) diff --git a/main/lib/idds/orm/processings.py b/main/lib/idds/orm/processings.py index 1f4fd3ce..d7130544 100644 --- a/main/lib/idds/orm/processings.py +++ b/main/lib/idds/orm/processings.py @@ -202,7 +202,8 @@ def get_processings_by_transform_id(transform_id=None, to_json=False, session=No @transactional_session def get_processings_by_status(status, period=None, processing_ids=[], locking=False, locking_for_update=False, - bulk_size=None, submitter=None, to_json=False, by_substatus=False, only_return_id=False, session=None): + bulk_size=None, submitter=None, to_json=False, by_substatus=False, only_return_id=False, + for_poller=False, session=None): """ Get processing or raise a NoObject exception. @@ -248,7 +249,9 @@ def get_processings_by_status(status, period=None, processing_ids=[], locking=Fa if submitter: query = query.filter(models.Processing.submitter == submitter) - if locking_for_update: + if for_poller: + query = query.order_by(asc(models.Processing.poller_updated_at)) + elif locking_for_update: query = query.with_for_update(skip_locked=True) else: query = query.order_by(asc(models.Processing.updated_at)) diff --git a/main/lib/idds/rest/v1/app.py b/main/lib/idds/rest/v1/app.py index 5e8974e2..7c979085 100644 --- a/main/lib/idds/rest/v1/app.py +++ b/main/lib/idds/rest/v1/app.py @@ -83,6 +83,10 @@ def generate_failed_auth_response(exc_msg=None): def before_request_auth(): + print("envs") + print(flask.request.environ) + print("headers") + print(flask.request.headers) auth_type = flask.request.headers.get('X-IDDS-Auth-Type', default='x509_proxy') vo = flask.request.headers.get('X-IDDS-Auth-VO', default=None) if auth_type in ['x509_proxy']: diff --git a/main/lib/idds/tests/auth_test_script.py b/main/lib/idds/tests/auth_test_script.py new file mode 100644 index 00000000..ba3715a7 --- /dev/null +++ b/main/lib/idds/tests/auth_test_script.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 - 2022 + + +""" +Test authentication. +""" + +try: + from urllib import urlencode # noqa F401 +except ImportError: + from urllib.parse import urlencode # noqa F401 + raw_input = input + +import datetime +import sys +import time + +import unittest2 as unittest +# from nose.tools import assert_equal +from idds.common.utils import setup_logging +from idds.common.authentication import OIDCAuthentication + + +setup_logging(__name__) + + +class TestAuthentication: + + def test_oidc_authentication(self): + vo = 'iamdev' + + oidc = OIDCAuthentication() + allow_vos = oidc.get_allow_vos() + print("allow_vos") + print(allow_vos) + assert(vo in allow_vos) + auth_config = oidc.get_auth_config(vo) + print("auth_config") + print(auth_config) + assert('vo' in auth_config) + assert(auth_config['vo'] == vo) + + endpoint_config = oidc.get_endpoint_config(auth_config) + print("endpoint_config") + print(endpoint_config) + assert('token_endpoint' in endpoint_config) + + status, sign_url = oidc.get_oidc_sign_url(vo) + print("sign_url") + print(sign_url) + assert('user_code' in sign_url) + print(("Please go to {0} and sign in. " + "Waiting until authentication is completed").format(sign_url['verification_uri_complete'])) + + print('Ready to get ID token?') + while True: + sys.stdout.write("[y/n] \n") + choice = raw_input().lower() + if choice == 'y': + break + elif choice == 'n': + print('aborted') + return + + if 'interval' in sign_url: + interval = sign_url['interval'] + else: + interval = 5 + + if 'expires_in' in sign_url: + expires_in = sign_url['expires_in'] + else: + expires_in = 60 + + token = None + start_time = datetime.datetime.utcnow() + while datetime.datetime.utcnow() - start_time < datetime.timedelta(seconds=expires_in): + try: + status, output = oidc.get_id_token(vo, sign_url['device_code']) + if status: + # print(output) + token = output + break + else: + if type(output) in [dict] and 'error' in output and output['error'] == 'authorization_pending': + time.sleep(interval) + else: + print(output) + break + except Exception as error: + print(error) + break + + if not token: + print("Failed to get a token") + else: + print(token) + assert('id_token' in token) + + status, new_token = oidc.refresh_id_token(vo, token['refresh_token']) + # print(new_token) + assert('id_token' in new_token) + + print("verifying the token") + status, decoded_token, username = oidc.verify_id_token(vo, token['id_token']) + if not status: + print("Failed to verify the token: %s" % decoded_token) + else: + print(username) + print(decoded_token) + + +if __name__ == '__main__': + test = TestAuthentication() + test.test_oidc_authentication() diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 78479bf9..5cbb787d 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -110,7 +110,7 @@ def show_works(req): # reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True) -reqs = get_requests(request_id=1098, with_request=True, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=1687, with_request=True, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/find_dependencies.py b/main/lib/idds/tests/find_dependencies.py new file mode 100644 index 00000000..18f94e35 --- /dev/null +++ b/main/lib/idds/tests/find_dependencies.py @@ -0,0 +1,75 @@ +import sys +import datetime + +from idds.common.utils import json_dumps # noqa F401 +from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking # noqa F401 +from idds.core.requests import get_requests # noqa F401 +from idds.core.messages import retrieve_messages # noqa F401 +from idds.core.transforms import get_transforms # noqa F401 +from idds.core.workprogress import get_workprogresses # noqa F401 +from idds.core.processings import get_processings # noqa F401 +from idds.core import transforms as core_transforms # noqa F401 +from idds.core import catalog as core_catalog # noqa F401 +from idds.orm.contents import get_input_contents +from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401 + + +def get_input_dep(contents, request_id, transform_id, coll_id, scope, name): + # print(request_id, transform_id, coll_id, scope, name) + map_id = None + for content in contents: + if content['transform_id'] == transform_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # print(content) + print("output name: %s, status: %s" % (content['name'], content['status'])) + map_id = content['map_id'] + # elif coll_id and content['coll_id'] == coll_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # # print(content) + # map_id = content['map_id'] + + print(map_id) + deps = [] + for content in contents: + if content['transform_id'] == transform_id and content['map_id'] == map_id and content['content_relation_type'] == ContentRelationType.InputDependency: + # print(content) + print("Input dependency name: %s, status: %s" % (content['name'], content['status'])) + deps.append({'request_id': content['request_id'], + 'transform_id': content['transform_id'], + 'coll_id': content['coll_id'], + 'scope': content['scope'], + 'name': content['name']}) + return deps + + +def get_transform_id(collections, coll_id): + for coll in collections: + if coll['coll_id'] == coll_id: + return coll['transform_id'] + + +def get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=1): + deps = get_input_dep(contents, request_id, transform_id, coll_id, scope, name) + print("Step: %s" % step) + print("(%s, %s, %s, %s, %s) depents on %s" % (request_id, transform_id, coll_id, scope, name, deps)) + step += 1 + for dep in deps: + coll_id = dep['coll_id'] + transform_id = get_transform_id(collections, coll_id) + dep['transform_id'] = transform_id + get_dep_link(collections=collections, contents=contents, step=step, **dep) + + +def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1): + collections = core_catalog.get_collections(request_id=request_id) + contents = core_catalog.get_contents(request_id=request_id) + get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=step) + + +if __name__ == '__main__': + request_id = 1689 + transform_id = 14713 + coll_id = None + scope = 'pseudo_dataset' + name = '94248742-a255-4541-ab38-ab69364a4c88_transformPreSourceTable_23224_72+qgraphNodeId:94248742-a255-4541-ab38-ab69364a4c88+qgraphId:1657127232.749359-46373' + + # get_input_dep(request_id, transform_id, scope, name) + get_dep_links(request_id, transform_id, coll_id, scope, name) diff --git a/main/lib/idds/tests/test_domapanda_workflow.py b/main/lib/idds/tests/test_domapanda_workflow.py index be0b7347..119c33ee 100644 --- a/main/lib/idds/tests/test_domapanda_workflow.py +++ b/main/lib/idds/tests/test_domapanda_workflow.py @@ -14,9 +14,12 @@ Test client. """ +import logging import string import random +logging.basicConfig(level=logging.DEBUG) + # import traceback # from rucio.client.client import Client as Rucio_Client @@ -130,7 +133,8 @@ def setup_workflow(): "token": "local", "type": "template", "value": "log.tgz"}, - task_cloud='LSST') + task_cloud='LSST', + task_priority=None) work2 = DomaPanDAWork(executable='echo', primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#2'}, output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#2'}], diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index b7666c1f..9a134b7c 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -34,14 +34,14 @@ def migrate(): cm1 = ClientManager(host=doma_host) # reqs = cm1.get_requests(request_id=290) # old_request_id = 298163 - old_request_id = 901 + old_request_id = 1685 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 # for old_request_id in [200]: # noqa E115 for old_request_id in [old_request_id]: # noqa E115 # doma 183 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=doma_google_host) + cm2 = ClientManager(host=doma_host) # print(reqs) print("num requests: %s" % len(reqs)) diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 368c6bfd..5f5390ac 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -12,7 +12,7 @@ request_ids = [368, 369, 370, 371, 372, 373, 374, 375, 376] -request_ids = [902] +request_ids = [1689] for request_id in request_ids: contents = get_contents(request_id=request_id, status=ContentStatus.Available) ret_contents = {} diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 0384e5e0..7c2e8610 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/main/tools/env/config_monitor.py b/main/tools/env/config_monitor.py new file mode 100644 index 00000000..336cca99 --- /dev/null +++ b/main/tools/env/config_monitor.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2022 + + +import argparse +import logging + + +def config_api_host(conf_file_template="data/conf.js.template", conf_file='data/conf.js', hostname=None): + with open(conf_file_template, 'r') as f: + template = f.read() + template = template.format(api_host_name=hostname) + with open(conf_file, 'w') as f: + f.write(template) + + +logging.getLogger().setLevel(logging.INFO) +parser = argparse.ArgumentParser(description="config iDDS monitor") +parser.add_argument('-s', '--source', default=None, help='Source config file path') +parser.add_argument('-d', '--destination', default=None, help='Destination file path') +parser.add_argument('--host', default=None, help='idds host name') +args = parser.parse_args() + + +config_api_host(conf_file_template=args.source, conf_file=args.destination, hostname=args.host) diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 6e196d31..2a964de1 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.0 - - idds-workflow==0.11.0 - - idds-client==0.11.0 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 + - idds-client==0.11.5 \ No newline at end of file diff --git a/main/tools/env/install_packages.sh b/main/tools/env/install_packages.sh new file mode 100755 index 00000000..70d615b8 --- /dev/null +++ b/main/tools/env/install_packages.sh @@ -0,0 +1,6 @@ +$'#!/bin/bash +set -m +for package in common main client workflow doma atlas website monitor ; +do + python3 -m pip install `ls $package/dist/*.tar.gz` +done diff --git a/main/tools/env/setup_dev.sh b/main/tools/env/setup_dev.sh index d2294598..e949e71e 100644 --- a/main/tools/env/setup_dev.sh +++ b/main/tools/env/setup_dev.sh @@ -25,5 +25,5 @@ conda activate $CondaDir export RUCIO_HOME=$RootDir export RUCIO_ACCOUNT=ddmadmin export X509_USER_PROXY=/tmp/x509up_u23959 -export PYTHONPATH=$PYTHONPATH:/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/current/lib/python3.6/site-packages/ +# export PYTHONPATH=$PYTHONPATH:/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/current/lib/python3.6/site-packages/ diff --git a/main/tools/env/setup_panda.sh b/main/tools/env/setup_panda.sh index f16611a3..21aa246f 100644 --- a/main/tools/env/setup_panda.sh +++ b/main/tools/env/setup_panda.sh @@ -1,11 +1,29 @@ #!/bin/bash -export PANDA_AUTH=oidc -export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda -export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda -export PANDAMON_URL=https://panda-doma.cern.ch -export PANDA_AUTH_VO=panda_dev +instance=doma +if [ "$#" -eq 1 ]; then + instance=$1 +fi -# export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ -export PANDA_CONFIG_ROOT=~/.panda/ +export PANDA_BEHIND_REAL_LB=true +# export PANDA_SYS=/opt/idds/ +if [ "$instance" == "k8s" ]; then + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda + export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + export PANDA_AUTH_VO=panda_dev + + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ +else + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda + export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + export PANDA_AUTH_VO=panda_dev + + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ +fi diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 3b39994b..f8739468 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus776.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus776.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus776.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 0384e5e0..7c2e8610 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/start-daemon.sh b/start-daemon.sh index 1c7828b9..df839ffe 100755 --- a/start-daemon.sh +++ b/start-daemon.sh @@ -9,10 +9,12 @@ export IDDS_HOME=/opt/idds if [ -f /etc/grid-security/hostkey.pem ]; then echo "host certificate is already created." + chmod 600 /etc/grid-security/hostkey.pem elif [ -f /opt/idds/configmap/hostkey.pem ]; then echo "mount /opt/idds/configmap/hostkey.pem to /etc/grid-security/hostkey.pem" ln -fs /opt/idds/configmap/hostkey.pem /etc/grid-security/hostkey.pem ln -fs /opt/idds/configmap/hostcert.pem /etc/grid-security/hostcert.pem + chmod 600 /etc/grid-security/hostkey.pem fi if [ -f /opt/idds/config/idds/idds.cfg ]; then @@ -82,7 +84,6 @@ if [ -f /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf ]; then echo "httpd conf already mounted." else echo "httpd conf not found. will use the default one." - sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf cp /opt/idds/config_default/httpd-idds-443-py39-cc7.conf /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf fi @@ -103,6 +104,7 @@ else -out /opt/idds/config/hostcert.pem ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem + chmod 600 /etc/grid-security/hostkey.pem fi mkdir -p /opt/idds/config/.panda/ @@ -131,11 +133,16 @@ if [ ! -z "$IDDS_PRINT_CFG" ]; then echo "" fi -sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf # create database if not exists python /opt/idds/tools/env/create_database.py python /opt/idds/tools/env/config_monitor.py -s ${IDDS_HOME}/monitor/data/conf.js.template -d ${IDDS_HOME}/monitor/data/conf.js --host ${IDDS_SERVER} -ln -s /opt/idds/configmap/idds2panda_token /opt/idds/config/.token + +if ! [ -f /opt/idds/config/.token ]; then + echo "/opt/idds/config/.token does not exist." + if [ -f /opt/idds/configmap/idds2panda_token ]; then + ln -s /opt/idds/configmap/idds2panda_token /opt/idds/config/.token + fi +fi if [ "${IDDS_SERVICE}" == "rest" ]; then echo "starting iDDS ${IDDS_SERVICE} service" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 0384e5e0..7c2e8610 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 0384e5e0..7c2e8610 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.5" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index bffb8864..a3eb0093 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.0 \ No newline at end of file + - idds-common==0.11.5 \ No newline at end of file