From 944b8da17dff1e8e5b8e4fa06f64dd30bbba306e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 28 May 2022 00:25:34 +0200 Subject: [PATCH 01/25] #76 fix idds client base class to set default x509 path --- client/lib/idds/client/base.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/client/lib/idds/client/base.py b/client/lib/idds/client/base.py index 1a4efffd..2f24f1e8 100644 --- a/client/lib/idds/client/base.py +++ b/client/lib/idds/client/base.py @@ -26,7 +26,7 @@ from idds.common import exceptions from idds.common.constants import HTTP_STATUS_CODE -from idds.common.utils import json_dumps, json_loads +from idds.common.utils import json_dumps, json_loads, get_proxy_path from idds.common.authentication import OIDCAuthenticationUtils @@ -68,6 +68,20 @@ def __init__(self, host=None, auth=None, timeout=None, client_proxy=None): self.check_auth() + def get_user_proxy(sellf): + """ + Get the user proxy. + + :returns: the path of the user proxy. + """ + + client_proxy = get_proxy_path() + + if not client_proxy or not os.path.exists(client_proxy): + raise exceptions.RestException("Cannot find a valid x509 proxy.") + + return client_proxy + def check_auth(self): """ To check whether the auth type is supported and the input for the auth is available. @@ -77,6 +91,8 @@ def check_auth(self): self.auth_type = 'x509_proxy' if self.auth_type in ['x509_proxy']: + if not self.client_proxy: + self.client_proxy = self.get_user_proxy() if not self.client_proxy or not os.path.exists(self.client_proxy): raise exceptions.RestException("Cannot find a valid x509 proxy.") elif self.auth_type in ['oidc']: From e38bafec8bf6b2739c853115991949768c2179dd Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 18 Jun 2022 09:34:47 +0200 Subject: [PATCH 02/25] add pandacache url --- atlas/lib/idds/atlas/version.py | 2 +- .../idds/atlas/workflowv2/atlaspandawork.py | 29 +++++++++++++++++ atlas/tools/env/environment.yml | 4 +-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 +-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- .../lib/idds/doma/workflowv2/domapandawork.py | 14 ++++++++ doma/tools/env/environment.yml | 4 +-- .../httpd-idds-443-py39-cc7.conf | 10 +++--- main/config_default/idds.cfg | 10 +++--- .../agents/conductor/plugins/messaging.py | 1 - main/lib/idds/core/processings.py | 13 +++++--- main/lib/idds/orm/base/models.py | 4 ++- main/lib/idds/orm/base/utils.py | 2 +- main/lib/idds/orm/processings.py | 7 ++-- main/lib/idds/rest/v1/app.py | 4 +++ main/lib/idds/tests/core_tests.py | 2 +- .../lib/idds/tests/test_domapanda_workflow.py | 6 +++- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 ++-- main/tools/env/setup_dev.sh | 2 +- main/tools/env/setup_panda.sh | 32 +++++++++++++++---- monitor/data/conf.js | 12 +++---- monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 28 files changed, 130 insertions(+), 54 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 0384e5e0..52bf8dd8 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py index bcd505bb..07fdf512 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -117,6 +117,23 @@ def load_panda_urls(self): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if panda_config.has_option('panda', 'pandacache_url'): + self.pandacache_url = panda_config.get('panda', 'pandacache_url') + os.environ['PANDACACHE_URL'] = self.pandacache_url + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if panda_config.has_option('panda', 'panda_verify_host'): + self.panda_verify_host = panda_config.get('panda', 'panda_verify_host') + os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) + if panda_config.has_option('panda', 'panda_auth'): + self.panda_auth = panda_config.get('panda', 'panda_auth') + os.environ['PANDA_AUTH'] = self.panda_auth + if panda_config.has_option('panda', 'panda_auth_vo'): + self.panda_auth_vo = panda_config.get('panda', 'panda_auth_vo') + os.environ['PANDA_AUTH_VO'] = self.panda_auth_vo + if panda_config.has_option('panda', 'panda_config_root'): + self.panda_config_root = panda_config.get('panda', 'panda_config_root') + os.environ['PANDA_CONFIG_ROOT'] = self.panda_config_root if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']: self.panda_monitor = os.environ['PANDA_MONITOR_URL'] @@ -127,6 +144,18 @@ def load_panda_urls(self): if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']: + self.pandacache_url = os.environ['PANDACACHE_URL'] + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']: + self.panda_verify_host = os.environ['PANDA_VERIFY_HOST'] + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) + if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']: + self.panda_auth = os.environ['PANDA_AUTH'] + if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']: + self.panda_auth_vo = os.environ['PANDA_AUTH_VO'] + if not self.panda_config_root and 'PANDA_CONFIG_ROOT' in os.environ and os.environ['PANDA_CONFIG_ROOT']: + self.panda_config_root = os.environ['PANDA_CONFIG_ROOT'] def set_agent_attributes(self, attrs, req_attributes=None): if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 077379cd..9151aec6 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.0 - - idds-workflow==0.11.0 \ No newline at end of file + - idds-common==0.11.1 + - idds-workflow==0.11.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 0384e5e0..52bf8dd8 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 51b4c3f8..c916c00e 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.0 - - idds-workflow==0.11.0 + - idds-common==0.11.1 + - idds-workflow==0.11.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 0384e5e0..52bf8dd8 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 8e5196ff..474b7ac8 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index e8bc1fbf..5452c8c9 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -150,6 +150,14 @@ def load_panda_urls(self): self.panda_url = panda_config.get('panda', 'panda_url') os.environ['PANDA_URL'] = self.panda_url # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if panda_config.has_option('panda', 'pandacache_url'): + self.pandacache_url = panda_config.get('panda', 'pandacache_url') + os.environ['PANDACACHE_URL'] = self.pandacache_url + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if panda_config.has_option('panda', 'panda_verify_host'): + self.panda_verify_host = panda_config.get('panda', 'panda_verify_host') + os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) if panda_config.has_option('panda', 'panda_url_ssl'): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl @@ -173,6 +181,12 @@ def load_panda_urls(self): if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']: + self.pandacache_url = os.environ['PANDACACHE_URL'] + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']: + self.panda_verify_host = os.environ['PANDA_VERIFY_HOST'] + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']: self.panda_auth = os.environ['PANDA_AUTH'] if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']: diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 61559a5d..7ed7bbfc 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.0 - - idds-workflow==0.11.0 \ No newline at end of file + - idds-common==0.11.1 + - idds-workflow==0.11.1 \ No newline at end of file diff --git a/main/config_default/httpd-idds-443-py39-cc7.conf b/main/config_default/httpd-idds-443-py39-cc7.conf index db09cc6b..abb0876e 100644 --- a/main/config_default/httpd-idds-443-py39-cc7.conf +++ b/main/config_default/httpd-idds-443-py39-cc7.conf @@ -34,11 +34,11 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages WSGIApplicationGroup %GLOBAL WSGIScriptAlias /idds /opt/idds/bin/idds.wsgi # WSGIScriptAliasMatch ^/idds/(.+)$ /opt/idds/etc/idds/rest/test.wsgi - WSGISocketPrefix /var/log/idds/wsgisocks/wsgi + WSGISocketPrefix /var/idds/wsgisocks/wsgi WSGIPassAuthorization On -Listen 8443 +Listen 443 RewriteEngine on RewriteCond %REQUEST_METHOD ^(TRACE|TRACK) @@ -49,8 +49,8 @@ TraceEnable off Alias "/website" "/opt/idds/website/data" Alias "/monitor" "/opt/idds/monitor/data" - - # ServerName aipanda182.cern.ch:8443 + + # ServerName aipanda182.cern.ch:443 ServerAdmin wguan@cern.ch SSLEngine on @@ -87,7 +87,7 @@ Alias "/monitor" "/opt/idds/monitor/data" GridSiteDNlists /etc/grid-security/dn-lists/ GridSiteGSIProxyLimit 16 GridSiteEnvs on - # GridSiteACLPath /opt/idds/etc/idds/rest/gacl + GridSiteACLPath /opt/idds/etc/idds/rest/gacl # GridSiteMethods GET diff --git a/main/config_default/idds.cfg b/main/config_default/idds.cfg index abc2370e..2a5b3d8f 100755 --- a/main/config_default/idds.cfg +++ b/main/config_default/idds.cfg @@ -26,15 +26,15 @@ agents = clerk, transformer, carrier, conductor [clerk] num_threads = 4 -poll_time_period = 120 -poll_operation_time_period = 120 -retrieve_bulk_size = 1 +poll_time_period = 60 +poll_operation_time_period = 60 +retrieve_bulk_size = 4 pending_time = 4 [transformer] num_threads = 8 poll_time_period = 60 -retrieve_bulk_size = 1 +retrieve_bulk_size = 4 poll_operation_time_period = 240 message_bulk_size = 1000 @@ -45,7 +45,7 @@ domapandawork.num_retries = 0 num_threads = 8 poll_time_period = 60 poll_operation_time_period = 240 -retrieve_bulk_size = 3 +retrieve_bulk_size = 4 message_bulk_size = 1000 diff --git a/main/lib/idds/agents/conductor/plugins/messaging.py b/main/lib/idds/agents/conductor/plugins/messaging.py index 1652f84d..cce41a02 100644 --- a/main/lib/idds/agents/conductor/plugins/messaging.py +++ b/main/lib/idds/agents/conductor/plugins/messaging.py @@ -104,7 +104,6 @@ def connect_to_messaging_brokers(self): vhost=self.vhost, keepalive=True, timeout=self.broker_timeout) - conn.set_listener('message-sender', MessagingListener(conn.transport._Transport__host_and_ports[0])) self.conns.append(conn) def send_message(self, msg): diff --git a/main/lib/idds/core/processings.py b/main/lib/idds/core/processings.py index 218652cd..4b39c738 100644 --- a/main/lib/idds/core/processings.py +++ b/main/lib/idds/core/processings.py @@ -138,7 +138,8 @@ def get_processings_with_messaging(locking=False, bulk_size=None, session=None): @transactional_session -def get_processings_by_status(status, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, with_messaging=False, session=None): +def get_processings_by_status(status, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, + with_messaging=False, for_poller=False, session=None): """ Get processing or raise a NoObject exception. @@ -163,12 +164,14 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size # then select with locking. proc_ids = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size * 2, to_json=False, locking_for_update=False, - by_substatus=by_substatus, only_return_id=True, session=session) + by_substatus=by_substatus, only_return_id=True, + for_poller=for_poller, session=session) if proc_ids: processing2s = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, processing_ids=proc_ids, bulk_size=None, to_json=to_json, locking_for_update=True, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, + for_poller=for_poller, session=session) if processing2s: # reqs = req2s[:bulk_size] # order requests @@ -188,7 +191,7 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size else: processings = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, locking_for_update=locking, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, for_poller=for_poller, session=session) parameters = {'locking': ProcessingLocking.Locking} for processing in processings: @@ -196,7 +199,7 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size else: processings = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, for_poller=for_poller, session=session) return processings diff --git a/main/lib/idds/orm/base/models.py b/main/lib/idds/orm/base/models.py index 0c9f957d..f35e557b 100644 --- a/main/lib/idds/orm/base/models.py +++ b/main/lib/idds/orm/base/models.py @@ -356,6 +356,7 @@ class Processing(BASE, ModelBase): created_at = Column("created_at", DateTime, default=datetime.datetime.utcnow) updated_at = Column("updated_at", DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) next_poll_at = Column("next_poll_at", DateTime, default=datetime.datetime.utcnow) + poller_updated_at = Column("poller_updated_at", DateTime, default=datetime.datetime.utcnow) submitted_at = Column("submitted_at", DateTime) finished_at = Column("finished_at", DateTime) expired_at = Column("expired_at", DateTime) @@ -561,7 +562,8 @@ def register_models(engine): models = (Request, Transform, Processing, Collection, Content, Health, Message) for model in models: - model.metadata.create_all(engine) # pylint: disable=maybe-no-member + if not engine.has_table(model.__tablename__, model.metadata.schema): + model.metadata.create_all(engine) # pylint: disable=maybe-no-member def unregister_models(engine): diff --git a/main/lib/idds/orm/base/utils.py b/main/lib/idds/orm/base/utils.py index 6391e4f7..d3ffe4a8 100644 --- a/main/lib/idds/orm/base/utils.py +++ b/main/lib/idds/orm/base/utils.py @@ -34,7 +34,7 @@ def build_database(echo=True, tests=False): if config_has_option('database', 'schema'): schema = config_get('database', 'schema') - if schema: + if schema and not engine.dialect.has_schema(engine, schema): print('Schema set in config, trying to create schema:', schema) try: engine.execute(CreateSchema(schema)) diff --git a/main/lib/idds/orm/processings.py b/main/lib/idds/orm/processings.py index 1f4fd3ce..d7130544 100644 --- a/main/lib/idds/orm/processings.py +++ b/main/lib/idds/orm/processings.py @@ -202,7 +202,8 @@ def get_processings_by_transform_id(transform_id=None, to_json=False, session=No @transactional_session def get_processings_by_status(status, period=None, processing_ids=[], locking=False, locking_for_update=False, - bulk_size=None, submitter=None, to_json=False, by_substatus=False, only_return_id=False, session=None): + bulk_size=None, submitter=None, to_json=False, by_substatus=False, only_return_id=False, + for_poller=False, session=None): """ Get processing or raise a NoObject exception. @@ -248,7 +249,9 @@ def get_processings_by_status(status, period=None, processing_ids=[], locking=Fa if submitter: query = query.filter(models.Processing.submitter == submitter) - if locking_for_update: + if for_poller: + query = query.order_by(asc(models.Processing.poller_updated_at)) + elif locking_for_update: query = query.with_for_update(skip_locked=True) else: query = query.order_by(asc(models.Processing.updated_at)) diff --git a/main/lib/idds/rest/v1/app.py b/main/lib/idds/rest/v1/app.py index 5e8974e2..7c979085 100644 --- a/main/lib/idds/rest/v1/app.py +++ b/main/lib/idds/rest/v1/app.py @@ -83,6 +83,10 @@ def generate_failed_auth_response(exc_msg=None): def before_request_auth(): + print("envs") + print(flask.request.environ) + print("headers") + print(flask.request.headers) auth_type = flask.request.headers.get('X-IDDS-Auth-Type', default='x509_proxy') vo = flask.request.headers.get('X-IDDS-Auth-VO', default=None) if auth_type in ['x509_proxy']: diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 78479bf9..e64603b7 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -110,7 +110,7 @@ def show_works(req): # reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True) -reqs = get_requests(request_id=1098, with_request=True, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=1439, with_request=True, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/test_domapanda_workflow.py b/main/lib/idds/tests/test_domapanda_workflow.py index be0b7347..119c33ee 100644 --- a/main/lib/idds/tests/test_domapanda_workflow.py +++ b/main/lib/idds/tests/test_domapanda_workflow.py @@ -14,9 +14,12 @@ Test client. """ +import logging import string import random +logging.basicConfig(level=logging.DEBUG) + # import traceback # from rucio.client.client import Client as Rucio_Client @@ -130,7 +133,8 @@ def setup_workflow(): "token": "local", "type": "template", "value": "log.tgz"}, - task_cloud='LSST') + task_cloud='LSST', + task_priority=None) work2 = DomaPanDAWork(executable='echo', primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#2'}, output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#2'}], diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 0384e5e0..52bf8dd8 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 6e196d31..e5968a00 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.0 - - idds-workflow==0.11.0 - - idds-client==0.11.0 \ No newline at end of file + - idds-common==0.11.1 + - idds-workflow==0.11.1 + - idds-client==0.11.1 \ No newline at end of file diff --git a/main/tools/env/setup_dev.sh b/main/tools/env/setup_dev.sh index d2294598..e949e71e 100644 --- a/main/tools/env/setup_dev.sh +++ b/main/tools/env/setup_dev.sh @@ -25,5 +25,5 @@ conda activate $CondaDir export RUCIO_HOME=$RootDir export RUCIO_ACCOUNT=ddmadmin export X509_USER_PROXY=/tmp/x509up_u23959 -export PYTHONPATH=$PYTHONPATH:/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/current/lib/python3.6/site-packages/ +# export PYTHONPATH=$PYTHONPATH:/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/current/lib/python3.6/site-packages/ diff --git a/main/tools/env/setup_panda.sh b/main/tools/env/setup_panda.sh index f16611a3..21aa246f 100644 --- a/main/tools/env/setup_panda.sh +++ b/main/tools/env/setup_panda.sh @@ -1,11 +1,29 @@ #!/bin/bash -export PANDA_AUTH=oidc -export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda -export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda -export PANDAMON_URL=https://panda-doma.cern.ch -export PANDA_AUTH_VO=panda_dev +instance=doma +if [ "$#" -eq 1 ]; then + instance=$1 +fi -# export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ -export PANDA_CONFIG_ROOT=~/.panda/ +export PANDA_BEHIND_REAL_LB=true +# export PANDA_SYS=/opt/idds/ +if [ "$instance" == "k8s" ]; then + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda + export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + export PANDA_AUTH_VO=panda_dev + + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ +else + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda + export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + export PANDA_AUTH_VO=panda_dev + + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ +fi diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 3b39994b..410cd4a1 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus776.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus776.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus776.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus740.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus740.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus740.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 0384e5e0..52bf8dd8 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 0384e5e0..52bf8dd8 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 0384e5e0..52bf8dd8 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index bffb8864..8aa1a003 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.0 \ No newline at end of file + - idds-common==0.11.1 \ No newline at end of file From eb43f8e3a42a2629e34ed215d3663976932b3f8b Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 18 Jun 2022 09:40:07 +0200 Subject: [PATCH 03/25] new version 0.11.2 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 9151aec6..a7979520 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.1 - - idds-workflow==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index c916c00e..fc3c21c2 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.1 - - idds-workflow==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 474b7ac8..1311630b 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 7ed7bbfc..a8aedf1a 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.1 - - idds-workflow==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index e5968a00..23847f57 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.1 - - idds-workflow==0.11.1 - - idds-client==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 + - idds-client==0.11.2 \ No newline at end of file diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 8aa1a003..3a543d41 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.1 \ No newline at end of file + - idds-common==0.11.2 \ No newline at end of file From aaf3a421aaecbe7818ee2e23a45f090b41966886 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 17:34:57 +0200 Subject: [PATCH 04/25] add default for maxattempt --- doma/lib/idds/doma/workflowv2/domapandawork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index 5452c8c9..9805f961 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -89,7 +89,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.prodSourceLabel = prodSourceLabel self.task_type = task_type self.maxWalltime = maxwalltime - self.maxAttempt = maxattempt + self.maxAttempt = maxattempt if maxattempt else 5 self.core_count = core_count self.task_log = task_log From aed32d6470670073f3f5ff18c36edc08a9988731 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 17:36:29 +0200 Subject: [PATCH 05/25] update idds rest default port 8443 --- main/config_default/httpd-idds-443-py39-cc7.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/config_default/httpd-idds-443-py39-cc7.conf b/main/config_default/httpd-idds-443-py39-cc7.conf index abb0876e..ffb5b3d1 100644 --- a/main/config_default/httpd-idds-443-py39-cc7.conf +++ b/main/config_default/httpd-idds-443-py39-cc7.conf @@ -38,7 +38,7 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages WSGIPassAuthorization On -Listen 443 +Listen 8443 RewriteEngine on RewriteCond %REQUEST_METHOD ^(TRACE|TRACK) @@ -49,8 +49,8 @@ TraceEnable off Alias "/website" "/opt/idds/website/data" Alias "/monitor" "/opt/idds/monitor/data" - - # ServerName aipanda182.cern.ch:443 + + # ServerName aipanda182.cern.ch:8443 ServerAdmin wguan@cern.ch SSLEngine on From cab19fce6649cefeda0cff81a41aa4618eeba25a Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 17:40:20 +0200 Subject: [PATCH 06/25] new version 0.11.3 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/data/conf.js | 12 ++++++------ monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index a7979520..35900670 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.2 - - idds-workflow==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index fc3c21c2..0fefdb0d 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.2 - - idds-workflow==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 1311630b..4d610cb1 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index a8aedf1a..5aacda7f 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.2 - - idds-workflow==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 23847f57..7291ea87 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.2 - - idds-workflow==0.11.2 - - idds-client==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 + - idds-client==0.11.3 \ No newline at end of file diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 410cd4a1..763b37e8 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus740.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus740.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus740.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus789.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus789.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus789.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 3a543d41..164c36be 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.2 \ No newline at end of file + - idds-common==0.11.3 \ No newline at end of file From 06edb66f7417126d3177b956cf5a84b498cfc8c1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 18:11:28 +0200 Subject: [PATCH 07/25] new version 0.11.4 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/config_default/supervisord_idds.ini | 2 +- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/data/conf.js | 12 ++++++------ monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 15 files changed, 25 insertions(+), 25 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 88e8d2cc..67b93aa1 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 35900670..dccd5ab3 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.3 - - idds-workflow==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 88e8d2cc..67b93aa1 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 0fefdb0d..5076e4f1 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.3 - - idds-workflow==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 88e8d2cc..67b93aa1 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 4d610cb1..e7dd71a7 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 5aacda7f..6224f7d4 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.3 - - idds-workflow==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 \ No newline at end of file diff --git a/main/config_default/supervisord_idds.ini b/main/config_default/supervisord_idds.ini index 3a4a75a4..30d7e2f4 100644 --- a/main/config_default/supervisord_idds.ini +++ b/main/config_default/supervisord_idds.ini @@ -7,7 +7,7 @@ environment = ;command=/opt/idds/bin/run-idds command=bash -c "source /etc/profile.d/conda.sh && conda activate /opt/idds && /opt/idds/bin/run-idds" process_name=%(process_num)02d -user=atlpan +# user=atlpan childlogdir=/var/log/idds stdout_logfile=/var/log/idds/%(program_name)s-stdout.log stderr_logfile=/var/log/idds/%(program_name)s-stderr.log diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 88e8d2cc..67b93aa1 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 7291ea87..a0a39f16 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.3 - - idds-workflow==0.11.3 - - idds-client==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 + - idds-client==0.11.4 \ No newline at end of file diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 763b37e8..39e1faff 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus789.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus789.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus789.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 88e8d2cc..67b93aa1 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 88e8d2cc..67b93aa1 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 88e8d2cc..67b93aa1 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 164c36be..b878a2c1 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.3 \ No newline at end of file + - idds-common==0.11.4 \ No newline at end of file From e65eb70e195ee5db07334a16d324cd68161d3df1 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 22:17:44 +0200 Subject: [PATCH 08/25] update docker file to suppor noRoot --- Dockerfile | 23 ++++++++++++++++++++++- start-daemon.sh | 13 ++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5bfccae2..48c892c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ RUN mkdir /var/idds RUN mkdir /var/idds/wsgisocks RUN chown atlpan -R /opt/idds # RUN chown atlpan -R /opt/idds_source -RUN chown atlpan /var/log/idds +RUN chown atlpan -R /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ # setup conda virtual env @@ -67,6 +67,8 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG +RUN chmod 777 /opt/idds/monitor/data +RUN chmod 777 /opt/idds/monitor/data/conf.js RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds # RUN mkdir /opt/idds/config_default @@ -93,12 +95,31 @@ RUN ln -fs /opt/idds/config/idds/auth.cfg /opt/idds/etc/idds/auth/auth.cfg RUN ln -fs /opt/idds/config/idds/gacl /opt/idds/etc/idds/rest/gacl RUN ln -fs /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf +# update http config +RUN sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf +RUN sed -i 's/Listen\ 80/#\ Listen\ 80/g' /etc/httpd/conf/httpd.conf +RUN sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf + # for idds daemons RUN ln -fs /opt/idds/config/idds/supervisord_idds.ini /etc/supervisord.d/idds.ini +RUN chmod -R 777 /opt/idds/config +RUN chmod -R 777 /var/log/idds +RUN chmod 777 /etc/grid-security +RUN chmod 777 /etc/httpd/conf.d +RUN chmod 777 /etc/httpd/conf/httpd.conf +RUN chmod 777 /etc/httpd/conf +RUN chmod 777 /run/httpd +RUN chmod 777 /var/log/supervisor/ +RUN chmod 777 /var/run/supervisor +RUN chmod 777 /var/run +RUN chmod 777 /etc/httpd/logs + ENV PATH /opt/idds/bin/:$PATH ADD start-daemon.sh /opt/idds/bin/ +RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back +ADD ssl.conf /etc/httpd/conf.d/ssl.conf VOLUME /var/log/idds VOLUME /opt/idds/config diff --git a/start-daemon.sh b/start-daemon.sh index 1c7828b9..df839ffe 100755 --- a/start-daemon.sh +++ b/start-daemon.sh @@ -9,10 +9,12 @@ export IDDS_HOME=/opt/idds if [ -f /etc/grid-security/hostkey.pem ]; then echo "host certificate is already created." + chmod 600 /etc/grid-security/hostkey.pem elif [ -f /opt/idds/configmap/hostkey.pem ]; then echo "mount /opt/idds/configmap/hostkey.pem to /etc/grid-security/hostkey.pem" ln -fs /opt/idds/configmap/hostkey.pem /etc/grid-security/hostkey.pem ln -fs /opt/idds/configmap/hostcert.pem /etc/grid-security/hostcert.pem + chmod 600 /etc/grid-security/hostkey.pem fi if [ -f /opt/idds/config/idds/idds.cfg ]; then @@ -82,7 +84,6 @@ if [ -f /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf ]; then echo "httpd conf already mounted." else echo "httpd conf not found. will use the default one." - sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf cp /opt/idds/config_default/httpd-idds-443-py39-cc7.conf /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf fi @@ -103,6 +104,7 @@ else -out /opt/idds/config/hostcert.pem ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem + chmod 600 /etc/grid-security/hostkey.pem fi mkdir -p /opt/idds/config/.panda/ @@ -131,11 +133,16 @@ if [ ! -z "$IDDS_PRINT_CFG" ]; then echo "" fi -sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf # create database if not exists python /opt/idds/tools/env/create_database.py python /opt/idds/tools/env/config_monitor.py -s ${IDDS_HOME}/monitor/data/conf.js.template -d ${IDDS_HOME}/monitor/data/conf.js --host ${IDDS_SERVER} -ln -s /opt/idds/configmap/idds2panda_token /opt/idds/config/.token + +if ! [ -f /opt/idds/config/.token ]; then + echo "/opt/idds/config/.token does not exist." + if [ -f /opt/idds/configmap/idds2panda_token ]; then + ln -s /opt/idds/configmap/idds2panda_token /opt/idds/config/.token + fi +fi if [ "${IDDS_SERVICE}" == "rest" ]; then echo "starting iDDS ${IDDS_SERVICE} service" From 3a3a177b4cefe17a08b0ddeffd5fa06e383e9a5c Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 15:31:46 +0200 Subject: [PATCH 09/25] #80 options to disable oidc components to verify ssl host certificate --- common/lib/idds/common/authentication.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/common/lib/idds/common/authentication.py b/common/lib/idds/common/authentication.py index d6336f36..1ed0a549 100644 --- a/common/lib/idds/common/authentication.py +++ b/common/lib/idds/common/authentication.py @@ -45,6 +45,12 @@ def decode_value(val): return int.from_bytes(decoded, 'big') +def should_verify(): + if os.environ.get('IDDS_AUTH_NO_VERIFY', None): + return False + return True + + class BaseAuthentication(object): def __init__(self, timeout=None): self.timeout = timeout @@ -98,7 +104,7 @@ def get_auth_config(self, vo): def get_http_content(self, url): try: - r = requests.get(url, allow_redirects=True) + r = requests.get(url, allow_redirects=True, verify=should_verify()) return r.content except Exception as error: return False, 'Failed to get http content for %s: %s' (str(url), str(error)) @@ -128,6 +134,7 @@ def get_oidc_sign_url(self, vo): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: @@ -172,6 +179,7 @@ def get_id_token(self, vo, device_code, interval=5, expires_in=60): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: if result.status_code == HTTP_STATUS_CODE.OK and result.text: @@ -203,6 +211,7 @@ def refresh_id_token(self, vo, refresh_token): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: From 4ba35d8b0ebf20deb32ebcd5001342fa6cfaf446 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 15:32:14 +0200 Subject: [PATCH 10/25] update tests --- main/lib/idds/tests/core_tests.py | 2 +- main/lib/idds/tests/test_migrate_requests.py | 4 ++-- main/lib/idds/tests/trigger_release.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index e64603b7..5cbb787d 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -110,7 +110,7 @@ def show_works(req): # reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True) -reqs = get_requests(request_id=1439, with_request=True, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=1687, with_request=True, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index b7666c1f..9a134b7c 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -34,14 +34,14 @@ def migrate(): cm1 = ClientManager(host=doma_host) # reqs = cm1.get_requests(request_id=290) # old_request_id = 298163 - old_request_id = 901 + old_request_id = 1685 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 # for old_request_id in [200]: # noqa E115 for old_request_id in [old_request_id]: # noqa E115 # doma 183 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=doma_google_host) + cm2 = ClientManager(host=doma_host) # print(reqs) print("num requests: %s" % len(reqs)) diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 368c6bfd..5f5390ac 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -12,7 +12,7 @@ request_ids = [368, 369, 370, 371, 372, 373, 374, 375, 376] -request_ids = [902] +request_ids = [1689] for request_id in request_ids: contents = get_contents(request_id=request_id, status=ContentStatus.Available) ret_contents = {} From 2509c00efb7cfcf4deda6ce809816163d72e021a Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 17:09:52 +0200 Subject: [PATCH 11/25] update docker deployment --- Dockerfile | 18 +++- main/etc/idds/rest/ssl.conf | 55 ++++++++++ main/etc/rucio.cfg.default | 20 ++++ main/etc/sql/postgresql.sql | 6 ++ main/lib/idds/tests/auth_test_script.py | 123 +++++++++++++++++++++++ main/lib/idds/tests/find_dependencies.py | 75 ++++++++++++++ main/tools/env/config_monitor.py | 32 ++++++ main/tools/env/install_packages.sh | 6 ++ monitor/data/conf.js | 12 +-- 9 files changed, 339 insertions(+), 8 deletions(-) create mode 100644 main/etc/idds/rest/ssl.conf create mode 100644 main/etc/rucio.cfg.default create mode 100644 main/etc/sql/postgresql.sql create mode 100644 main/lib/idds/tests/auth_test_script.py create mode 100644 main/lib/idds/tests/find_dependencies.py create mode 100644 main/tools/env/config_monitor.py create mode 100755 main/tools/env/install_packages.sh diff --git a/Dockerfile b/Dockerfile index 48c892c8..b0529616 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,7 +65,20 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client -RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG + + +WORKDIR /tmp/src +COPY . . + +RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ + if [[ -z "$TAG" ]] ; then \ + python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \ + else \ + python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ + fi + +WORKDIR /tmp +RUN rm -rf /tmp/src RUN chmod 777 /opt/idds/monitor/data RUN chmod 777 /opt/idds/monitor/data/conf.js @@ -119,7 +132,8 @@ ENV PATH /opt/idds/bin/:$PATH ADD start-daemon.sh /opt/idds/bin/ RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back -ADD ssl.conf /etc/httpd/conf.d/ssl.conf +# ADD ssl.conf /etc/httpd/conf.d/ssl.conf +RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf VOLUME /var/log/idds VOLUME /opt/idds/config diff --git a/main/etc/idds/rest/ssl.conf b/main/etc/idds/rest/ssl.conf new file mode 100644 index 00000000..e713647c --- /dev/null +++ b/main/etc/idds/rest/ssl.conf @@ -0,0 +1,55 @@ +# +# When we also provide SSL we have to listen to the +# the HTTPS port in addition. +# +# Listen 443 https + +## +## SSL Global Context +## +## All SSL configuration in this context applies both to +## the main server and all SSL-enabled virtual hosts. +## + +# Pass Phrase Dialog: +# Configure the pass phrase gathering process. +# The filtering dialog program (`builtin' is a internal +# terminal dialog) has to provide the pass phrase on stdout. +SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +# Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism +# to use and second the expiring timeout (in seconds). +SSLSessionCache shmcb:/run/httpd/sslcache(512000) +SSLSessionCacheTimeout 300 + +# Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the +# SSL library. The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +# +# Use "SSLCryptoDevice" to enable any supported hardware +# accelerators. Use "openssl engine -v" to list supported +# engine names. NOTE: If you enable an accelerator and the +# server does not start, consult the error logs and ensure +# your accelerator is functioning properly. +# +SSLCryptoDevice builtin +#SSLCryptoDevice ubsec + +## +## SSL Virtual Host Context +## + diff --git a/main/etc/rucio.cfg.default b/main/etc/rucio.cfg.default new file mode 100644 index 00000000..d7045960 --- /dev/null +++ b/main/etc/rucio.cfg.default @@ -0,0 +1,20 @@ +# Copyright European Organization for Nuclear Research (CERN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Vincent Garonne, , 2013 + +[common] + +[client] +rucio_host = https://voatlasrucio-server-prod.cern.ch:443 +auth_host = https://voatlasrucio-auth-prod.cern.ch:443 +ca_cert = $RUCIO_HOME/etc/ca.crt +client_cert = ~/.globus/usercert.pem +client_key = ~/.globus/userkey.pem +client_x509_proxy = $X509_USER_PROXY +auth_type = x509_proxy +request_retries = 3 diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql new file mode 100644 index 00000000..b6f5e9c5 --- /dev/null +++ b/main/etc/sql/postgresql.sql @@ -0,0 +1,6 @@ +CREATE USER doma_idds_r WITH PASSWORD 'Tiaroa4dr_idds'; +GRANT CONNECT ON DATABASE doma_idds TO doma_idds_r; +GRANT USAGE ON SCHEMA doma_idds TO doma_idds_r; +GRANT SELECT ON ALL TABLES IN SCHEMA doma_idds TO doma_idds_r; +ALTER DEFAULT PRIVILEGES IN SCHEMA doma_idds GRANT SELECT ON TABLES TO doma_idds_r; + diff --git a/main/lib/idds/tests/auth_test_script.py b/main/lib/idds/tests/auth_test_script.py new file mode 100644 index 00000000..ba3715a7 --- /dev/null +++ b/main/lib/idds/tests/auth_test_script.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 - 2022 + + +""" +Test authentication. +""" + +try: + from urllib import urlencode # noqa F401 +except ImportError: + from urllib.parse import urlencode # noqa F401 + raw_input = input + +import datetime +import sys +import time + +import unittest2 as unittest +# from nose.tools import assert_equal +from idds.common.utils import setup_logging +from idds.common.authentication import OIDCAuthentication + + +setup_logging(__name__) + + +class TestAuthentication: + + def test_oidc_authentication(self): + vo = 'iamdev' + + oidc = OIDCAuthentication() + allow_vos = oidc.get_allow_vos() + print("allow_vos") + print(allow_vos) + assert(vo in allow_vos) + auth_config = oidc.get_auth_config(vo) + print("auth_config") + print(auth_config) + assert('vo' in auth_config) + assert(auth_config['vo'] == vo) + + endpoint_config = oidc.get_endpoint_config(auth_config) + print("endpoint_config") + print(endpoint_config) + assert('token_endpoint' in endpoint_config) + + status, sign_url = oidc.get_oidc_sign_url(vo) + print("sign_url") + print(sign_url) + assert('user_code' in sign_url) + print(("Please go to {0} and sign in. " + "Waiting until authentication is completed").format(sign_url['verification_uri_complete'])) + + print('Ready to get ID token?') + while True: + sys.stdout.write("[y/n] \n") + choice = raw_input().lower() + if choice == 'y': + break + elif choice == 'n': + print('aborted') + return + + if 'interval' in sign_url: + interval = sign_url['interval'] + else: + interval = 5 + + if 'expires_in' in sign_url: + expires_in = sign_url['expires_in'] + else: + expires_in = 60 + + token = None + start_time = datetime.datetime.utcnow() + while datetime.datetime.utcnow() - start_time < datetime.timedelta(seconds=expires_in): + try: + status, output = oidc.get_id_token(vo, sign_url['device_code']) + if status: + # print(output) + token = output + break + else: + if type(output) in [dict] and 'error' in output and output['error'] == 'authorization_pending': + time.sleep(interval) + else: + print(output) + break + except Exception as error: + print(error) + break + + if not token: + print("Failed to get a token") + else: + print(token) + assert('id_token' in token) + + status, new_token = oidc.refresh_id_token(vo, token['refresh_token']) + # print(new_token) + assert('id_token' in new_token) + + print("verifying the token") + status, decoded_token, username = oidc.verify_id_token(vo, token['id_token']) + if not status: + print("Failed to verify the token: %s" % decoded_token) + else: + print(username) + print(decoded_token) + + +if __name__ == '__main__': + test = TestAuthentication() + test.test_oidc_authentication() diff --git a/main/lib/idds/tests/find_dependencies.py b/main/lib/idds/tests/find_dependencies.py new file mode 100644 index 00000000..18f94e35 --- /dev/null +++ b/main/lib/idds/tests/find_dependencies.py @@ -0,0 +1,75 @@ +import sys +import datetime + +from idds.common.utils import json_dumps # noqa F401 +from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking # noqa F401 +from idds.core.requests import get_requests # noqa F401 +from idds.core.messages import retrieve_messages # noqa F401 +from idds.core.transforms import get_transforms # noqa F401 +from idds.core.workprogress import get_workprogresses # noqa F401 +from idds.core.processings import get_processings # noqa F401 +from idds.core import transforms as core_transforms # noqa F401 +from idds.core import catalog as core_catalog # noqa F401 +from idds.orm.contents import get_input_contents +from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401 + + +def get_input_dep(contents, request_id, transform_id, coll_id, scope, name): + # print(request_id, transform_id, coll_id, scope, name) + map_id = None + for content in contents: + if content['transform_id'] == transform_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # print(content) + print("output name: %s, status: %s" % (content['name'], content['status'])) + map_id = content['map_id'] + # elif coll_id and content['coll_id'] == coll_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # # print(content) + # map_id = content['map_id'] + + print(map_id) + deps = [] + for content in contents: + if content['transform_id'] == transform_id and content['map_id'] == map_id and content['content_relation_type'] == ContentRelationType.InputDependency: + # print(content) + print("Input dependency name: %s, status: %s" % (content['name'], content['status'])) + deps.append({'request_id': content['request_id'], + 'transform_id': content['transform_id'], + 'coll_id': content['coll_id'], + 'scope': content['scope'], + 'name': content['name']}) + return deps + + +def get_transform_id(collections, coll_id): + for coll in collections: + if coll['coll_id'] == coll_id: + return coll['transform_id'] + + +def get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=1): + deps = get_input_dep(contents, request_id, transform_id, coll_id, scope, name) + print("Step: %s" % step) + print("(%s, %s, %s, %s, %s) depents on %s" % (request_id, transform_id, coll_id, scope, name, deps)) + step += 1 + for dep in deps: + coll_id = dep['coll_id'] + transform_id = get_transform_id(collections, coll_id) + dep['transform_id'] = transform_id + get_dep_link(collections=collections, contents=contents, step=step, **dep) + + +def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1): + collections = core_catalog.get_collections(request_id=request_id) + contents = core_catalog.get_contents(request_id=request_id) + get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=step) + + +if __name__ == '__main__': + request_id = 1689 + transform_id = 14713 + coll_id = None + scope = 'pseudo_dataset' + name = '94248742-a255-4541-ab38-ab69364a4c88_transformPreSourceTable_23224_72+qgraphNodeId:94248742-a255-4541-ab38-ab69364a4c88+qgraphId:1657127232.749359-46373' + + # get_input_dep(request_id, transform_id, scope, name) + get_dep_links(request_id, transform_id, coll_id, scope, name) diff --git a/main/tools/env/config_monitor.py b/main/tools/env/config_monitor.py new file mode 100644 index 00000000..336cca99 --- /dev/null +++ b/main/tools/env/config_monitor.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2022 + + +import argparse +import logging + + +def config_api_host(conf_file_template="data/conf.js.template", conf_file='data/conf.js', hostname=None): + with open(conf_file_template, 'r') as f: + template = f.read() + template = template.format(api_host_name=hostname) + with open(conf_file, 'w') as f: + f.write(template) + + +logging.getLogger().setLevel(logging.INFO) +parser = argparse.ArgumentParser(description="config iDDS monitor") +parser.add_argument('-s', '--source', default=None, help='Source config file path') +parser.add_argument('-d', '--destination', default=None, help='Destination file path') +parser.add_argument('--host', default=None, help='idds host name') +args = parser.parse_args() + + +config_api_host(conf_file_template=args.source, conf_file=args.destination, hostname=args.host) diff --git a/main/tools/env/install_packages.sh b/main/tools/env/install_packages.sh new file mode 100755 index 00000000..70d615b8 --- /dev/null +++ b/main/tools/env/install_packages.sh @@ -0,0 +1,6 @@ +$'#!/bin/bash +set -m +for package in common main client workflow doma atlas website monitor ; +do + python3 -m pip install `ls $package/dist/*.tar.gz` +done diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 39e1faff..f8739468 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true" } From cef1922ece8a9a5dae8474098fdc34c70c654e53 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 17:10:35 +0200 Subject: [PATCH 12/25] new version 0.11.5 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 67b93aa1..7c2e8610 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index dccd5ab3..fdfef10d 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.4 - - idds-workflow==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 67b93aa1..7c2e8610 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 5076e4f1..45756d68 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.4 - - idds-workflow==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 67b93aa1..7c2e8610 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index e7dd71a7..a9a272c5 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 6224f7d4..4847bdb4 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.4 - - idds-workflow==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 67b93aa1..7c2e8610 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index a0a39f16..2a964de1 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.4 - - idds-workflow==0.11.4 - - idds-client==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 + - idds-client==0.11.5 \ No newline at end of file diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 67b93aa1..7c2e8610 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 67b93aa1..7c2e8610 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 67b93aa1..7c2e8610 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index b878a2c1..a3eb0093 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.4 \ No newline at end of file + - idds-common==0.11.5 \ No newline at end of file From d8ac4c68a0966681a714d0debfdb866b5bf5dcf6 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 28 May 2022 00:25:34 +0200 Subject: [PATCH 13/25] #76 fix idds client base class to set default x509 path --- client/lib/idds/client/base.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/client/lib/idds/client/base.py b/client/lib/idds/client/base.py index 1a4efffd..2f24f1e8 100644 --- a/client/lib/idds/client/base.py +++ b/client/lib/idds/client/base.py @@ -26,7 +26,7 @@ from idds.common import exceptions from idds.common.constants import HTTP_STATUS_CODE -from idds.common.utils import json_dumps, json_loads +from idds.common.utils import json_dumps, json_loads, get_proxy_path from idds.common.authentication import OIDCAuthenticationUtils @@ -68,6 +68,20 @@ def __init__(self, host=None, auth=None, timeout=None, client_proxy=None): self.check_auth() + def get_user_proxy(sellf): + """ + Get the user proxy. + + :returns: the path of the user proxy. + """ + + client_proxy = get_proxy_path() + + if not client_proxy or not os.path.exists(client_proxy): + raise exceptions.RestException("Cannot find a valid x509 proxy.") + + return client_proxy + def check_auth(self): """ To check whether the auth type is supported and the input for the auth is available. @@ -77,6 +91,8 @@ def check_auth(self): self.auth_type = 'x509_proxy' if self.auth_type in ['x509_proxy']: + if not self.client_proxy: + self.client_proxy = self.get_user_proxy() if not self.client_proxy or not os.path.exists(self.client_proxy): raise exceptions.RestException("Cannot find a valid x509 proxy.") elif self.auth_type in ['oidc']: From 7f1b48df29ad997cfd4ea1e35b1a59297e4b7b60 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 18 Jun 2022 09:34:47 +0200 Subject: [PATCH 14/25] add pandacache url --- atlas/lib/idds/atlas/version.py | 2 +- .../idds/atlas/workflowv2/atlaspandawork.py | 29 +++++++++++++++++ atlas/tools/env/environment.yml | 4 +-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 +-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- .../lib/idds/doma/workflowv2/domapandawork.py | 14 ++++++++ doma/tools/env/environment.yml | 4 +-- .../httpd-idds-443-py39-cc7.conf | 10 +++--- main/config_default/idds.cfg | 10 +++--- .../agents/conductor/plugins/messaging.py | 1 - main/lib/idds/core/processings.py | 13 +++++--- main/lib/idds/orm/base/models.py | 4 ++- main/lib/idds/orm/base/utils.py | 2 +- main/lib/idds/orm/processings.py | 7 ++-- main/lib/idds/rest/v1/app.py | 4 +++ main/lib/idds/tests/core_tests.py | 2 +- .../lib/idds/tests/test_domapanda_workflow.py | 6 +++- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 ++-- main/tools/env/setup_dev.sh | 2 +- main/tools/env/setup_panda.sh | 32 +++++++++++++++---- monitor/data/conf.js | 12 +++---- monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 28 files changed, 130 insertions(+), 54 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 0384e5e0..52bf8dd8 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py index bcd505bb..07fdf512 100644 --- a/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py +++ b/atlas/lib/idds/atlas/workflowv2/atlaspandawork.py @@ -117,6 +117,23 @@ def load_panda_urls(self): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if panda_config.has_option('panda', 'pandacache_url'): + self.pandacache_url = panda_config.get('panda', 'pandacache_url') + os.environ['PANDACACHE_URL'] = self.pandacache_url + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if panda_config.has_option('panda', 'panda_verify_host'): + self.panda_verify_host = panda_config.get('panda', 'panda_verify_host') + os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) + if panda_config.has_option('panda', 'panda_auth'): + self.panda_auth = panda_config.get('panda', 'panda_auth') + os.environ['PANDA_AUTH'] = self.panda_auth + if panda_config.has_option('panda', 'panda_auth_vo'): + self.panda_auth_vo = panda_config.get('panda', 'panda_auth_vo') + os.environ['PANDA_AUTH_VO'] = self.panda_auth_vo + if panda_config.has_option('panda', 'panda_config_root'): + self.panda_config_root = panda_config.get('panda', 'panda_config_root') + os.environ['PANDA_CONFIG_ROOT'] = self.panda_config_root if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']: self.panda_monitor = os.environ['PANDA_MONITOR_URL'] @@ -127,6 +144,18 @@ def load_panda_urls(self): if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']: + self.pandacache_url = os.environ['PANDACACHE_URL'] + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']: + self.panda_verify_host = os.environ['PANDA_VERIFY_HOST'] + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) + if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']: + self.panda_auth = os.environ['PANDA_AUTH'] + if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']: + self.panda_auth_vo = os.environ['PANDA_AUTH_VO'] + if not self.panda_config_root and 'PANDA_CONFIG_ROOT' in os.environ and os.environ['PANDA_CONFIG_ROOT']: + self.panda_config_root = os.environ['PANDA_CONFIG_ROOT'] def set_agent_attributes(self, attrs, req_attributes=None): if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0: diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 077379cd..9151aec6 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.0 - - idds-workflow==0.11.0 \ No newline at end of file + - idds-common==0.11.1 + - idds-workflow==0.11.1 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 0384e5e0..52bf8dd8 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 51b4c3f8..c916c00e 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.0 - - idds-workflow==0.11.0 + - idds-common==0.11.1 + - idds-workflow==0.11.1 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 0384e5e0..52bf8dd8 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 8e5196ff..474b7ac8 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index e8bc1fbf..5452c8c9 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -150,6 +150,14 @@ def load_panda_urls(self): self.panda_url = panda_config.get('panda', 'panda_url') os.environ['PANDA_URL'] = self.panda_url # self.logger.debug("Panda url: %s" % str(self.panda_url)) + if panda_config.has_option('panda', 'pandacache_url'): + self.pandacache_url = panda_config.get('panda', 'pandacache_url') + os.environ['PANDACACHE_URL'] = self.pandacache_url + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if panda_config.has_option('panda', 'panda_verify_host'): + self.panda_verify_host = panda_config.get('panda', 'panda_verify_host') + os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) if panda_config.has_option('panda', 'panda_url_ssl'): self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl') os.environ['PANDA_URL_SSL'] = self.panda_url_ssl @@ -173,6 +181,12 @@ def load_panda_urls(self): if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']: self.panda_url_ssl = os.environ['PANDA_URL_SSL'] # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl)) + if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']: + self.pandacache_url = os.environ['PANDACACHE_URL'] + # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url)) + if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']: + self.panda_verify_host = os.environ['PANDA_VERIFY_HOST'] + # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host)) if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']: self.panda_auth = os.environ['PANDA_AUTH'] if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']: diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 61559a5d..7ed7bbfc 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.0 - - idds-workflow==0.11.0 \ No newline at end of file + - idds-common==0.11.1 + - idds-workflow==0.11.1 \ No newline at end of file diff --git a/main/config_default/httpd-idds-443-py39-cc7.conf b/main/config_default/httpd-idds-443-py39-cc7.conf index db09cc6b..abb0876e 100644 --- a/main/config_default/httpd-idds-443-py39-cc7.conf +++ b/main/config_default/httpd-idds-443-py39-cc7.conf @@ -34,11 +34,11 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages WSGIApplicationGroup %GLOBAL WSGIScriptAlias /idds /opt/idds/bin/idds.wsgi # WSGIScriptAliasMatch ^/idds/(.+)$ /opt/idds/etc/idds/rest/test.wsgi - WSGISocketPrefix /var/log/idds/wsgisocks/wsgi + WSGISocketPrefix /var/idds/wsgisocks/wsgi WSGIPassAuthorization On -Listen 8443 +Listen 443 RewriteEngine on RewriteCond %REQUEST_METHOD ^(TRACE|TRACK) @@ -49,8 +49,8 @@ TraceEnable off Alias "/website" "/opt/idds/website/data" Alias "/monitor" "/opt/idds/monitor/data" - - # ServerName aipanda182.cern.ch:8443 + + # ServerName aipanda182.cern.ch:443 ServerAdmin wguan@cern.ch SSLEngine on @@ -87,7 +87,7 @@ Alias "/monitor" "/opt/idds/monitor/data" GridSiteDNlists /etc/grid-security/dn-lists/ GridSiteGSIProxyLimit 16 GridSiteEnvs on - # GridSiteACLPath /opt/idds/etc/idds/rest/gacl + GridSiteACLPath /opt/idds/etc/idds/rest/gacl # GridSiteMethods GET diff --git a/main/config_default/idds.cfg b/main/config_default/idds.cfg index abc2370e..2a5b3d8f 100755 --- a/main/config_default/idds.cfg +++ b/main/config_default/idds.cfg @@ -26,15 +26,15 @@ agents = clerk, transformer, carrier, conductor [clerk] num_threads = 4 -poll_time_period = 120 -poll_operation_time_period = 120 -retrieve_bulk_size = 1 +poll_time_period = 60 +poll_operation_time_period = 60 +retrieve_bulk_size = 4 pending_time = 4 [transformer] num_threads = 8 poll_time_period = 60 -retrieve_bulk_size = 1 +retrieve_bulk_size = 4 poll_operation_time_period = 240 message_bulk_size = 1000 @@ -45,7 +45,7 @@ domapandawork.num_retries = 0 num_threads = 8 poll_time_period = 60 poll_operation_time_period = 240 -retrieve_bulk_size = 3 +retrieve_bulk_size = 4 message_bulk_size = 1000 diff --git a/main/lib/idds/agents/conductor/plugins/messaging.py b/main/lib/idds/agents/conductor/plugins/messaging.py index 1652f84d..cce41a02 100644 --- a/main/lib/idds/agents/conductor/plugins/messaging.py +++ b/main/lib/idds/agents/conductor/plugins/messaging.py @@ -104,7 +104,6 @@ def connect_to_messaging_brokers(self): vhost=self.vhost, keepalive=True, timeout=self.broker_timeout) - conn.set_listener('message-sender', MessagingListener(conn.transport._Transport__host_and_ports[0])) self.conns.append(conn) def send_message(self, msg): diff --git a/main/lib/idds/core/processings.py b/main/lib/idds/core/processings.py index 218652cd..4b39c738 100644 --- a/main/lib/idds/core/processings.py +++ b/main/lib/idds/core/processings.py @@ -138,7 +138,8 @@ def get_processings_with_messaging(locking=False, bulk_size=None, session=None): @transactional_session -def get_processings_by_status(status, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, with_messaging=False, session=None): +def get_processings_by_status(status, time_period=None, locking=False, bulk_size=None, to_json=False, by_substatus=False, + with_messaging=False, for_poller=False, session=None): """ Get processing or raise a NoObject exception. @@ -163,12 +164,14 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size # then select with locking. proc_ids = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size * 2, to_json=False, locking_for_update=False, - by_substatus=by_substatus, only_return_id=True, session=session) + by_substatus=by_substatus, only_return_id=True, + for_poller=for_poller, session=session) if proc_ids: processing2s = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, processing_ids=proc_ids, bulk_size=None, to_json=to_json, locking_for_update=True, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, + for_poller=for_poller, session=session) if processing2s: # reqs = req2s[:bulk_size] # order requests @@ -188,7 +191,7 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size else: processings = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, locking_for_update=locking, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, for_poller=for_poller, session=session) parameters = {'locking': ProcessingLocking.Locking} for processing in processings: @@ -196,7 +199,7 @@ def get_processings_by_status(status, time_period=None, locking=False, bulk_size else: processings = orm_processings.get_processings_by_status(status=status, period=time_period, locking=locking, bulk_size=bulk_size, to_json=to_json, - by_substatus=by_substatus, session=session) + by_substatus=by_substatus, for_poller=for_poller, session=session) return processings diff --git a/main/lib/idds/orm/base/models.py b/main/lib/idds/orm/base/models.py index 0c9f957d..f35e557b 100644 --- a/main/lib/idds/orm/base/models.py +++ b/main/lib/idds/orm/base/models.py @@ -356,6 +356,7 @@ class Processing(BASE, ModelBase): created_at = Column("created_at", DateTime, default=datetime.datetime.utcnow) updated_at = Column("updated_at", DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow) next_poll_at = Column("next_poll_at", DateTime, default=datetime.datetime.utcnow) + poller_updated_at = Column("poller_updated_at", DateTime, default=datetime.datetime.utcnow) submitted_at = Column("submitted_at", DateTime) finished_at = Column("finished_at", DateTime) expired_at = Column("expired_at", DateTime) @@ -561,7 +562,8 @@ def register_models(engine): models = (Request, Transform, Processing, Collection, Content, Health, Message) for model in models: - model.metadata.create_all(engine) # pylint: disable=maybe-no-member + if not engine.has_table(model.__tablename__, model.metadata.schema): + model.metadata.create_all(engine) # pylint: disable=maybe-no-member def unregister_models(engine): diff --git a/main/lib/idds/orm/base/utils.py b/main/lib/idds/orm/base/utils.py index 6391e4f7..d3ffe4a8 100644 --- a/main/lib/idds/orm/base/utils.py +++ b/main/lib/idds/orm/base/utils.py @@ -34,7 +34,7 @@ def build_database(echo=True, tests=False): if config_has_option('database', 'schema'): schema = config_get('database', 'schema') - if schema: + if schema and not engine.dialect.has_schema(engine, schema): print('Schema set in config, trying to create schema:', schema) try: engine.execute(CreateSchema(schema)) diff --git a/main/lib/idds/orm/processings.py b/main/lib/idds/orm/processings.py index 1f4fd3ce..d7130544 100644 --- a/main/lib/idds/orm/processings.py +++ b/main/lib/idds/orm/processings.py @@ -202,7 +202,8 @@ def get_processings_by_transform_id(transform_id=None, to_json=False, session=No @transactional_session def get_processings_by_status(status, period=None, processing_ids=[], locking=False, locking_for_update=False, - bulk_size=None, submitter=None, to_json=False, by_substatus=False, only_return_id=False, session=None): + bulk_size=None, submitter=None, to_json=False, by_substatus=False, only_return_id=False, + for_poller=False, session=None): """ Get processing or raise a NoObject exception. @@ -248,7 +249,9 @@ def get_processings_by_status(status, period=None, processing_ids=[], locking=Fa if submitter: query = query.filter(models.Processing.submitter == submitter) - if locking_for_update: + if for_poller: + query = query.order_by(asc(models.Processing.poller_updated_at)) + elif locking_for_update: query = query.with_for_update(skip_locked=True) else: query = query.order_by(asc(models.Processing.updated_at)) diff --git a/main/lib/idds/rest/v1/app.py b/main/lib/idds/rest/v1/app.py index 5e8974e2..7c979085 100644 --- a/main/lib/idds/rest/v1/app.py +++ b/main/lib/idds/rest/v1/app.py @@ -83,6 +83,10 @@ def generate_failed_auth_response(exc_msg=None): def before_request_auth(): + print("envs") + print(flask.request.environ) + print("headers") + print(flask.request.headers) auth_type = flask.request.headers.get('X-IDDS-Auth-Type', default='x509_proxy') vo = flask.request.headers.get('X-IDDS-Auth-VO', default=None) if auth_type in ['x509_proxy']: diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index 78479bf9..e64603b7 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -110,7 +110,7 @@ def show_works(req): # reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True) -reqs = get_requests(request_id=1098, with_request=True, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=1439, with_request=True, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/test_domapanda_workflow.py b/main/lib/idds/tests/test_domapanda_workflow.py index be0b7347..119c33ee 100644 --- a/main/lib/idds/tests/test_domapanda_workflow.py +++ b/main/lib/idds/tests/test_domapanda_workflow.py @@ -14,9 +14,12 @@ Test client. """ +import logging import string import random +logging.basicConfig(level=logging.DEBUG) + # import traceback # from rucio.client.client import Client as Rucio_Client @@ -130,7 +133,8 @@ def setup_workflow(): "token": "local", "type": "template", "value": "log.tgz"}, - task_cloud='LSST') + task_cloud='LSST', + task_priority=None) work2 = DomaPanDAWork(executable='echo', primary_input_collection={'scope': 'pseudo_dataset', 'name': 'pseudo_input_collection#2'}, output_collections=[{'scope': 'pseudo_dataset', 'name': 'pseudo_output_collection#2'}], diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 0384e5e0..52bf8dd8 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 6e196d31..e5968a00 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.0 - - idds-workflow==0.11.0 - - idds-client==0.11.0 \ No newline at end of file + - idds-common==0.11.1 + - idds-workflow==0.11.1 + - idds-client==0.11.1 \ No newline at end of file diff --git a/main/tools/env/setup_dev.sh b/main/tools/env/setup_dev.sh index d2294598..e949e71e 100644 --- a/main/tools/env/setup_dev.sh +++ b/main/tools/env/setup_dev.sh @@ -25,5 +25,5 @@ conda activate $CondaDir export RUCIO_HOME=$RootDir export RUCIO_ACCOUNT=ddmadmin export X509_USER_PROXY=/tmp/x509up_u23959 -export PYTHONPATH=$PYTHONPATH:/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/current/lib/python3.6/site-packages/ +# export PYTHONPATH=$PYTHONPATH:/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/current/lib/python3.6/site-packages/ diff --git a/main/tools/env/setup_panda.sh b/main/tools/env/setup_panda.sh index f16611a3..21aa246f 100644 --- a/main/tools/env/setup_panda.sh +++ b/main/tools/env/setup_panda.sh @@ -1,11 +1,29 @@ #!/bin/bash -export PANDA_AUTH=oidc -export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda -export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda -export PANDAMON_URL=https://panda-doma.cern.ch -export PANDA_AUTH_VO=panda_dev +instance=doma +if [ "$#" -eq 1 ]; then + instance=$1 +fi -# export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ -export PANDA_CONFIG_ROOT=~/.panda/ +export PANDA_BEHIND_REAL_LB=true +# export PANDA_SYS=/opt/idds/ +if [ "$instance" == "k8s" ]; then + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda + export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + export PANDA_AUTH_VO=panda_dev + + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ +else + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda + export PANDA_URL=http://pandaserver-doma.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + export PANDA_AUTH_VO=panda_dev + + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ +fi diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 3b39994b..410cd4a1 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus776.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus776.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus776.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus776.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus740.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus740.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus740.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 0384e5e0..52bf8dd8 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 0384e5e0..52bf8dd8 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 0384e5e0..52bf8dd8 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.0" +release_version = "0.11.1" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index bffb8864..8aa1a003 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.0 \ No newline at end of file + - idds-common==0.11.1 \ No newline at end of file From f82286657ab1b66f9c8ee0a362042985e58d6821 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Sat, 18 Jun 2022 09:40:07 +0200 Subject: [PATCH 15/25] new version 0.11.2 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 9151aec6..a7979520 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.1 - - idds-workflow==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index c916c00e..fc3c21c2 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.1 - - idds-workflow==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 474b7ac8..1311630b 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 7ed7bbfc..a8aedf1a 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.1 - - idds-workflow==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index e5968a00..23847f57 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.1 - - idds-workflow==0.11.1 - - idds-client==0.11.1 \ No newline at end of file + - idds-common==0.11.2 + - idds-workflow==0.11.2 + - idds-client==0.11.2 \ No newline at end of file diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 52bf8dd8..7d2b2b6e 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.1" +release_version = "0.11.2" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 8aa1a003..3a543d41 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.1 \ No newline at end of file + - idds-common==0.11.2 \ No newline at end of file From 6da7a2098048d7ff408805eaa76d24183e7f01fb Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 17:34:57 +0200 Subject: [PATCH 16/25] add default for maxattempt --- doma/lib/idds/doma/workflowv2/domapandawork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py index 5452c8c9..9805f961 100644 --- a/doma/lib/idds/doma/workflowv2/domapandawork.py +++ b/doma/lib/idds/doma/workflowv2/domapandawork.py @@ -89,7 +89,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None, self.prodSourceLabel = prodSourceLabel self.task_type = task_type self.maxWalltime = maxwalltime - self.maxAttempt = maxattempt + self.maxAttempt = maxattempt if maxattempt else 5 self.core_count = core_count self.task_log = task_log From a34415a41652d3ceda9d350ec4fd7977d103ab0f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 17:36:29 +0200 Subject: [PATCH 17/25] update idds rest default port 8443 --- main/config_default/httpd-idds-443-py39-cc7.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/config_default/httpd-idds-443-py39-cc7.conf b/main/config_default/httpd-idds-443-py39-cc7.conf index abb0876e..ffb5b3d1 100644 --- a/main/config_default/httpd-idds-443-py39-cc7.conf +++ b/main/config_default/httpd-idds-443-py39-cc7.conf @@ -38,7 +38,7 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages WSGIPassAuthorization On -Listen 443 +Listen 8443 RewriteEngine on RewriteCond %REQUEST_METHOD ^(TRACE|TRACK) @@ -49,8 +49,8 @@ TraceEnable off Alias "/website" "/opt/idds/website/data" Alias "/monitor" "/opt/idds/monitor/data" - - # ServerName aipanda182.cern.ch:443 + + # ServerName aipanda182.cern.ch:8443 ServerAdmin wguan@cern.ch SSLEngine on From 5ba299f5db6c5131b69f6adf1176d66ff1cdf857 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 17:40:20 +0200 Subject: [PATCH 18/25] new version 0.11.3 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/data/conf.js | 12 ++++++------ monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index a7979520..35900670 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.2 - - idds-workflow==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index fc3c21c2..0fefdb0d 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.2 - - idds-workflow==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 1311630b..4d610cb1 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index a8aedf1a..5aacda7f 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.2 - - idds-workflow==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 23847f57..7291ea87 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.2 - - idds-workflow==0.11.2 - - idds-client==0.11.2 \ No newline at end of file + - idds-common==0.11.3 + - idds-workflow==0.11.3 + - idds-client==0.11.3 \ No newline at end of file diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 410cd4a1..763b37e8 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus740.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus740.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus740.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus740.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus789.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus789.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus789.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 7d2b2b6e..88e8d2cc 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.2" +release_version = "0.11.3" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 3a543d41..164c36be 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.2 \ No newline at end of file + - idds-common==0.11.3 \ No newline at end of file From 7e70c7b6a95e06a55a55f4ba3d221b000211c42f Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 18:11:28 +0200 Subject: [PATCH 19/25] new version 0.11.4 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/config_default/supervisord_idds.ini | 2 +- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/data/conf.js | 12 ++++++------ monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 15 files changed, 25 insertions(+), 25 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 88e8d2cc..67b93aa1 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index 35900670..dccd5ab3 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.3 - - idds-workflow==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 88e8d2cc..67b93aa1 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 0fefdb0d..5076e4f1 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.3 - - idds-workflow==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 88e8d2cc..67b93aa1 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index 4d610cb1..e7dd71a7 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 5aacda7f..6224f7d4 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.3 - - idds-workflow==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 \ No newline at end of file diff --git a/main/config_default/supervisord_idds.ini b/main/config_default/supervisord_idds.ini index 3a4a75a4..30d7e2f4 100644 --- a/main/config_default/supervisord_idds.ini +++ b/main/config_default/supervisord_idds.ini @@ -7,7 +7,7 @@ environment = ;command=/opt/idds/bin/run-idds command=bash -c "source /etc/profile.d/conda.sh && conda activate /opt/idds && /opt/idds/bin/run-idds" process_name=%(process_num)02d -user=atlpan +# user=atlpan childlogdir=/var/log/idds stdout_logfile=/var/log/idds/%(program_name)s-stdout.log stderr_logfile=/var/log/idds/%(program_name)s-stderr.log diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 88e8d2cc..67b93aa1 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index 7291ea87..a0a39f16 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.3 - - idds-workflow==0.11.3 - - idds-client==0.11.3 \ No newline at end of file + - idds-common==0.11.4 + - idds-workflow==0.11.4 + - idds-client==0.11.4 \ No newline at end of file diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 763b37e8..39e1faff 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus789.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus789.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus789.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus789.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 88e8d2cc..67b93aa1 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 88e8d2cc..67b93aa1 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 88e8d2cc..67b93aa1 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.3" +release_version = "0.11.4" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 164c36be..b878a2c1 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.3 \ No newline at end of file + - idds-common==0.11.4 \ No newline at end of file From 78b1ba89b6f49fe3f5047de10150e8d572bee301 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Tue, 5 Jul 2022 22:17:44 +0200 Subject: [PATCH 20/25] update docker file to suppor noRoot --- Dockerfile | 23 ++++++++++++++++++++++- start-daemon.sh | 13 ++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 34d021f6..46db1d33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ RUN mkdir /var/idds RUN mkdir /var/idds/wsgisocks RUN chown atlpan -R /opt/idds # RUN chown atlpan -R /opt/idds_source -RUN chown atlpan /var/log/idds +RUN chown atlpan -R /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ # to run with non-root PID @@ -93,6 +93,8 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ WORKDIR /tmp RUN rm -rf /tmp/src +RUN chmod 777 /opt/idds/monitor/data +RUN chmod 777 /opt/idds/monitor/data/conf.js RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds @@ -131,12 +133,31 @@ RUN ln -fs /opt/idds/config/idds/auth.cfg /opt/idds/etc/idds/auth/auth.cfg RUN ln -fs /opt/idds/config/idds/gacl /opt/idds/etc/idds/rest/gacl RUN ln -fs /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf +# update http config +RUN sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf +RUN sed -i 's/Listen\ 80/#\ Listen\ 80/g' /etc/httpd/conf/httpd.conf +RUN sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf + # for idds daemons RUN ln -fs /opt/idds/config/idds/supervisord_idds.ini /etc/supervisord.d/idds.ini +RUN chmod -R 777 /opt/idds/config +RUN chmod -R 777 /var/log/idds +RUN chmod 777 /etc/grid-security +RUN chmod 777 /etc/httpd/conf.d +RUN chmod 777 /etc/httpd/conf/httpd.conf +RUN chmod 777 /etc/httpd/conf +RUN chmod 777 /run/httpd +RUN chmod 777 /var/log/supervisor/ +RUN chmod 777 /var/run/supervisor +RUN chmod 777 /var/run +RUN chmod 777 /etc/httpd/logs + ENV PATH /opt/idds/bin/:$PATH ADD start-daemon.sh /opt/idds/bin/ +RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back +ADD ssl.conf /etc/httpd/conf.d/ssl.conf VOLUME /var/log/idds VOLUME /opt/idds/config diff --git a/start-daemon.sh b/start-daemon.sh index 1c7828b9..df839ffe 100755 --- a/start-daemon.sh +++ b/start-daemon.sh @@ -9,10 +9,12 @@ export IDDS_HOME=/opt/idds if [ -f /etc/grid-security/hostkey.pem ]; then echo "host certificate is already created." + chmod 600 /etc/grid-security/hostkey.pem elif [ -f /opt/idds/configmap/hostkey.pem ]; then echo "mount /opt/idds/configmap/hostkey.pem to /etc/grid-security/hostkey.pem" ln -fs /opt/idds/configmap/hostkey.pem /etc/grid-security/hostkey.pem ln -fs /opt/idds/configmap/hostcert.pem /etc/grid-security/hostcert.pem + chmod 600 /etc/grid-security/hostkey.pem fi if [ -f /opt/idds/config/idds/idds.cfg ]; then @@ -82,7 +84,6 @@ if [ -f /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf ]; then echo "httpd conf already mounted." else echo "httpd conf not found. will use the default one." - sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf cp /opt/idds/config_default/httpd-idds-443-py39-cc7.conf /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf fi @@ -103,6 +104,7 @@ else -out /opt/idds/config/hostcert.pem ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem + chmod 600 /etc/grid-security/hostkey.pem fi mkdir -p /opt/idds/config/.panda/ @@ -131,11 +133,16 @@ if [ ! -z "$IDDS_PRINT_CFG" ]; then echo "" fi -sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf # create database if not exists python /opt/idds/tools/env/create_database.py python /opt/idds/tools/env/config_monitor.py -s ${IDDS_HOME}/monitor/data/conf.js.template -d ${IDDS_HOME}/monitor/data/conf.js --host ${IDDS_SERVER} -ln -s /opt/idds/configmap/idds2panda_token /opt/idds/config/.token + +if ! [ -f /opt/idds/config/.token ]; then + echo "/opt/idds/config/.token does not exist." + if [ -f /opt/idds/configmap/idds2panda_token ]; then + ln -s /opt/idds/configmap/idds2panda_token /opt/idds/config/.token + fi +fi if [ "${IDDS_SERVICE}" == "rest" ]; then echo "starting iDDS ${IDDS_SERVICE} service" From a33d4e01599657a4a63f29d0a4d1114ebf8a5bd7 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 15:31:46 +0200 Subject: [PATCH 21/25] #80 options to disable oidc components to verify ssl host certificate --- common/lib/idds/common/authentication.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/common/lib/idds/common/authentication.py b/common/lib/idds/common/authentication.py index d6336f36..1ed0a549 100644 --- a/common/lib/idds/common/authentication.py +++ b/common/lib/idds/common/authentication.py @@ -45,6 +45,12 @@ def decode_value(val): return int.from_bytes(decoded, 'big') +def should_verify(): + if os.environ.get('IDDS_AUTH_NO_VERIFY', None): + return False + return True + + class BaseAuthentication(object): def __init__(self, timeout=None): self.timeout = timeout @@ -98,7 +104,7 @@ def get_auth_config(self, vo): def get_http_content(self, url): try: - r = requests.get(url, allow_redirects=True) + r = requests.get(url, allow_redirects=True, verify=should_verify()) return r.content except Exception as error: return False, 'Failed to get http content for %s: %s' (str(url), str(error)) @@ -128,6 +134,7 @@ def get_oidc_sign_url(self, vo): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: @@ -172,6 +179,7 @@ def get_id_token(self, vo, device_code, interval=5, expires_in=60): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: if result.status_code == HTTP_STATUS_CODE.OK and result.text: @@ -203,6 +211,7 @@ def refresh_id_token(self, vo, refresh_token): # data=json.dumps(data), urlencode(data).encode(), timeout=self.timeout, + verify=should_verify(), headers=headers) if result is not None: From 4e5dcd68f0d42d39b31c8a0944fd05105d82d6fa Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 15:32:14 +0200 Subject: [PATCH 22/25] update tests --- main/lib/idds/tests/core_tests.py | 2 +- main/lib/idds/tests/test_migrate_requests.py | 4 ++-- main/lib/idds/tests/trigger_release.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/main/lib/idds/tests/core_tests.py b/main/lib/idds/tests/core_tests.py index e64603b7..5cbb787d 100644 --- a/main/lib/idds/tests/core_tests.py +++ b/main/lib/idds/tests/core_tests.py @@ -110,7 +110,7 @@ def show_works(req): # reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True) # reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True) -reqs = get_requests(request_id=1439, with_request=True, with_detail=False, with_metadata=True) +reqs = get_requests(request_id=1687, with_request=True, with_detail=False, with_metadata=True) for req in reqs: # print(req['request_id']) # print(rets) diff --git a/main/lib/idds/tests/test_migrate_requests.py b/main/lib/idds/tests/test_migrate_requests.py index b7666c1f..9a134b7c 100644 --- a/main/lib/idds/tests/test_migrate_requests.py +++ b/main/lib/idds/tests/test_migrate_requests.py @@ -34,14 +34,14 @@ def migrate(): cm1 = ClientManager(host=doma_host) # reqs = cm1.get_requests(request_id=290) # old_request_id = 298163 - old_request_id = 901 + old_request_id = 1685 # for old_request_id in [152]: # for old_request_id in [60]: # noqa E115 # for old_request_id in [200]: # noqa E115 for old_request_id in [old_request_id]: # noqa E115 # doma 183 reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True) - cm2 = ClientManager(host=doma_google_host) + cm2 = ClientManager(host=doma_host) # print(reqs) print("num requests: %s" % len(reqs)) diff --git a/main/lib/idds/tests/trigger_release.py b/main/lib/idds/tests/trigger_release.py index 368c6bfd..5f5390ac 100644 --- a/main/lib/idds/tests/trigger_release.py +++ b/main/lib/idds/tests/trigger_release.py @@ -12,7 +12,7 @@ request_ids = [368, 369, 370, 371, 372, 373, 374, 375, 376] -request_ids = [902] +request_ids = [1689] for request_id in request_ids: contents = get_contents(request_id=request_id, status=ContentStatus.Available) ret_contents = {} From 02e951cd604ae2664cf3c3dd5cfe44c3a8b5b97e Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 17:09:52 +0200 Subject: [PATCH 23/25] update docker deployment --- Dockerfile | 10 +- main/etc/idds/rest/ssl.conf | 55 ++++++++++ main/etc/rucio.cfg.default | 20 ++++ main/etc/sql/postgresql.sql | 6 ++ main/lib/idds/tests/auth_test_script.py | 123 +++++++++++++++++++++++ main/lib/idds/tests/find_dependencies.py | 75 ++++++++++++++ main/tools/env/config_monitor.py | 32 ++++++ main/tools/env/install_packages.sh | 6 ++ monitor/data/conf.js | 12 +-- 9 files changed, 332 insertions(+), 7 deletions(-) create mode 100644 main/etc/idds/rest/ssl.conf create mode 100644 main/etc/rucio.cfg.default create mode 100644 main/etc/sql/postgresql.sql create mode 100644 main/lib/idds/tests/auth_test_script.py create mode 100644 main/lib/idds/tests/find_dependencies.py create mode 100644 main/tools/env/config_monitor.py create mode 100755 main/tools/env/install_packages.sh diff --git a/Dockerfile b/Dockerfile index 46db1d33..c6bb0ded 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,6 +86,13 @@ done \n ' > inst_packages.sh RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ if [[ -z "$TAG" ]] ; then \ python3 setup.py sdist && chmod +x inst_packages.sh && ./inst_packages.sh ; \ + +WORKDIR /tmp/src +COPY . . + +RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ + if [[ -z "$TAG" ]] ; then \ + python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \ else \ python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ fi @@ -157,7 +164,8 @@ ENV PATH /opt/idds/bin/:$PATH ADD start-daemon.sh /opt/idds/bin/ RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back -ADD ssl.conf /etc/httpd/conf.d/ssl.conf +# ADD ssl.conf /etc/httpd/conf.d/ssl.conf +RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf VOLUME /var/log/idds VOLUME /opt/idds/config diff --git a/main/etc/idds/rest/ssl.conf b/main/etc/idds/rest/ssl.conf new file mode 100644 index 00000000..e713647c --- /dev/null +++ b/main/etc/idds/rest/ssl.conf @@ -0,0 +1,55 @@ +# +# When we also provide SSL we have to listen to the +# the HTTPS port in addition. +# +# Listen 443 https + +## +## SSL Global Context +## +## All SSL configuration in this context applies both to +## the main server and all SSL-enabled virtual hosts. +## + +# Pass Phrase Dialog: +# Configure the pass phrase gathering process. +# The filtering dialog program (`builtin' is a internal +# terminal dialog) has to provide the pass phrase on stdout. +SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +# Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism +# to use and second the expiring timeout (in seconds). +SSLSessionCache shmcb:/run/httpd/sslcache(512000) +SSLSessionCacheTimeout 300 + +# Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the +# SSL library. The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +# +# Use "SSLCryptoDevice" to enable any supported hardware +# accelerators. Use "openssl engine -v" to list supported +# engine names. NOTE: If you enable an accelerator and the +# server does not start, consult the error logs and ensure +# your accelerator is functioning properly. +# +SSLCryptoDevice builtin +#SSLCryptoDevice ubsec + +## +## SSL Virtual Host Context +## + diff --git a/main/etc/rucio.cfg.default b/main/etc/rucio.cfg.default new file mode 100644 index 00000000..d7045960 --- /dev/null +++ b/main/etc/rucio.cfg.default @@ -0,0 +1,20 @@ +# Copyright European Organization for Nuclear Research (CERN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Vincent Garonne, , 2013 + +[common] + +[client] +rucio_host = https://voatlasrucio-server-prod.cern.ch:443 +auth_host = https://voatlasrucio-auth-prod.cern.ch:443 +ca_cert = $RUCIO_HOME/etc/ca.crt +client_cert = ~/.globus/usercert.pem +client_key = ~/.globus/userkey.pem +client_x509_proxy = $X509_USER_PROXY +auth_type = x509_proxy +request_retries = 3 diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql new file mode 100644 index 00000000..b6f5e9c5 --- /dev/null +++ b/main/etc/sql/postgresql.sql @@ -0,0 +1,6 @@ +CREATE USER doma_idds_r WITH PASSWORD 'Tiaroa4dr_idds'; +GRANT CONNECT ON DATABASE doma_idds TO doma_idds_r; +GRANT USAGE ON SCHEMA doma_idds TO doma_idds_r; +GRANT SELECT ON ALL TABLES IN SCHEMA doma_idds TO doma_idds_r; +ALTER DEFAULT PRIVILEGES IN SCHEMA doma_idds GRANT SELECT ON TABLES TO doma_idds_r; + diff --git a/main/lib/idds/tests/auth_test_script.py b/main/lib/idds/tests/auth_test_script.py new file mode 100644 index 00000000..ba3715a7 --- /dev/null +++ b/main/lib/idds/tests/auth_test_script.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 - 2022 + + +""" +Test authentication. +""" + +try: + from urllib import urlencode # noqa F401 +except ImportError: + from urllib.parse import urlencode # noqa F401 + raw_input = input + +import datetime +import sys +import time + +import unittest2 as unittest +# from nose.tools import assert_equal +from idds.common.utils import setup_logging +from idds.common.authentication import OIDCAuthentication + + +setup_logging(__name__) + + +class TestAuthentication: + + def test_oidc_authentication(self): + vo = 'iamdev' + + oidc = OIDCAuthentication() + allow_vos = oidc.get_allow_vos() + print("allow_vos") + print(allow_vos) + assert(vo in allow_vos) + auth_config = oidc.get_auth_config(vo) + print("auth_config") + print(auth_config) + assert('vo' in auth_config) + assert(auth_config['vo'] == vo) + + endpoint_config = oidc.get_endpoint_config(auth_config) + print("endpoint_config") + print(endpoint_config) + assert('token_endpoint' in endpoint_config) + + status, sign_url = oidc.get_oidc_sign_url(vo) + print("sign_url") + print(sign_url) + assert('user_code' in sign_url) + print(("Please go to {0} and sign in. " + "Waiting until authentication is completed").format(sign_url['verification_uri_complete'])) + + print('Ready to get ID token?') + while True: + sys.stdout.write("[y/n] \n") + choice = raw_input().lower() + if choice == 'y': + break + elif choice == 'n': + print('aborted') + return + + if 'interval' in sign_url: + interval = sign_url['interval'] + else: + interval = 5 + + if 'expires_in' in sign_url: + expires_in = sign_url['expires_in'] + else: + expires_in = 60 + + token = None + start_time = datetime.datetime.utcnow() + while datetime.datetime.utcnow() - start_time < datetime.timedelta(seconds=expires_in): + try: + status, output = oidc.get_id_token(vo, sign_url['device_code']) + if status: + # print(output) + token = output + break + else: + if type(output) in [dict] and 'error' in output and output['error'] == 'authorization_pending': + time.sleep(interval) + else: + print(output) + break + except Exception as error: + print(error) + break + + if not token: + print("Failed to get a token") + else: + print(token) + assert('id_token' in token) + + status, new_token = oidc.refresh_id_token(vo, token['refresh_token']) + # print(new_token) + assert('id_token' in new_token) + + print("verifying the token") + status, decoded_token, username = oidc.verify_id_token(vo, token['id_token']) + if not status: + print("Failed to verify the token: %s" % decoded_token) + else: + print(username) + print(decoded_token) + + +if __name__ == '__main__': + test = TestAuthentication() + test.test_oidc_authentication() diff --git a/main/lib/idds/tests/find_dependencies.py b/main/lib/idds/tests/find_dependencies.py new file mode 100644 index 00000000..18f94e35 --- /dev/null +++ b/main/lib/idds/tests/find_dependencies.py @@ -0,0 +1,75 @@ +import sys +import datetime + +from idds.common.utils import json_dumps # noqa F401 +from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking # noqa F401 +from idds.core.requests import get_requests # noqa F401 +from idds.core.messages import retrieve_messages # noqa F401 +from idds.core.transforms import get_transforms # noqa F401 +from idds.core.workprogress import get_workprogresses # noqa F401 +from idds.core.processings import get_processings # noqa F401 +from idds.core import transforms as core_transforms # noqa F401 +from idds.core import catalog as core_catalog # noqa F401 +from idds.orm.contents import get_input_contents +from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401 + + +def get_input_dep(contents, request_id, transform_id, coll_id, scope, name): + # print(request_id, transform_id, coll_id, scope, name) + map_id = None + for content in contents: + if content['transform_id'] == transform_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # print(content) + print("output name: %s, status: %s" % (content['name'], content['status'])) + map_id = content['map_id'] + # elif coll_id and content['coll_id'] == coll_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # # print(content) + # map_id = content['map_id'] + + print(map_id) + deps = [] + for content in contents: + if content['transform_id'] == transform_id and content['map_id'] == map_id and content['content_relation_type'] == ContentRelationType.InputDependency: + # print(content) + print("Input dependency name: %s, status: %s" % (content['name'], content['status'])) + deps.append({'request_id': content['request_id'], + 'transform_id': content['transform_id'], + 'coll_id': content['coll_id'], + 'scope': content['scope'], + 'name': content['name']}) + return deps + + +def get_transform_id(collections, coll_id): + for coll in collections: + if coll['coll_id'] == coll_id: + return coll['transform_id'] + + +def get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=1): + deps = get_input_dep(contents, request_id, transform_id, coll_id, scope, name) + print("Step: %s" % step) + print("(%s, %s, %s, %s, %s) depents on %s" % (request_id, transform_id, coll_id, scope, name, deps)) + step += 1 + for dep in deps: + coll_id = dep['coll_id'] + transform_id = get_transform_id(collections, coll_id) + dep['transform_id'] = transform_id + get_dep_link(collections=collections, contents=contents, step=step, **dep) + + +def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1): + collections = core_catalog.get_collections(request_id=request_id) + contents = core_catalog.get_contents(request_id=request_id) + get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=step) + + +if __name__ == '__main__': + request_id = 1689 + transform_id = 14713 + coll_id = None + scope = 'pseudo_dataset' + name = '94248742-a255-4541-ab38-ab69364a4c88_transformPreSourceTable_23224_72+qgraphNodeId:94248742-a255-4541-ab38-ab69364a4c88+qgraphId:1657127232.749359-46373' + + # get_input_dep(request_id, transform_id, scope, name) + get_dep_links(request_id, transform_id, coll_id, scope, name) diff --git a/main/tools/env/config_monitor.py b/main/tools/env/config_monitor.py new file mode 100644 index 00000000..336cca99 --- /dev/null +++ b/main/tools/env/config_monitor.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2022 + + +import argparse +import logging + + +def config_api_host(conf_file_template="data/conf.js.template", conf_file='data/conf.js', hostname=None): + with open(conf_file_template, 'r') as f: + template = f.read() + template = template.format(api_host_name=hostname) + with open(conf_file, 'w') as f: + f.write(template) + + +logging.getLogger().setLevel(logging.INFO) +parser = argparse.ArgumentParser(description="config iDDS monitor") +parser.add_argument('-s', '--source', default=None, help='Source config file path') +parser.add_argument('-d', '--destination', default=None, help='Destination file path') +parser.add_argument('--host', default=None, help='idds host name') +args = parser.parse_args() + + +config_api_host(conf_file_template=args.source, conf_file=args.destination, hostname=args.host) diff --git a/main/tools/env/install_packages.sh b/main/tools/env/install_packages.sh new file mode 100755 index 00000000..70d615b8 --- /dev/null +++ b/main/tools/env/install_packages.sh @@ -0,0 +1,6 @@ +$'#!/bin/bash +set -m +for package in common main client workflow doma atlas website monitor ; +do + python3 -m pip install `ls $package/dist/*.tar.gz` +done diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 39e1faff..f8739468 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true" } From ae6c685a50715faf46eab100d7e6eceffad25df4 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 17:10:35 +0200 Subject: [PATCH 24/25] new version 0.11.5 --- atlas/lib/idds/atlas/version.py | 2 +- atlas/tools/env/environment.yml | 4 ++-- client/lib/idds/client/version.py | 2 +- client/tools/env/environment.yml | 4 ++-- common/lib/idds/common/version.py | 2 +- doma/lib/idds/doma/version.py | 2 +- doma/tools/env/environment.yml | 4 ++-- main/lib/idds/version.py | 2 +- main/tools/env/environment.yml | 6 +++--- monitor/lib/idds/monitor/version.py | 2 +- website/lib/idds/website/version.py | 2 +- workflow/lib/idds/workflow/version.py | 2 +- workflow/tools/env/environment.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/atlas/lib/idds/atlas/version.py b/atlas/lib/idds/atlas/version.py index 67b93aa1..7c2e8610 100644 --- a/atlas/lib/idds/atlas/version.py +++ b/atlas/lib/idds/atlas/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/atlas/tools/env/environment.yml b/atlas/tools/env/environment.yml index dccd5ab3..fdfef10d 100644 --- a/atlas/tools/env/environment.yml +++ b/atlas/tools/env/environment.yml @@ -13,5 +13,5 @@ dependencies: - panda-client # panda client - rucio-clients - rucio-clients-atlas - - idds-common==0.11.4 - - idds-workflow==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/client/lib/idds/client/version.py b/client/lib/idds/client/version.py index 67b93aa1..7c2e8610 100644 --- a/client/lib/idds/client/version.py +++ b/client/lib/idds/client/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/client/tools/env/environment.yml b/client/tools/env/environment.yml index 5076e4f1..45756d68 100644 --- a/client/tools/env/environment.yml +++ b/client/tools/env/environment.yml @@ -14,5 +14,5 @@ dependencies: - nose # nose test tools - tabulate - argcomplete - - idds-common==0.11.4 - - idds-workflow==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/common/lib/idds/common/version.py b/common/lib/idds/common/version.py index 67b93aa1..7c2e8610 100644 --- a/common/lib/idds/common/version.py +++ b/common/lib/idds/common/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/doma/lib/idds/doma/version.py b/doma/lib/idds/doma/version.py index e7dd71a7..a9a272c5 100644 --- a/doma/lib/idds/doma/version.py +++ b/doma/lib/idds/doma/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2020 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/doma/tools/env/environment.yml b/doma/tools/env/environment.yml index 6224f7d4..4847bdb4 100644 --- a/doma/tools/env/environment.yml +++ b/doma/tools/env/environment.yml @@ -10,5 +10,5 @@ dependencies: - pytest # python testing tool - nose # nose test tools - panda-client # panda client - - idds-common==0.11.4 - - idds-workflow==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 \ No newline at end of file diff --git a/main/lib/idds/version.py b/main/lib/idds/version.py index 67b93aa1..7c2e8610 100644 --- a/main/lib/idds/version.py +++ b/main/lib/idds/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/main/tools/env/environment.yml b/main/tools/env/environment.yml index a0a39f16..2a964de1 100644 --- a/main/tools/env/environment.yml +++ b/main/tools/env/environment.yml @@ -24,6 +24,6 @@ dependencies: - sphinx-rtd-theme # sphinx readthedoc theme - nevergrad # nevergrad hyper parameter optimization - psycopg2-binary - - idds-common==0.11.4 - - idds-workflow==0.11.4 - - idds-client==0.11.4 \ No newline at end of file + - idds-common==0.11.5 + - idds-workflow==0.11.5 + - idds-client==0.11.5 \ No newline at end of file diff --git a/monitor/lib/idds/monitor/version.py b/monitor/lib/idds/monitor/version.py index 67b93aa1..7c2e8610 100644 --- a/monitor/lib/idds/monitor/version.py +++ b/monitor/lib/idds/monitor/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/website/lib/idds/website/version.py b/website/lib/idds/website/version.py index 67b93aa1..7c2e8610 100644 --- a/website/lib/idds/website/version.py +++ b/website/lib/idds/website/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/workflow/lib/idds/workflow/version.py b/workflow/lib/idds/workflow/version.py index 67b93aa1..7c2e8610 100644 --- a/workflow/lib/idds/workflow/version.py +++ b/workflow/lib/idds/workflow/version.py @@ -9,4 +9,4 @@ # - Wen Guan, , 2019 - 2021 -release_version = "0.11.4" +release_version = "0.11.5" diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index b878a2c1..a3eb0093 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -8,4 +8,4 @@ dependencies: - flake8 # Wrapper around PyFlakes&pep8 - pytest # python testing tool - nose # nose test tools - - idds-common==0.11.4 \ No newline at end of file + - idds-common==0.11.5 \ No newline at end of file From 0e823a8ded6f329e4888ff370869990691ecd829 Mon Sep 17 00:00:00 2001 From: Wen Guan Date: Mon, 11 Jul 2022 17:20:21 +0200 Subject: [PATCH 25/25] merge --- Dockerfile | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6bb0ded..b0529616 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,13 +48,6 @@ RUN chown atlpan -R /opt/idds RUN chown atlpan -R /var/log/idds RUN chown apache -R /var/idds/wsgisocks/ -# to run with non-root PID -RUN chmod -R 777 /var/log/idds -RUN chmod -R 777 /var/idds -RUN chmod -R 777 /etc/httpd/conf.d -RUN chmod -R 777 /run/httpd -RUN chmod -R 777 /var/log/httpd - # setup conda virtual env ADD requirements.yaml /opt/idds/ # ADD start-daemon.sh /opt/idds/ @@ -73,19 +66,6 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client -WORKDIR /tmp/src -COPY . . - -RUN echo $'#!/bin/bash \n\ -set -m \n\ -for package in common main client workflow doma atlas website monitor ; \n\ -do \n\ - python3 -m pip install `ls $package/dist/*.tar.gz` \n\ -done \n ' > inst_packages.sh - -RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ - if [[ -z "$TAG" ]] ; then \ - python3 setup.py sdist && chmod +x inst_packages.sh && ./inst_packages.sh ; \ WORKDIR /tmp/src COPY . . @@ -104,10 +84,7 @@ RUN chmod 777 /opt/idds/monitor/data RUN chmod 777 /opt/idds/monitor/data/conf.js RUN mkdir /opt/idds/config RUN mkdir /opt/idds/config/idds - -# to run with non-root PID -RUN chmod -R 777 /opt/idds/config -RUN chmod -R 777 /opt/idds/config_default +# RUN mkdir /opt/idds/config_default # ADD idds.cfg.default /opt/idds/config @@ -115,15 +92,6 @@ RUN chmod -R 777 /opt/idds/config_default # RUN ls /opt/idds/config; ls /opt/idds/config/idds; # for rest service - -# to grant low-numbered port access to non-root -RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd -RUN chmod -R 777 /etc/grid-security - -# required for ssl.conf to run with non-root PID -RUN chmod a+r /etc/pki/tls/certs/localhost.crt -RUN chmod a+r /etc/pki/tls/private/localhost.key - # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem