Merge pull request #83 from HSF/dev

merge dev to master

Dockerfile

@@ -45,16 +45,9 @@ RUN mkdir /var/idds
 RUN mkdir /var/idds/wsgisocks
 RUN chown atlpan -R /opt/idds
 # RUN chown atlpan -R /opt/idds_source
-RUN chown atlpan /var/log/idds
+RUN chown atlpan -R /var/log/idds
 RUN chown apache -R /var/idds/wsgisocks/
 
-# to run with non-root PID
-RUN chmod -R 777 /var/log/idds
-RUN chmod -R 777 /var/idds
-RUN chmod -R 777 /etc/httpd/conf.d
-RUN chmod -R 777 /run/httpd
-RUN chmod -R 777 /var/log/httpd
-
 # setup conda virtual env
 ADD requirements.yaml /opt/idds/
 # ADD start-daemon.sh /opt/idds/
@@ -73,48 +66,32 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client
 
+
 WORKDIR /tmp/src
 COPY . .
 
-RUN echo $'#!/bin/bash \n\
-set -m \n\
-for package in common main client workflow doma atlas website monitor ; \n\
-do \n\
-  python3 -m pip install `ls $package/dist/*.tar.gz` \n\
-done \n ' > inst_packages.sh
-
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \
   if [[ -z "$TAG" ]] ; then \
-  python3 setup.py sdist && chmod +x inst_packages.sh && ./inst_packages.sh ; \
+  python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \
   else \
   python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \
   fi
 
 WORKDIR /tmp
 RUN rm -rf /tmp/src
 
+RUN chmod 777 /opt/idds/monitor/data
+RUN chmod 777 /opt/idds/monitor/data/conf.js
 RUN mkdir /opt/idds/config
 RUN mkdir /opt/idds/config/idds
-
-# to run with non-root PID
-RUN chmod -R 777 /opt/idds/config
-RUN chmod -R 777 /opt/idds/config_default
+# RUN mkdir /opt/idds/config_default
 
 # ADD idds.cfg.default /opt/idds/config
 
 # RUN ls /opt/idds/etc; ls /opt/idds/etc/idds; ls /opt/idds/etc/panda;
 # RUN ls /opt/idds/config; ls /opt/idds/config/idds;
 
 # for rest service
-
-# to grant low-numbered port access to non-root
-RUN setcap CAP_NET_BIND_SERVICE=+eip /usr/sbin/httpd
-RUN chmod -R 777 /etc/grid-security
-
-# required for ssl.conf to run with non-root PID
-RUN chmod a+r /etc/pki/tls/certs/localhost.crt
-RUN chmod a+r /etc/pki/tls/private/localhost.key
-
 # RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem
 # RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem
 
@@ -131,12 +108,32 @@ RUN ln -fs /opt/idds/config/idds/auth.cfg /opt/idds/etc/idds/auth/auth.cfg
 RUN ln -fs /opt/idds/config/idds/gacl /opt/idds/etc/idds/rest/gacl
 RUN ln -fs /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf
 
+# update http config
+RUN sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf
+RUN sed -i 's/Listen\ 80/#\ Listen\ 80/g' /etc/httpd/conf/httpd.conf
+RUN sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf
+
 # for idds daemons
 RUN ln -fs /opt/idds/config/idds/supervisord_idds.ini /etc/supervisord.d/idds.ini
 
+RUN chmod -R 777 /opt/idds/config
+RUN chmod -R 777 /var/log/idds
+RUN chmod 777 /etc/grid-security
+RUN chmod 777 /etc/httpd/conf.d
+RUN chmod 777 /etc/httpd/conf/httpd.conf
+RUN chmod 777 /etc/httpd/conf
+RUN chmod 777 /run/httpd
+RUN chmod 777 /var/log/supervisor/
+RUN chmod 777 /var/run/supervisor
+RUN chmod 777 /var/run
+RUN chmod 777 /etc/httpd/logs
+
 ENV PATH /opt/idds/bin/:$PATH
 
 ADD start-daemon.sh /opt/idds/bin/
+RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back
+# ADD ssl.conf /etc/httpd/conf.d/ssl.conf
+RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf
 
 VOLUME /var/log/idds
 VOLUME /opt/idds/config

atlas/lib/idds/atlas/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

atlas/lib/idds/atlas/workflowv2/atlaspandawork.py

@@ -117,6 +117,23 @@ def load_panda_urls(self):
                 self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl')
                 os.environ['PANDA_URL_SSL'] = self.panda_url_ssl
                 # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl))
+            if panda_config.has_option('panda', 'pandacache_url'):
+                self.pandacache_url = panda_config.get('panda', 'pandacache_url')
+                os.environ['PANDACACHE_URL'] = self.pandacache_url
+                # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+            if panda_config.has_option('panda', 'panda_verify_host'):
+                self.panda_verify_host = panda_config.get('panda', 'panda_verify_host')
+                os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host
+                # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
+            if panda_config.has_option('panda', 'panda_auth'):
+                self.panda_auth = panda_config.get('panda', 'panda_auth')
+                os.environ['PANDA_AUTH'] = self.panda_auth
+            if panda_config.has_option('panda', 'panda_auth_vo'):
+                self.panda_auth_vo = panda_config.get('panda', 'panda_auth_vo')
+                os.environ['PANDA_AUTH_VO'] = self.panda_auth_vo
+            if panda_config.has_option('panda', 'panda_config_root'):
+                self.panda_config_root = panda_config.get('panda', 'panda_config_root')
+                os.environ['PANDA_CONFIG_ROOT'] = self.panda_config_root
 
         if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']:
             self.panda_monitor = os.environ['PANDA_MONITOR_URL']
@@ -127,6 +144,18 @@ def load_panda_urls(self):
         if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']:
             self.panda_url_ssl = os.environ['PANDA_URL_SSL']
             # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl))
+        if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']:
+            self.pandacache_url = os.environ['PANDACACHE_URL']
+            # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+        if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']:
+            self.panda_verify_host = os.environ['PANDA_VERIFY_HOST']
+            # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
+        if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']:
+            self.panda_auth = os.environ['PANDA_AUTH']
+        if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']:
+            self.panda_auth_vo = os.environ['PANDA_AUTH_VO']
+        if not self.panda_config_root and 'PANDA_CONFIG_ROOT' in os.environ and os.environ['PANDA_CONFIG_ROOT']:
+            self.panda_config_root = os.environ['PANDA_CONFIG_ROOT']
 
     def set_agent_attributes(self, attrs, req_attributes=None):
         if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0:

atlas/tools/env/environment.yml

@@ -13,5 +13,5 @@ dependencies:
   - panda-client     # panda client
   - rucio-clients
   - rucio-clients-atlas
-  - idds-common==0.11.0
-  - idds-workflow==0.11.0
+  - idds-common==0.11.5
+  - idds-workflow==0.11.5

client/lib/idds/client/base.py

@@ -26,7 +26,7 @@
 
 from idds.common import exceptions
 from idds.common.constants import HTTP_STATUS_CODE
-from idds.common.utils import json_dumps, json_loads
+from idds.common.utils import json_dumps, json_loads, get_proxy_path
 from idds.common.authentication import OIDCAuthenticationUtils
 
 
@@ -68,6 +68,20 @@ def __init__(self, host=None, auth=None, timeout=None, client_proxy=None):
 
         self.check_auth()
 
+    def get_user_proxy(sellf):
+        """
+        Get the user proxy.
+
+        :returns: the path of the user proxy.
+        """
+
+        client_proxy = get_proxy_path()
+
+        if not client_proxy or not os.path.exists(client_proxy):
+            raise exceptions.RestException("Cannot find a valid x509 proxy.")
+
+        return client_proxy
+
     def check_auth(self):
         """
         To check whether the auth type is supported and the input for the auth is available.
@@ -77,6 +91,8 @@ def check_auth(self):
             self.auth_type = 'x509_proxy'
 
         if self.auth_type in ['x509_proxy']:
+            if not self.client_proxy:
+                self.client_proxy = self.get_user_proxy()
             if not self.client_proxy or not os.path.exists(self.client_proxy):
                 raise exceptions.RestException("Cannot find a valid x509 proxy.")
         elif self.auth_type in ['oidc']:

client/lib/idds/client/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

client/tools/env/environment.yml

@@ -14,5 +14,5 @@ dependencies:
   - nose             # nose test tools
   - tabulate
   - argcomplete
-  - idds-common==0.11.0
-  - idds-workflow==0.11.0
+  - idds-common==0.11.5
+  - idds-workflow==0.11.5

common/lib/idds/common/authentication.py

@@ -45,6 +45,12 @@ def decode_value(val):
     return int.from_bytes(decoded, 'big')
 
 
+def should_verify():
+    if os.environ.get('IDDS_AUTH_NO_VERIFY', None):
+        return False
+    return True
+
+
 class BaseAuthentication(object):
     def __init__(self, timeout=None):
         self.timeout = timeout
@@ -98,7 +104,7 @@ def get_auth_config(self, vo):
 
     def get_http_content(self, url):
         try:
-            r = requests.get(url, allow_redirects=True)
+            r = requests.get(url, allow_redirects=True, verify=should_verify())
             return r.content
         except Exception as error:
             return False, 'Failed to get http content for %s: %s' (str(url), str(error))
@@ -128,6 +134,7 @@ def get_oidc_sign_url(self, vo):
                                              # data=json.dumps(data),
                                              urlencode(data).encode(),
                                              timeout=self.timeout,
+                                             verify=should_verify(),
                                              headers=headers)
 
             if result is not None:
@@ -172,6 +179,7 @@ def get_id_token(self, vo, device_code, interval=5, expires_in=60):
                                              # data=json.dumps(data),
                                              urlencode(data).encode(),
                                              timeout=self.timeout,
+                                             verify=should_verify(),
                                              headers=headers)
             if result is not None:
                 if result.status_code == HTTP_STATUS_CODE.OK and result.text:
@@ -203,6 +211,7 @@ def refresh_id_token(self, vo, refresh_token):
                                              # data=json.dumps(data),
                                              urlencode(data).encode(),
                                              timeout=self.timeout,
+                                             verify=should_verify(),
                                              headers=headers)
 
             if result is not None:

common/lib/idds/common/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

doma/lib/idds/doma/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2020 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

doma/lib/idds/doma/workflowv2/domapandawork.py

@@ -89,7 +89,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None,
         self.prodSourceLabel = prodSourceLabel
         self.task_type = task_type
         self.maxWalltime = maxwalltime
-        self.maxAttempt = maxattempt
+        self.maxAttempt = maxattempt if maxattempt else 5
         self.core_count = core_count
         self.task_log = task_log
 
@@ -150,6 +150,14 @@ def load_panda_urls(self):
                 self.panda_url = panda_config.get('panda', 'panda_url')
                 os.environ['PANDA_URL'] = self.panda_url
                 # self.logger.debug("Panda url: %s" % str(self.panda_url))
+            if panda_config.has_option('panda', 'pandacache_url'):
+                self.pandacache_url = panda_config.get('panda', 'pandacache_url')
+                os.environ['PANDACACHE_URL'] = self.pandacache_url
+                # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+            if panda_config.has_option('panda', 'panda_verify_host'):
+                self.panda_verify_host = panda_config.get('panda', 'panda_verify_host')
+                os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host
+                # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
             if panda_config.has_option('panda', 'panda_url_ssl'):
                 self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl')
                 os.environ['PANDA_URL_SSL'] = self.panda_url_ssl
@@ -173,6 +181,12 @@ def load_panda_urls(self):
         if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']:
             self.panda_url_ssl = os.environ['PANDA_URL_SSL']
             # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl))
+        if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']:
+            self.pandacache_url = os.environ['PANDACACHE_URL']
+            # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+        if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']:
+            self.panda_verify_host = os.environ['PANDA_VERIFY_HOST']
+            # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
         if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']:
             self.panda_auth = os.environ['PANDA_AUTH']
         if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']:

doma/tools/env/environment.yml

@@ -10,5 +10,5 @@ dependencies:
   - pytest           # python testing tool
   - nose             # nose test tools
   - panda-client     # panda client
-  - idds-common==0.11.0
-  - idds-workflow==0.11.0
+  - idds-common==0.11.5
+  - idds-workflow==0.11.5

main/config_default/httpd-idds-443-py39-cc7.conf

@@ -34,7 +34,7 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages
     WSGIApplicationGroup %GLOBAL
     WSGIScriptAlias /idds /opt/idds/bin/idds.wsgi
     # WSGIScriptAliasMatch ^/idds/(.+)$ /opt/idds/etc/idds/rest/test.wsgi
-    WSGISocketPrefix /var/log/idds/wsgisocks/wsgi
+    WSGISocketPrefix /var/idds/wsgisocks/wsgi
     WSGIPassAuthorization On
 </IfModule>
 
@@ -87,7 +87,7 @@ Alias "/monitor"     "/opt/idds/monitor/data"
         GridSiteDNlists /etc/grid-security/dn-lists/
         GridSiteGSIProxyLimit 16
         GridSiteEnvs on
-        # GridSiteACLPath /opt/idds/etc/idds/rest/gacl
+        GridSiteACLPath /opt/idds/etc/idds/rest/gacl
         # GridSiteMethods GET
     </LocationMatch>
 

main/config_default/idds.cfg

@@ -26,15 +26,15 @@ agents = clerk, transformer, carrier, conductor
 
 [clerk]
 num_threads = 4
-poll_time_period = 120
-poll_operation_time_period = 120
-retrieve_bulk_size = 1
+poll_time_period = 60
+poll_operation_time_period = 60
+retrieve_bulk_size = 4
 pending_time = 4
 
 [transformer]
 num_threads = 8
 poll_time_period = 60
-retrieve_bulk_size = 1
+retrieve_bulk_size = 4
 poll_operation_time_period = 240
 message_bulk_size = 1000
 
@@ -45,7 +45,7 @@ domapandawork.num_retries = 0
 num_threads = 8
 poll_time_period = 60
 poll_operation_time_period = 240
-retrieve_bulk_size = 3
+retrieve_bulk_size = 4
 message_bulk_size = 1000
 
 

main/config_default/supervisord_idds.ini

@@ -7,7 +7,7 @@ environment =
 ;command=/opt/idds/bin/run-idds
 command=bash -c "source /etc/profile.d/conda.sh && conda activate /opt/idds && /opt/idds/bin/run-idds"
 process_name=%(process_num)02d
-user=atlpan
+# user=atlpan
 childlogdir=/var/log/idds
 stdout_logfile=/var/log/idds/%(program_name)s-stdout.log
 stderr_logfile=/var/log/idds/%(program_name)s-stderr.log