Merge pull request #81 from wguanicedew/dev

Dev

Dockerfile

@@ -45,7 +45,7 @@ RUN mkdir /var/idds
 RUN mkdir /var/idds/wsgisocks
 RUN chown atlpan -R /opt/idds
 # RUN chown atlpan -R /opt/idds_source
-RUN chown atlpan /var/log/idds
+RUN chown atlpan -R /var/log/idds
 RUN chown apache -R /var/idds/wsgisocks/
 
 # setup conda virtual env
@@ -65,8 +65,23 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle  unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client
-RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG
 
+
+WORKDIR /tmp/src
+COPY . .
+
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \
+  if [[ -z "$TAG" ]] ; then \
+  python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \
+  else \
+  python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \
+  fi
+
+WORKDIR /tmp
+RUN rm -rf /tmp/src
+
+RUN chmod 777 /opt/idds/monitor/data
+RUN chmod 777 /opt/idds/monitor/data/conf.js
 RUN mkdir /opt/idds/config
 RUN mkdir /opt/idds/config/idds
 # RUN mkdir /opt/idds/config_default
@@ -93,12 +108,32 @@ RUN ln -fs /opt/idds/config/idds/auth.cfg /opt/idds/etc/idds/auth/auth.cfg
 RUN ln -fs /opt/idds/config/idds/gacl /opt/idds/etc/idds/rest/gacl
 RUN ln -fs /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf
 
+# update http config
+RUN sed -i 's/Listen\ 443/#\ Listen\ 443/g' /etc/httpd/conf.d/ssl.conf
+RUN sed -i 's/Listen\ 80/#\ Listen\ 80/g' /etc/httpd/conf/httpd.conf
+RUN sed -i "s/WSGISocketPrefix\ \/var\/log\/idds\/wsgisocks\/wsgi/WSGISocketPrefix\ \/var\/idds\/wsgisocks\/wsgi/g" /opt/idds/config_default/httpd-idds-443-py39-cc7.conf
+
 # for idds daemons
 RUN ln -fs /opt/idds/config/idds/supervisord_idds.ini /etc/supervisord.d/idds.ini
 
+RUN chmod -R 777 /opt/idds/config
+RUN chmod -R 777 /var/log/idds
+RUN chmod 777 /etc/grid-security
+RUN chmod 777 /etc/httpd/conf.d
+RUN chmod 777 /etc/httpd/conf/httpd.conf
+RUN chmod 777 /etc/httpd/conf
+RUN chmod 777 /run/httpd
+RUN chmod 777 /var/log/supervisor/
+RUN chmod 777 /var/run/supervisor
+RUN chmod 777 /var/run
+RUN chmod 777 /etc/httpd/logs
+
 ENV PATH /opt/idds/bin/:$PATH
 
 ADD start-daemon.sh /opt/idds/bin/
+RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back
+# ADD ssl.conf /etc/httpd/conf.d/ssl.conf
+RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf
 
 VOLUME /var/log/idds
 VOLUME /opt/idds/config

atlas/lib/idds/atlas/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

atlas/lib/idds/atlas/workflowv2/atlaspandawork.py

@@ -117,6 +117,23 @@ def load_panda_urls(self):
                 self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl')
                 os.environ['PANDA_URL_SSL'] = self.panda_url_ssl
                 # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl))
+            if panda_config.has_option('panda', 'pandacache_url'):
+                self.pandacache_url = panda_config.get('panda', 'pandacache_url')
+                os.environ['PANDACACHE_URL'] = self.pandacache_url
+                # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+            if panda_config.has_option('panda', 'panda_verify_host'):
+                self.panda_verify_host = panda_config.get('panda', 'panda_verify_host')
+                os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host
+                # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
+            if panda_config.has_option('panda', 'panda_auth'):
+                self.panda_auth = panda_config.get('panda', 'panda_auth')
+                os.environ['PANDA_AUTH'] = self.panda_auth
+            if panda_config.has_option('panda', 'panda_auth_vo'):
+                self.panda_auth_vo = panda_config.get('panda', 'panda_auth_vo')
+                os.environ['PANDA_AUTH_VO'] = self.panda_auth_vo
+            if panda_config.has_option('panda', 'panda_config_root'):
+                self.panda_config_root = panda_config.get('panda', 'panda_config_root')
+                os.environ['PANDA_CONFIG_ROOT'] = self.panda_config_root
 
         if not self.panda_monitor and 'PANDA_MONITOR_URL' in os.environ and os.environ['PANDA_MONITOR_URL']:
             self.panda_monitor = os.environ['PANDA_MONITOR_URL']
@@ -127,6 +144,18 @@ def load_panda_urls(self):
         if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']:
             self.panda_url_ssl = os.environ['PANDA_URL_SSL']
             # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl))
+        if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']:
+            self.pandacache_url = os.environ['PANDACACHE_URL']
+            # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+        if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']:
+            self.panda_verify_host = os.environ['PANDA_VERIFY_HOST']
+            # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
+        if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']:
+            self.panda_auth = os.environ['PANDA_AUTH']
+        if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']:
+            self.panda_auth_vo = os.environ['PANDA_AUTH_VO']
+        if not self.panda_config_root and 'PANDA_CONFIG_ROOT' in os.environ and os.environ['PANDA_CONFIG_ROOT']:
+            self.panda_config_root = os.environ['PANDA_CONFIG_ROOT']
 
     def set_agent_attributes(self, attrs, req_attributes=None):
         if self.class_name not in attrs or 'life_time' not in attrs[self.class_name] or int(attrs[self.class_name]['life_time']) <= 0:

atlas/tools/env/environment.yml

@@ -13,5 +13,5 @@ dependencies:
   - panda-client     # panda client
   - rucio-clients
   - rucio-clients-atlas
-  - idds-common==0.11.0
-  - idds-workflow==0.11.0
+  - idds-common==0.11.5
+  - idds-workflow==0.11.5

client/lib/idds/client/base.py

@@ -26,7 +26,7 @@
 
 from idds.common import exceptions
 from idds.common.constants import HTTP_STATUS_CODE
-from idds.common.utils import json_dumps, json_loads
+from idds.common.utils import json_dumps, json_loads, get_proxy_path
 from idds.common.authentication import OIDCAuthenticationUtils
 
 
@@ -68,6 +68,20 @@ def __init__(self, host=None, auth=None, timeout=None, client_proxy=None):
 
         self.check_auth()
 
+    def get_user_proxy(sellf):
+        """
+        Get the user proxy.
+
+        :returns: the path of the user proxy.
+        """
+
+        client_proxy = get_proxy_path()
+
+        if not client_proxy or not os.path.exists(client_proxy):
+            raise exceptions.RestException("Cannot find a valid x509 proxy.")
+
+        return client_proxy
+
     def check_auth(self):
         """
         To check whether the auth type is supported and the input for the auth is available.
@@ -77,6 +91,8 @@ def check_auth(self):
             self.auth_type = 'x509_proxy'
 
         if self.auth_type in ['x509_proxy']:
+            if not self.client_proxy:
+                self.client_proxy = self.get_user_proxy()
             if not self.client_proxy or not os.path.exists(self.client_proxy):
                 raise exceptions.RestException("Cannot find a valid x509 proxy.")
         elif self.auth_type in ['oidc']:

client/lib/idds/client/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

client/tools/env/environment.yml

@@ -14,5 +14,5 @@ dependencies:
   - nose             # nose test tools
   - tabulate
   - argcomplete
-  - idds-common==0.11.0
-  - idds-workflow==0.11.0
+  - idds-common==0.11.5
+  - idds-workflow==0.11.5

common/lib/idds/common/authentication.py

@@ -45,6 +45,12 @@ def decode_value(val):
     return int.from_bytes(decoded, 'big')
 
 
+def should_verify():
+    if os.environ.get('IDDS_AUTH_NO_VERIFY', None):
+        return False
+    return True
+
+
 class BaseAuthentication(object):
     def __init__(self, timeout=None):
         self.timeout = timeout
@@ -98,7 +104,7 @@ def get_auth_config(self, vo):
 
     def get_http_content(self, url):
         try:
-            r = requests.get(url, allow_redirects=True)
+            r = requests.get(url, allow_redirects=True, verify=should_verify())
             return r.content
         except Exception as error:
             return False, 'Failed to get http content for %s: %s' (str(url), str(error))
@@ -128,6 +134,7 @@ def get_oidc_sign_url(self, vo):
                                              # data=json.dumps(data),
                                              urlencode(data).encode(),
                                              timeout=self.timeout,
+                                             verify=should_verify(),
                                              headers=headers)
 
             if result is not None:
@@ -172,6 +179,7 @@ def get_id_token(self, vo, device_code, interval=5, expires_in=60):
                                              # data=json.dumps(data),
                                              urlencode(data).encode(),
                                              timeout=self.timeout,
+                                             verify=should_verify(),
                                              headers=headers)
             if result is not None:
                 if result.status_code == HTTP_STATUS_CODE.OK and result.text:
@@ -203,6 +211,7 @@ def refresh_id_token(self, vo, refresh_token):
                                              # data=json.dumps(data),
                                              urlencode(data).encode(),
                                              timeout=self.timeout,
+                                             verify=should_verify(),
                                              headers=headers)
 
             if result is not None:

common/lib/idds/common/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

doma/lib/idds/doma/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2020 - 2021
 
 
-release_version = "0.11.0"
+release_version = "0.11.5"

doma/lib/idds/doma/workflowv2/domapandawork.py

@@ -89,7 +89,7 @@ def __init__(self, executable=None, arguments=None, parameters=None, setup=None,
         self.prodSourceLabel = prodSourceLabel
         self.task_type = task_type
         self.maxWalltime = maxwalltime
-        self.maxAttempt = maxattempt
+        self.maxAttempt = maxattempt if maxattempt else 5
         self.core_count = core_count
         self.task_log = task_log
 
@@ -150,6 +150,14 @@ def load_panda_urls(self):
                 self.panda_url = panda_config.get('panda', 'panda_url')
                 os.environ['PANDA_URL'] = self.panda_url
                 # self.logger.debug("Panda url: %s" % str(self.panda_url))
+            if panda_config.has_option('panda', 'pandacache_url'):
+                self.pandacache_url = panda_config.get('panda', 'pandacache_url')
+                os.environ['PANDACACHE_URL'] = self.pandacache_url
+                # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+            if panda_config.has_option('panda', 'panda_verify_host'):
+                self.panda_verify_host = panda_config.get('panda', 'panda_verify_host')
+                os.environ['PANDA_VERIFY_HOST'] = self.panda_verify_host
+                # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
             if panda_config.has_option('panda', 'panda_url_ssl'):
                 self.panda_url_ssl = panda_config.get('panda', 'panda_url_ssl')
                 os.environ['PANDA_URL_SSL'] = self.panda_url_ssl
@@ -173,6 +181,12 @@ def load_panda_urls(self):
         if not self.panda_url_ssl and 'PANDA_URL_SSL' in os.environ and os.environ['PANDA_URL_SSL']:
             self.panda_url_ssl = os.environ['PANDA_URL_SSL']
             # self.logger.debug("Panda url ssl: %s" % str(self.panda_url_ssl))
+        if not self.pandacache_url and 'PANDACACHE_URL' in os.environ and os.environ['PANDACACHE_URL']:
+            self.pandacache_url = os.environ['PANDACACHE_URL']
+            # self.logger.debug("Pandacache url: %s" % str(self.pandacache_url))
+        if not self.panda_verify_host and 'PANDA_VERIFY_HOST' in os.environ and os.environ['PANDA_VERIFY_HOST']:
+            self.panda_verify_host = os.environ['PANDA_VERIFY_HOST']
+            # self.logger.debug("Panda verify host: %s" % str(self.panda_verify_host))
         if not self.panda_auth and 'PANDA_AUTH' in os.environ and os.environ['PANDA_AUTH']:
             self.panda_auth = os.environ['PANDA_AUTH']
         if not self.panda_auth_vo and 'PANDA_AUTH_VO' in os.environ and os.environ['PANDA_AUTH_VO']:

doma/tools/env/environment.yml

@@ -10,5 +10,5 @@ dependencies:
   - pytest           # python testing tool
   - nose             # nose test tools
   - panda-client     # panda client
-  - idds-common==0.11.0
-  - idds-workflow==0.11.0
+  - idds-common==0.11.5
+  - idds-workflow==0.11.5

main/config_default/httpd-idds-443-py39-cc7.conf

@@ -34,7 +34,7 @@ WSGIPythonPath /opt/idds/lib/python3.9/site-packages
     WSGIApplicationGroup %GLOBAL
     WSGIScriptAlias /idds /opt/idds/bin/idds.wsgi
     # WSGIScriptAliasMatch ^/idds/(.+)$ /opt/idds/etc/idds/rest/test.wsgi
-    WSGISocketPrefix /var/log/idds/wsgisocks/wsgi
+    WSGISocketPrefix /var/idds/wsgisocks/wsgi
     WSGIPassAuthorization On
 </IfModule>
 
@@ -87,7 +87,7 @@ Alias "/monitor"     "/opt/idds/monitor/data"
         GridSiteDNlists /etc/grid-security/dn-lists/
         GridSiteGSIProxyLimit 16
         GridSiteEnvs on
-        # GridSiteACLPath /opt/idds/etc/idds/rest/gacl
+        GridSiteACLPath /opt/idds/etc/idds/rest/gacl
         # GridSiteMethods GET
     </LocationMatch>
 

main/config_default/idds.cfg

@@ -26,15 +26,15 @@ agents = clerk, transformer, carrier, conductor
 
 [clerk]
 num_threads = 4
-poll_time_period = 120
-poll_operation_time_period = 120
-retrieve_bulk_size = 1
+poll_time_period = 60
+poll_operation_time_period = 60
+retrieve_bulk_size = 4
 pending_time = 4
 
 [transformer]
 num_threads = 8
 poll_time_period = 60
-retrieve_bulk_size = 1
+retrieve_bulk_size = 4
 poll_operation_time_period = 240
 message_bulk_size = 1000
 
@@ -45,7 +45,7 @@ domapandawork.num_retries = 0
 num_threads = 8
 poll_time_period = 60
 poll_operation_time_period = 240
-retrieve_bulk_size = 3
+retrieve_bulk_size = 4
 message_bulk_size = 1000
 
 

main/config_default/supervisord_idds.ini

@@ -7,7 +7,7 @@ environment =
 ;command=/opt/idds/bin/run-idds
 command=bash -c "source /etc/profile.d/conda.sh && conda activate /opt/idds && /opt/idds/bin/run-idds"
 process_name=%(process_num)02d
-user=atlpan
+# user=atlpan
 childlogdir=/var/log/idds
 stdout_logfile=/var/log/idds/%(program_name)s-stdout.log
 stderr_logfile=/var/log/idds/%(program_name)s-stderr.log

main/etc/idds/rest/ssl.conf

@@ -0,0 +1,55 @@
+#
+# When we also provide SSL we have to listen to the 
+# the HTTPS port in addition.
+#
+# Listen 443 https
+
+##
+##  SSL Global Context
+##
+##  All SSL configuration in this context applies both to
+##  the main server and all SSL-enabled virtual hosts.
+##
+
+#   Pass Phrase Dialog:
+#   Configure the pass phrase gathering process.
+#   The filtering dialog program (`builtin' is a internal
+#   terminal dialog) has to provide the pass phrase on stdout.
+SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog
+
+#   Inter-Process Session Cache:
+#   Configure the SSL Session Cache: First the mechanism 
+#   to use and second the expiring timeout (in seconds).
+SSLSessionCache         shmcb:/run/httpd/sslcache(512000)
+SSLSessionCacheTimeout  300
+
+#   Pseudo Random Number Generator (PRNG):
+#   Configure one or more sources to seed the PRNG of the 
+#   SSL library. The seed data should be of good random quality.
+#   WARNING! On some platforms /dev/random blocks if not enough entropy
+#   is available. This means you then cannot use the /dev/random device
+#   because it would lead to very long connection times (as long as
+#   it requires to make more entropy available). But usually those
+#   platforms additionally provide a /dev/urandom device which doesn't
+#   block. So, if available, use this one instead. Read the mod_ssl User
+#   Manual for more details.
+SSLRandomSeed startup file:/dev/urandom  256
+SSLRandomSeed connect builtin
+#SSLRandomSeed startup file:/dev/random  512
+#SSLRandomSeed connect file:/dev/random  512
+#SSLRandomSeed connect file:/dev/urandom 512
+
+#
+# Use "SSLCryptoDevice" to enable any supported hardware
+# accelerators. Use "openssl engine -v" to list supported
+# engine names.  NOTE: If you enable an accelerator and the
+# server does not start, consult the error logs and ensure
+# your accelerator is functioning properly. 
+#
+SSLCryptoDevice builtin
+#SSLCryptoDevice ubsec
+
+##
+## SSL Virtual Host Context
+##
+