Merge pull request #78 from HSF/dev

new version 0.11.0

Dockerfile

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <[email protected]>, 2022
+
+
+FROM docker.io/centos:7
+
+ARG TAG
+
+WORKDIR /tmp
+
+RUN yum install -y epel-release.noarch && \
+     yum clean all && \
+     rm -rf /var/cache/yum
+RUN yum upgrade -y && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+RUN yum install -y httpd.x86_64 conda gridsite mod_ssl.x86_64 httpd-devel.x86_64 gcc.x86_64 supervisor.noarch fetch-crl.noarch lcg-CA postgresql postgresql-contrib postgresql-static postgresql-libs postgresql-devel && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+# RUN curl http://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo -o /etc/yum.repos.d/EGI-trustanchors.repo
+RUN curl https://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo -o /etc/yum.repos.d/EGI-trustanchors.repo
+
+RUN yum install -y fetch-crl.noarch lcg-CA ca-policy-egi-core && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+# setup env
+RUN adduser atlpan
+RUN groupadd zp
+RUN usermod -a -G zp atlpan
+
+RUN mkdir /opt/idds
+RUN mkdir /var/log/idds
+RUN mkdir /var/log/idds/wsgisocks/
+RUN mkdir /var/idds
+RUN mkdir /var/idds/wsgisocks
+RUN chown atlpan -R /opt/idds
+# RUN chown atlpan -R /opt/idds_source
+RUN chown atlpan /var/log/idds
+RUN chown apache -R /var/idds/wsgisocks/
+
+# setup conda virtual env
+ADD requirements.yaml /opt/idds/
+# ADD start-daemon.sh /opt/idds/
+RUN conda env create --prefix=/opt/idds -f /opt/idds/requirements.yaml
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds
+# RUN conda activate /opt/idds
+
+# Make RUN commands use the new environment:
+# SHELL ["conda", "run", "-p", "/opt/idds", "/bin/bash", "-c"]
+
+# install required packages
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade pip
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade setuptools
+
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle  unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG
+
+RUN mkdir /opt/idds/config
+RUN mkdir /opt/idds/config/idds
+# RUN mkdir /opt/idds/config_default
+
+# ADD idds.cfg.default /opt/idds/config
+
+# RUN ls /opt/idds/etc; ls /opt/idds/etc/idds; ls /opt/idds/etc/panda;
+# RUN ls /opt/idds/config; ls /opt/idds/config/idds;
+
+# for rest service
+# RUN ln -fs /opt/idds/config/hostkey.pem /etc/grid-security/hostkey.pem
+# RUN ln -fs /opt/idds/config/hostcert.pem /etc/grid-security/hostcert.pem
+
+# to authenticate to rucio
+RUN ln -fs /opt/idds/config/ca.crt /opt/idds/etc/ca.crt
+RUN ln -fs /opt/idds/config/rucio.cfg /opt/idds/etc/rucio.cfg
+
+# for panda client to access panda
+RUN ln -fs /opt/idds/config/panda.cfg /opt/idds/etc/panda/panda.cfg
+
+# for idds rest service
+RUN ln -fs /opt/idds/config/idds/idds.cfg /opt/idds/etc/idds/idds.cfg
+RUN ln -fs /opt/idds/config/idds/auth.cfg /opt/idds/etc/idds/auth/auth.cfg
+RUN ln -fs /opt/idds/config/idds/gacl /opt/idds/etc/idds/rest/gacl
+RUN ln -fs /opt/idds/config/idds/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf
+
+# for idds daemons
+RUN ln -fs /opt/idds/config/idds/supervisord_idds.ini /etc/supervisord.d/idds.ini
+
+ENV PATH /opt/idds/bin/:$PATH
+
+ADD start-daemon.sh /opt/idds/bin/
+
+VOLUME /var/log/idds
+VOLUME /opt/idds/config
+
+ENTRYPOINT ["start-daemon.sh"]
+
+STOPSIGNAL SIGINT
+
+EXPOSE 443
+CMD ["all"]

README.md

@@ -6,7 +6,7 @@ data for High Energy Physics workloads.
 
 Home page
 ---------
-https://idds.cern.ch
+https://iddsserver.cern.ch/website/
 
 
 Documents

atlas/lib/idds/atlas/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.10.5"
+release_version = "0.11.0"

atlas/tools/env/environment.yml

@@ -10,7 +10,8 @@ dependencies:
   - pytest           # python testing tool
   - nose             # nose test tools
   - stomp.py
+  - panda-client     # panda client
   - rucio-clients
   - rucio-clients-atlas
-  - idds-common==0.10.5
-  - idds-workflow==0.10.5
+  - idds-common==0.11.0
+  - idds-workflow==0.11.0

client/bin/idds

@@ -298,7 +298,7 @@ if __name__ == '__main__':
 
     try:
         if args.verbose:
-            logging.setLevel(logging.DEBUG)
+            logging.getLogger().setLevel(logging.DEBUG)
         start_time = time.time()
 
         result = args.function(args)

client/lib/idds/client/clientmanager.py

@@ -106,7 +106,7 @@ def get_local_cfg_file(self):
         return local_cfg
 
     def get_config_value(self, configuration, section, name, current, default):
-        if type(configuration) in [str]:
+        if configuration and type(configuration) in [str]:
             config = ConfigParser.SafeConfigParser()
             config.read(configuration)
             configuration = config
@@ -116,11 +116,18 @@ def get_config_value(self, configuration, section, name, current, default):
     def get_local_configuration(self):
         local_cfg = self.get_local_cfg_file()
         config = ConfigParser.SafeConfigParser()
-        if os.path.exists(local_cfg):
+        if not local_cfg:
+            logging.debug("local configuration file does not exist, will only load idds default value.")
+        if local_cfg and os.path.exists(local_cfg):
             config.read(local_cfg)
 
-        self.config = self.get_config_value(config, section='common', name='config', current=self.config,
-                                            default=os.path.join(self.get_local_config_root(), 'idds.cfg'))
+        if self.get_local_config_root():
+            self.config = self.get_config_value(config, section='common', name='config', current=self.config,
+                                                default=os.path.join(self.get_local_config_root(), 'idds.cfg'))
+        else:
+            self.config = self.get_config_value(config, section='common', name='config', current=self.config,
+                                                default=None)
+
         self.auth_type = self.get_config_value(config, 'common', 'auth_type', current=self.auth_type, default='x509_proxy')
 
         self.host = self.get_config_value(config, 'rest', 'host', current=self.host, default=None)
@@ -133,16 +140,24 @@ def get_local_configuration(self):
             if proxy:
                 self.x509_proxy = proxy
 
-        self.oidc_token = self.get_config_value(config, 'oidc', 'oidc_token', current=self.oidc_token,
-                                                default=os.path.join(self.get_local_config_root(), '.oidc_token'))
+        if self.get_local_config_root():
+            self.oidc_token = self.get_config_value(config, 'oidc', 'oidc_token', current=self.oidc_token,
+                                                    default=os.path.join(self.get_local_config_root(), '.oidc_token'))
+        else:
+            self.oidc_token = self.get_config_value(config, 'oidc', 'oidc_token', current=self.oidc_token,
+                                                    default=None)
+
         self.vo = self.get_config_value(config, self.auth_type, 'vo', current=self.vo, default=None)
 
         self.configuration = config
 
     def save_local_configuration(self):
         local_cfg = self.get_local_cfg_file()
-        with open(local_cfg, 'w') as configfile:
-            self.configuration.write(configfile)
+        if not local_cfg:
+            logging.debug("local configuration file does not exist, will not store current setup.")
+        else:
+            with open(local_cfg, 'w') as configfile:
+                self.configuration.write(configfile)
 
     def setup_local_configuration(self, local_config_root=None, config=None, host=None,
                                   auth_type=None, auth_type_host=None, x509_proxy=None,

client/lib/idds/client/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.10.5"
+release_version = "0.11.0"

client/tools/env/environment.yml

@@ -6,7 +6,6 @@ dependencies:
   - requests                         # requests
   - urllib3                          # url connections
   - flask                            # web service
-  - futures                          # multiple process/threads
   - stomp.py                         # Messaging broker client
   - unittest2        # unit test tool
   - pep8             # checks for PEP8 code style compliance
@@ -15,5 +14,5 @@ dependencies:
   - nose             # nose test tools
   - tabulate
   - argcomplete
-  - idds-common==0.10.5
-  - idds-workflow==0.10.5
+  - idds-common==0.11.0
+  - idds-workflow==0.11.0

common/lib/idds/common/cache.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <[email protected]>, 2022
+
+import json
+from dogpile.cache import make_region
+
+from idds.common.config import config_has_section, config_has_option, config_get
+
+
+def get_cache_url():
+    if config_has_section('cache') and config_has_option('cache', 'url'):
+        return config_get('cache', 'url')
+    return '127.0.0.1:11211'
+
+
+def make_region_memcached(expiration_time, function_key_generator=None):
+    """
+    Make and configure a dogpile.cache.memcached region
+    """
+    if function_key_generator:
+        region = make_region(function_key_generator=function_key_generator)
+    else:
+        region = make_region()
+
+    region.configure(
+        'dogpile.cache.memcached',
+        expiration_time=expiration_time,
+        arguments={
+            'url': get_cache_url,
+            'distributed_lock': True,
+            'memcached_expire_time': expiration_time + 60,  # must be bigger than expiration_time
+        }
+    )
+
+    return region
+
+
+REGION = make_region_memcached(expiration_time=3600)
+
+
+def update_cache(key, data):
+    REGION.set(key, json.dumps(data))
+
+
+def get_cache(key):
+    data = REGION.get(key)
+    if data:
+        return json.loads(data)
+    return data
+
+
+def delete_cache(key):
+    REGION.delete(key)

common/lib/idds/common/config.py

@@ -27,6 +27,19 @@
     import configparser as ConfigParser
 
 
+def is_client():
+    if 'IDDS_CLIENT_MODE' in os.environ and os.environ['IDDS_CLIENT_MODE']:
+        client_mode = True
+    else:
+        client_mode = False
+        try:
+            import idds.agents.common                # noqa F401
+        except ModuleNotFoundError as ex:            # noqa F841
+            client_mode = True
+
+    return client_mode
+
+
 def config_has_section(section):
     """
     Return where there is a section
@@ -122,22 +135,36 @@ def get_local_config_root(local_config_root=None):
 
     if local_config_root is None:
         # local_config_root = "~/.idds"
-        local_config_root = os.path.join(os.path.expanduser("~"), ".idds")
-
-    if not os.path.exists(local_config_root):
-        os.makedirs(local_config_root)
+        home_dir = os.path.expanduser("~")
+        if os.access(home_dir, os.W_OK):
+            local_config_root = os.path.join(home_dir, ".idds")
+        else:
+            # username = os.getlogin()
+            # if username == 'root':
+            #     local_config_root = os.path.join('/tmp', ".idds")
+            # else:
+            #     local_config_root = os.path.join(os.path.join("/tmp", username), ".idds")
+            pass
+
+    if local_config_root and not os.path.exists(local_config_root):
+        try:
+            os.makedirs(local_config_root, exist_ok=True)
+        except Exception as ex:
+            print("Failed to create %s: %s", local_config_root, str(ex))
     return local_config_root
 
 
 def get_local_cfg_file(local_config_root=None):
     local_config_root = get_local_config_root(local_config_root)
-    local_cfg = os.path.join(local_config_root, 'idds_local.cfg')
-    return local_cfg
+    if local_config_root:
+        local_cfg = os.path.join(local_config_root, 'idds_local.cfg')
+        return local_cfg
+    return None
 
 
 def get_local_config_value(configuration, section, name, current, default):
     value = None
-    if configuration.has_section(section) and configuration.has_option(section, name):
+    if configuration and configuration.has_section(section) and configuration.has_option(section, name):
         if name in ['oidc_refresh_lifetime']:
             value = configuration.getint(section, name)
         elif name in ['oidc_auto', 'oidc_polling']:
@@ -149,14 +176,15 @@ def get_local_config_value(configuration, section, name, current, default):
     elif value is None:
         value = default
 
-    if not configuration.has_section(section):
+    if configuration and not configuration.has_section(section):
         configuration.add_section(section)
     if value is not None:
         if name in ['oidc_refresh_lifetime']:
             value = str(value)
         elif name in ['oidc_auto', 'oidc_polling']:
             value = str(value).lower()
-        configuration.set(section, name, value)
+        if configuration:
+            configuration.set(section, name, value)
     return value
 
 
@@ -187,11 +215,12 @@ def get_config():
             __CONFIG.read(local_cfg)
             __HAS_CONFIG = True
         else:
-            raise Exception("Could not load configuration file."
-                            "For iDDS client, please run 'idds setup' to create local config file."
-                            "For an iDDS server, IDDS looks for a configuration file, in order:"
-                            "\n\t${IDDS_CONFIG}"
-                            "\n\t${IDDS_HOME}/etc/idds/idds.cfg"
-                            "\n\t/etc/idds/idds.cfg"
-                            "\n\t${VIRTUAL_ENV}/etc/idds/idds.cfg")
+            if not is_client():
+                raise Exception("Could not load configuration file."
+                                "For iDDS client, please run 'idds setup' to create local config file."
+                                "For an iDDS server, IDDS looks for a configuration file, in order:"
+                                "\n\t${IDDS_CONFIG}"
+                                "\n\t${IDDS_HOME}/etc/idds/idds.cfg"
+                                "\n\t/etc/idds/idds.cfg"
+                                "\n\t${VIRTUAL_ENV}/etc/idds/idds.cfg")
     return __CONFIG