Merge pull request #66 from HSF/dev

new version 0.10.2

atlas/lib/idds/atlas/version.py

@@ -9,4 +9,4 @@
 # - Wen Guan, <[email protected]>, 2019 - 2021
 
 
-release_version = "0.9.1"
+release_version = "0.10.2"

atlas/lib/idds/atlas/workflowv2/atlaslocalpandawork.py

@@ -86,6 +86,7 @@ def parse_task_parameters(self, task_parameters):
             self.add_errors(str(ex))
 
     def renew_parameters_from_attributes(self):
+        super(ATLASLocalPandaWork, self).renew_parameters_from_attributes()
         if not self.task_parameters:
             return
 

atlas/lib/idds/atlas/workflowv2/atlaspandawork.py

@@ -207,9 +207,14 @@ def renew_parameters_from_attributes(self):
             return
 
         try:
+            for key in self.task_parameters:
+                if self.task_parameters[key] and type(self.task_parameters[key]) in [str]:
+                    self.task_parameters[key] = self.renew_parameter(self.task_parameters[key])
+
             if 'taskName' in self.task_parameters:
                 self.task_name = self.task_parameters['taskName']
                 self.task_name = self.renew_parameter(self.task_name)
+                self.task_parameters['taskName'] = self.task_name
                 self.set_work_name(self.task_name)
 
             if 'prodSourceLabel' in self.task_parameters:
@@ -222,6 +227,13 @@ def renew_parameters_from_attributes(self):
                         for key in jobP:
                             if jobP[key] and type(jobP[key]) in [str]:
                                 jobP[key] = self.renew_parameter(jobP[key])
+
+            if 'log' in self.task_parameters:
+                log = self.task_parameters['log']
+                for key in log:
+                    if log[key] and type(log[key]) in [str]:
+                        self.task_parameters['log'][key] = self.renew_parameter(log[key])
+
             for coll_id in self.collections:
                 coll_name = self.collections[coll_id].name
                 self.collections[coll_id].name = self.renew_parameter(coll_name)

atlas/tools/env/environment.yml

@@ -11,5 +11,5 @@ dependencies:
   - nose             # nose test tools
   - rucio-clients
   - rucio-clients-atlas
-  - idds-common==0.9.1
-  - idds-workflow==0.9.1
+  - idds-common==0.10.2
+  - idds-workflow==0.10.2

client/bin/idds

@@ -6,7 +6,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0OA
 #
 # Authors:
-# - Wen Guan, <[email protected]>, 2020
+# - Wen Guan, <[email protected]>, 2020 - 2022
 
 """
 iDDS CLI
@@ -21,73 +21,112 @@ import logging
 import os
 import sys
 import time
+# import traceback
 
 from idds.client.version import release_version
 from idds.client.clientmanager import ClientManager
 
 
+def setup(args):
+    cm = ClientManager(host=args.host, setup_client=False)
+    cm.setup_local_configuration(local_config_root=args.local_config_root,
+                                 config=args.config, host=args.host,
+                                 auth_type=args.auth_type,
+                                 auth_type_host=args.auth_type_host,
+                                 x509_proxy=args.x509_proxy,
+                                 vo=args.vo,
+                                 oidc_token=args.oidc_token)
+    return cm
+
+
+def setup_oidc_token(args):
+    cm = ClientManager(host=args.host, setup_client=True)
+    cm.setup_oidc_token()
+
+
+def clean_oidc_token(args):
+    cm = ClientManager(host=args.host, setup_client=False)
+    cm.clean_oidc_token()
+
+
+def check_oidc_token_status(args):
+    cm = ClientManager(host=args.host, setup_client=False)
+    cm.check_oidc_token_status()
+
+
+def refresh_oidc_token(args):
+    cm = ClientManager(host=args.host, setup_client=False)
+    cm.refresh_oidc_token()
+
+
+def ping(args):
+    cm = ClientManager(host=args.host, setup_client=False)
+    status = cm.ping()
+    print(status)
+
+
 def get_requests_status(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     ret = wm.get_status(request_id=args.request_id, workload_id=args.workload_id, with_detail=args.with_detail)
     print(ret)
 
 
 def abort_requests(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.abort(request_id=args.request_id, workload_id=args.workload_id)
 
 
 def suspend_requests(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.suspend(request_id=args.request_id, workload_id=args.workload_id)
 
 
 def resume_requests(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.resume(request_id=args.request_id, workload_id=args.workload_id)
 
 
 def retry_requests(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.retry(request_id=args.request_id, workload_id=args.workload_id)
 
 
 def finish_requests(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.finish(request_id=args.request_id, workload_id=args.workload_id, set_all_finished=args.set_all_finished)
 
 
 def download_logs(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.download_logs(request_id=args.request_id, workload_id=args.workload_id, dest_dir=args.dest_dir, filename=args.dest_filename)
 
 
 def upload_to_cacher(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.upload_to_cacher(args.filename)
 
 
 def download_from_cacher(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     wm.download_from_cacher(args.filename)
 
 
 def get_hyperparameters(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     ret = wm.get_hyperparameters(workload_id=args.workload_id, request_id=args.request_id, id=args.id, status=args.status, limit=args.limit)
     # print(json.dumps(ret, sort_keys=True, indent=4))
     for k in ret:
         print(k)
 
 
 def update_hyperparameter(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     ret = wm.update_hyperparameter(workload_id=args.workload_id, request_id=args.request_id, id=args.id, loss=args.loss)
     print(ret)
 
 
 def get_messages(args):
-    wm = ClientManager(host=args.host)
+    wm = ClientManager(host=args.host, setup_client=True)
     ret = wm.get_messages(request_id=args.request_id, workload_id=args.workload_id)
     status, msgs = ret
     print("status: %s" % status)
@@ -103,9 +142,50 @@ def get_parser():
 
     # common items
     oparser.add_argument('--version', action='version', version='%(prog)s ' + release_version)
-    oparser.add_argument('--config', dest="config", help="The iDDS configuration file to use.")
+    oparser.add_argument('--local_config_root', dest="local_config_root", default=None, help="The root path of local configurations. Default is ~/.idds/.")
+    oparser.add_argument('--config', dest=None, help="The iDDS configuration file to use. Default is ~/.idds/idds.cfg.")
     oparser.add_argument('--verbose', '-v', default=False, action='store_true', help="Print more verbose output.")
-    oparser.add_argument('-H', '--host', dest="host", metavar="ADDRESS", help="The iDDS Rest host. For example: https://iddsserver.cern.ch:443/idds")
+    oparser.add_argument('-H', '--host', dest="host", metavar="ADDRESS", help="The iDDS Rest host. For example: https://hostname:443/idds")
+
+    # setup
+    setup_parser = subparsers.add_parser('setup', help='Setup local configuration')
+    setup_parser.set_defaults(function=setup)
+    setup_parser.add_argument('--host', dest="auth_type_host", metavar="ADDRESS", help="The iDDS Rest host for the current auth type. For example: https://hostname:443/idds")
+    setup_parser.add_argument('--auth_type', dest='auth_type', action='store', choices=['x509_proxy', 'oidc'], default=None, help='The auth_type in [x509_proxy, oidc]. Default is x509_proxy.')
+    setup_parser.add_argument('--x509_proxy', dest='x509_proxy', action='store', default=None, help='The x509 proxy path. Default is /tmp/x509up_u%d.' % os.geteuid())
+    setup_parser.add_argument('--vo', dest='vo', action='store', default=None, help='The virtual organization for authentication.')
+    setup_parser.add_argument('--oidc_token', dest='oidc_token', default=None, help='The oidc token path. Default is {local_config_root}/.oidc_token.')
+
+    # setup token
+    token_setup_parser = subparsers.add_parser('setup_oidc_token', help='Setup authentication token')
+    token_setup_parser.set_defaults(function=setup_oidc_token)
+    # token_setup_parser.add_argument('--oidc_refresh_lifetime', dest='oidc_refresh_lifetime', default=None, help='The oidc refresh lifetime')
+    # token_setup_parser.add_argument('--oidc_issuer', dest='oidc_issuer', default=None, help='The oidc issuer')
+    # token_setup_parser.add_argument('--oidc_audience', dest='oidc_audience', default=None, help='The oidc audience')
+    # token_setup_parser.add_argument('--oidc_token', dest='oidc_token', default=None, help='The oidc token path. Default is {local_config_root}/.oidc_token.')
+    # token_setup_parser.add_argument('--oidc_auto', dest='oidc_auto', default=False, action='store_true', help='Get oidc token automatically, requiring oidc_username and oidc_password')
+    # token_setup_parser.add_argument('--oidc_username', dest='oidc_username', default=None, help='The oidc username for getting oidc token, with --oidc_auto')
+    # token_setup_parser.add_argument('--oidc_password', dest='oidc_password', default=None, help='The oidc password for getting oidc token, with --oidc_auto')
+    # token_setup_parser.add_argument('--oidc_scope', dest='oidc_scope', default=None, help='The oidc scope. Default is openid profile.')
+    # token_setup_parser.add_argument('--oidc_polling', dest='oidc_polling', default=False, help='whether polling oidc')
+    # token_setup_parser.add_argument('--saml_username', dest='saml_username', default=None, help='The SAML username')
+    # token_setup_parser.add_argument('--saml_password', dest='saml_password', default=None, help='The saml password')
+
+    # clean token
+    token_clean_parser = subparsers.add_parser('clean_oidc_token', help='Clean authentication token')
+    token_clean_parser.set_defaults(function=clean_oidc_token)
+
+    # check token status
+    token_check_parser = subparsers.add_parser('get_oidc_token_info', help='Check authentication token information')
+    token_check_parser.set_defaults(function=check_oidc_token_status)
+
+    # refresh token
+    token_refresh_parser = subparsers.add_parser('refresh_oidc_token', help='Refresh authentication token')
+    token_refresh_parser.set_defaults(function=refresh_oidc_token)
+
+    # ping
+    ping_parser = subparsers.add_parser('ping', help='Ping idds server')
+    ping_parser.set_defaults(function=ping)
 
     # get request status
     req_status_parser = subparsers.add_parser('get_requests_status', help='Get the requests status')
@@ -209,11 +289,13 @@ if __name__ == '__main__':
         if args.verbose:
             logging.setLevel(logging.DEBUG)
         start_time = time.time()
+
         result = args.function(args)
         end_time = time.time()
         if args.verbose:
             print("Completed in %-0.4f sec." % (end_time - start_time))
         sys.exit(0)
     except Exception as error:
         logging.error("Strange error: {0}".format(error))
+        # logging.error(traceback.format_exc())
         sys.exit(-1)

client/lib/idds/client/authclient.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <[email protected]>, 2022
+
+
+"""
+Auth Rest client to access IDDS system.
+"""
+
+import os
+
+from idds.client.base import BaseRestClient
+# from idds.common.constants import RequestType, RequestStatus
+
+
+class AuthClient(BaseRestClient):
+
+    """Authentication Rest client"""
+
+    AUTH_BASEURL = 'auth'
+
+    def __init__(self, host=None, auth=None, timeout=None):
+        """
+        Constructor of the BaseRestClient.
+
+        :param host: the address of the IDDS server.
+        :param client_proxy: the client certificate proxy.
+        :param timeout: timeout in seconds.
+        """
+        super(AuthClient, self).__init__(host=host, auth=auth, timeout=timeout)
+
+    def get_oidc_sign_url(self, vo):
+        """
+        Get url from the Head service for users to sign in.
+
+        :param vo: the virtual organization.
+
+        :raise exceptions if it's not got successfully.
+        """
+        path = self.AUTH_BASEURL + "/url"
+        url = self.build_url(self.host, path=os.path.join(path, str(vo), str(self.auth_type)))
+
+        sign_url = self.get_request_response(url, type='GET', auth_setup_step=True)
+
+        return sign_url
+
+    def get_id_token(self, vo, device_code, interval=5, expires_in=60):
+        """
+        Get token from the Head service.
+
+        :param vo: the virtual organization.
+        :param device_code: the device code.
+        :param interval: the interval to poll the token.
+        :param expires_in: the time in seconds to expire for polling.
+
+        :raise exceptions if it's not got successfully.
+        """
+        path = self.AUTH_BASEURL + "/token"
+        url = self.build_url(self.host, path=os.path.join(path, str(vo), str(device_code), str(interval), str(expires_in)))
+
+        token = self.get_request_response(url, type='GET', auth_setup_step=True)
+
+        return token
+
+    def refresh_id_token(self, vo, refresh_token):
+        """
+        Refresh token from the Head service.
+
+        :param vo: the virtual organization.
+        :param refresh_token: the token from refreshing.
+
+        :raise exceptions if it's not got successfully.
+        """
+        path = self.AUTH_BASEURL + "/token"
+        url = self.build_url(self.host, path=os.path.join(path, str(vo)))
+
+        data = {'refresh_token': refresh_token}
+        token = self.get_request_response(url, type='POST', data=data)
+
+        return token