update docker deployment

Dockerfile

@@ -65,7 +65,20 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle  unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client
-RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG
+
+
+WORKDIR /tmp/src
+COPY . .
+
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \
+  if [[ -z "$TAG" ]] ; then \
+  python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \
+  else \
+  python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \
+  fi
+
+WORKDIR /tmp
+RUN rm -rf /tmp/src
 
 RUN chmod 777 /opt/idds/monitor/data
 RUN chmod 777 /opt/idds/monitor/data/conf.js
@@ -119,7 +132,8 @@ ENV PATH /opt/idds/bin/:$PATH
 
 ADD start-daemon.sh /opt/idds/bin/
 RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back
-ADD ssl.conf /etc/httpd/conf.d/ssl.conf
+# ADD ssl.conf /etc/httpd/conf.d/ssl.conf
+RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf
 
 VOLUME /var/log/idds
 VOLUME /opt/idds/config

main/etc/idds/rest/ssl.conf

@@ -0,0 +1,55 @@
+#
+# When we also provide SSL we have to listen to the 
+# the HTTPS port in addition.
+#
+# Listen 443 https
+
+##
+##  SSL Global Context
+##
+##  All SSL configuration in this context applies both to
+##  the main server and all SSL-enabled virtual hosts.
+##
+
+#   Pass Phrase Dialog:
+#   Configure the pass phrase gathering process.
+#   The filtering dialog program (`builtin' is a internal
+#   terminal dialog) has to provide the pass phrase on stdout.
+SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog
+
+#   Inter-Process Session Cache:
+#   Configure the SSL Session Cache: First the mechanism 
+#   to use and second the expiring timeout (in seconds).
+SSLSessionCache         shmcb:/run/httpd/sslcache(512000)
+SSLSessionCacheTimeout  300
+
+#   Pseudo Random Number Generator (PRNG):
+#   Configure one or more sources to seed the PRNG of the 
+#   SSL library. The seed data should be of good random quality.
+#   WARNING! On some platforms /dev/random blocks if not enough entropy
+#   is available. This means you then cannot use the /dev/random device
+#   because it would lead to very long connection times (as long as
+#   it requires to make more entropy available). But usually those
+#   platforms additionally provide a /dev/urandom device which doesn't
+#   block. So, if available, use this one instead. Read the mod_ssl User
+#   Manual for more details.
+SSLRandomSeed startup file:/dev/urandom  256
+SSLRandomSeed connect builtin
+#SSLRandomSeed startup file:/dev/random  512
+#SSLRandomSeed connect file:/dev/random  512
+#SSLRandomSeed connect file:/dev/urandom 512
+
+#
+# Use "SSLCryptoDevice" to enable any supported hardware
+# accelerators. Use "openssl engine -v" to list supported
+# engine names.  NOTE: If you enable an accelerator and the
+# server does not start, consult the error logs and ensure
+# your accelerator is functioning properly. 
+#
+SSLCryptoDevice builtin
+#SSLCryptoDevice ubsec
+
+##
+## SSL Virtual Host Context
+##
+

main/etc/rucio.cfg.default

@@ -0,0 +1,20 @@
+# Copyright European Organization for Nuclear Research (CERN)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Authors:
+# - Vincent Garonne, <[email protected]>, 2013
+
+[common]
+
+[client]
+rucio_host = https://voatlasrucio-server-prod.cern.ch:443
+auth_host = https://voatlasrucio-auth-prod.cern.ch:443
+ca_cert = $RUCIO_HOME/etc/ca.crt
+client_cert = ~/.globus/usercert.pem
+client_key = ~/.globus/userkey.pem
+client_x509_proxy = $X509_USER_PROXY
+auth_type = x509_proxy
+request_retries = 3

main/etc/sql/postgresql.sql

@@ -0,0 +1,6 @@
+CREATE USER doma_idds_r WITH PASSWORD 'Tiaroa4dr_idds';
+GRANT CONNECT ON DATABASE doma_idds TO doma_idds_r;
+GRANT USAGE ON SCHEMA doma_idds TO doma_idds_r;
+GRANT SELECT ON ALL TABLES IN SCHEMA doma_idds TO doma_idds_r;
+ALTER DEFAULT PRIVILEGES IN SCHEMA doma_idds GRANT SELECT ON TABLES TO doma_idds_r;
+

main/lib/idds/tests/auth_test_script.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <[email protected]>, 2021 - 2022
+
+
+"""
+Test authentication.
+"""
+
+try:
+    from urllib import urlencode        # noqa F401
+except ImportError:
+    from urllib.parse import urlencode  # noqa F401
+    raw_input = input
+
+import datetime
+import sys
+import time
+
+import unittest2 as unittest
+# from nose.tools import assert_equal
+from idds.common.utils import setup_logging
+from idds.common.authentication import OIDCAuthentication
+
+
+setup_logging(__name__)
+
+
+class TestAuthentication:
+
+    def test_oidc_authentication(self):
+        vo = 'iamdev'
+
+        oidc = OIDCAuthentication()
+        allow_vos = oidc.get_allow_vos()
+        print("allow_vos")
+        print(allow_vos)
+        assert(vo in allow_vos)
+        auth_config = oidc.get_auth_config(vo)
+        print("auth_config")
+        print(auth_config)
+        assert('vo' in auth_config)
+        assert(auth_config['vo'] == vo)
+
+        endpoint_config = oidc.get_endpoint_config(auth_config)
+        print("endpoint_config")
+        print(endpoint_config)
+        assert('token_endpoint' in endpoint_config)
+
+        status, sign_url = oidc.get_oidc_sign_url(vo)
+        print("sign_url")
+        print(sign_url)
+        assert('user_code' in sign_url)
+        print(("Please go to {0} and sign in. "
+               "Waiting until authentication is completed").format(sign_url['verification_uri_complete']))
+
+        print('Ready to get ID token?')
+        while True:
+            sys.stdout.write("[y/n] \n")
+            choice = raw_input().lower()
+            if choice == 'y':
+                break
+            elif choice == 'n':
+                print('aborted')
+                return
+
+        if 'interval' in sign_url:
+            interval = sign_url['interval']
+        else:
+            interval = 5
+
+        if 'expires_in' in sign_url:
+            expires_in = sign_url['expires_in']
+        else:
+            expires_in = 60
+
+        token = None
+        start_time = datetime.datetime.utcnow()
+        while datetime.datetime.utcnow() - start_time < datetime.timedelta(seconds=expires_in):
+            try:
+                status, output = oidc.get_id_token(vo, sign_url['device_code'])
+                if status:
+                    # print(output)
+                    token = output
+                    break
+                else:
+                    if type(output) in [dict] and 'error' in output and output['error'] == 'authorization_pending':
+                        time.sleep(interval)
+                    else:
+                        print(output)
+                        break
+            except Exception as error:
+                print(error)
+                break
+
+        if not token:
+            print("Failed to get a token")
+        else:
+            print(token)
+            assert('id_token' in token)
+
+            status, new_token = oidc.refresh_id_token(vo, token['refresh_token'])
+            # print(new_token)
+            assert('id_token' in new_token)
+
+            print("verifying the token")
+            status, decoded_token, username = oidc.verify_id_token(vo, token['id_token'])
+            if not status:
+                print("Failed to verify the token: %s" % decoded_token)
+            else:
+                print(username)
+                print(decoded_token)
+
+
+if __name__ == '__main__':
+    test = TestAuthentication()
+    test.test_oidc_authentication()

main/lib/idds/tests/find_dependencies.py

@@ -0,0 +1,75 @@
+import sys
+import datetime
+
+from idds.common.utils import json_dumps                 # noqa F401
+from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking          # noqa F401
+from idds.core.requests import get_requests              # noqa F401
+from idds.core.messages import retrieve_messages         # noqa F401
+from idds.core.transforms import get_transforms          # noqa F401
+from idds.core.workprogress import get_workprogresses    # noqa F401
+from idds.core.processings import get_processings        # noqa F401
+from idds.core import transforms as core_transforms      # noqa F401
+from idds.core import catalog as core_catalog            # noqa F401
+from idds.orm.contents import get_input_contents
+from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old     # noqa F401
+
+
+def get_input_dep(contents, request_id, transform_id, coll_id, scope, name):
+    # print(request_id, transform_id, coll_id, scope, name)
+    map_id = None
+    for content in contents:
+        if content['transform_id'] == transform_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output:
+            # print(content)
+            print("output name: %s, status: %s" % (content['name'], content['status']))
+            map_id = content['map_id']
+        # elif coll_id and content['coll_id'] == coll_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output:
+        #    # print(content)
+        #    map_id = content['map_id']
+
+    print(map_id)
+    deps = []
+    for content in contents:
+        if content['transform_id'] == transform_id and content['map_id'] == map_id and content['content_relation_type'] == ContentRelationType.InputDependency:
+            # print(content)
+            print("Input dependency name: %s, status: %s" % (content['name'], content['status']))
+            deps.append({'request_id': content['request_id'],
+                         'transform_id': content['transform_id'],
+                         'coll_id': content['coll_id'],
+                         'scope': content['scope'],
+                         'name': content['name']})
+    return deps
+
+
+def get_transform_id(collections, coll_id):
+    for coll in collections:
+        if coll['coll_id'] == coll_id:
+            return coll['transform_id']
+
+
+def get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=1):
+    deps = get_input_dep(contents, request_id, transform_id, coll_id, scope, name)
+    print("Step: %s" % step)
+    print("(%s, %s, %s, %s, %s) depents on %s" % (request_id, transform_id, coll_id, scope, name, deps))
+    step += 1
+    for dep in deps:
+        coll_id = dep['coll_id']
+        transform_id = get_transform_id(collections, coll_id)
+        dep['transform_id'] = transform_id
+        get_dep_link(collections=collections, contents=contents, step=step, **dep)
+
+
+def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1):
+    collections = core_catalog.get_collections(request_id=request_id)
+    contents = core_catalog.get_contents(request_id=request_id)
+    get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=step)
+
+
+if __name__ == '__main__':
+    request_id = 1689
+    transform_id = 14713
+    coll_id = None
+    scope = 'pseudo_dataset'
+    name = '94248742-a255-4541-ab38-ab69364a4c88_transformPreSourceTable_23224_72+qgraphNodeId:94248742-a255-4541-ab38-ab69364a4c88+qgraphId:1657127232.749359-46373'
+
+    # get_input_dep(request_id, transform_id, scope, name)
+    get_dep_links(request_id, transform_id, coll_id, scope, name)

main/tools/env/config_monitor.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0OA
+#
+# Authors:
+# - Wen Guan, <[email protected]>, 2022
+
+
+import argparse
+import logging
+
+
+def config_api_host(conf_file_template="data/conf.js.template", conf_file='data/conf.js', hostname=None):
+    with open(conf_file_template, 'r') as f:
+        template = f.read()
+    template = template.format(api_host_name=hostname)
+    with open(conf_file, 'w') as f:
+        f.write(template)
+
+
+logging.getLogger().setLevel(logging.INFO)
+parser = argparse.ArgumentParser(description="config iDDS monitor")
+parser.add_argument('-s', '--source', default=None, help='Source config file path')
+parser.add_argument('-d', '--destination', default=None, help='Destination file path')
+parser.add_argument('--host', default=None, help='idds host name')
+args = parser.parse_args()
+
+
+config_api_host(conf_file_template=args.source, conf_file=args.destination, hostname=args.host)

main/tools/env/install_packages.sh

@@ -0,0 +1,6 @@
+$'#!/bin/bash
+set -m
+for package in common main client workflow doma atlas website monitor ;
+do
+  python3 -m pip install `ls $package/dist/*.tar.gz`
+done

monitor/data/conf.js

@@ -1,9 +1,9 @@
 
 var appConfig = {
-    'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null",
-    'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null",
-    'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null",
-    'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false",
-    'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false",
-    'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true"
+    'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null",
+    'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null",
+    'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null",
+    'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false",
+    'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false",
+    'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true"
 }