diff --git a/Dockerfile b/Dockerfile index 48c892c8..b0529616 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,7 +65,20 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip ins RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client -RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG + + +WORKDIR /tmp/src +COPY . . + +RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; \ + if [[ -z "$TAG" ]] ; then \ + python3 setup.py sdist bdist_wheel && main/tools/env/install_packages.sh ; \ + else \ + python3 -m pip install --no-cache-dir --upgrade idds-common==$TAG idds-workflow==$TAG idds-server==$TAG idds-client==$TAG idds-doma==$TAG idds-atlas==$TAG idds-website==$TAG idds-monitor==$TAG ; \ + fi + +WORKDIR /tmp +RUN rm -rf /tmp/src RUN chmod 777 /opt/idds/monitor/data RUN chmod 777 /opt/idds/monitor/data/conf.js @@ -119,7 +132,8 @@ ENV PATH /opt/idds/bin/:$PATH ADD start-daemon.sh /opt/idds/bin/ RUN mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.back -ADD ssl.conf /etc/httpd/conf.d/ssl.conf +# ADD ssl.conf /etc/httpd/conf.d/ssl.conf +RUN ln -s /opt/idds/etc/idds/rest/ssl.conf /etc/httpd/conf.d/ssl.conf VOLUME /var/log/idds VOLUME /opt/idds/config diff --git a/main/etc/idds/rest/ssl.conf b/main/etc/idds/rest/ssl.conf new file mode 100644 index 00000000..e713647c --- /dev/null +++ b/main/etc/idds/rest/ssl.conf @@ -0,0 +1,55 @@ +# +# When we also provide SSL we have to listen to the +# the HTTPS port in addition. +# +# Listen 443 https + +## +## SSL Global Context +## +## All SSL configuration in this context applies both to +## the main server and all SSL-enabled virtual hosts. +## + +# Pass Phrase Dialog: +# Configure the pass phrase gathering process. +# The filtering dialog program (`builtin' is a internal +# terminal dialog) has to provide the pass phrase on stdout. +SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog + +# Inter-Process Session Cache: +# Configure the SSL Session Cache: First the mechanism +# to use and second the expiring timeout (in seconds). +SSLSessionCache shmcb:/run/httpd/sslcache(512000) +SSLSessionCacheTimeout 300 + +# Pseudo Random Number Generator (PRNG): +# Configure one or more sources to seed the PRNG of the +# SSL library. The seed data should be of good random quality. +# WARNING! On some platforms /dev/random blocks if not enough entropy +# is available. This means you then cannot use the /dev/random device +# because it would lead to very long connection times (as long as +# it requires to make more entropy available). But usually those +# platforms additionally provide a /dev/urandom device which doesn't +# block. So, if available, use this one instead. Read the mod_ssl User +# Manual for more details. +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +#SSLRandomSeed startup file:/dev/random 512 +#SSLRandomSeed connect file:/dev/random 512 +#SSLRandomSeed connect file:/dev/urandom 512 + +# +# Use "SSLCryptoDevice" to enable any supported hardware +# accelerators. Use "openssl engine -v" to list supported +# engine names. NOTE: If you enable an accelerator and the +# server does not start, consult the error logs and ensure +# your accelerator is functioning properly. +# +SSLCryptoDevice builtin +#SSLCryptoDevice ubsec + +## +## SSL Virtual Host Context +## + diff --git a/main/etc/rucio.cfg.default b/main/etc/rucio.cfg.default new file mode 100644 index 00000000..d7045960 --- /dev/null +++ b/main/etc/rucio.cfg.default @@ -0,0 +1,20 @@ +# Copyright European Organization for Nuclear Research (CERN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Authors: +# - Vincent Garonne, , 2013 + +[common] + +[client] +rucio_host = https://voatlasrucio-server-prod.cern.ch:443 +auth_host = https://voatlasrucio-auth-prod.cern.ch:443 +ca_cert = $RUCIO_HOME/etc/ca.crt +client_cert = ~/.globus/usercert.pem +client_key = ~/.globus/userkey.pem +client_x509_proxy = $X509_USER_PROXY +auth_type = x509_proxy +request_retries = 3 diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql new file mode 100644 index 00000000..b6f5e9c5 --- /dev/null +++ b/main/etc/sql/postgresql.sql @@ -0,0 +1,6 @@ +CREATE USER doma_idds_r WITH PASSWORD 'Tiaroa4dr_idds'; +GRANT CONNECT ON DATABASE doma_idds TO doma_idds_r; +GRANT USAGE ON SCHEMA doma_idds TO doma_idds_r; +GRANT SELECT ON ALL TABLES IN SCHEMA doma_idds TO doma_idds_r; +ALTER DEFAULT PRIVILEGES IN SCHEMA doma_idds GRANT SELECT ON TABLES TO doma_idds_r; + diff --git a/main/lib/idds/tests/auth_test_script.py b/main/lib/idds/tests/auth_test_script.py new file mode 100644 index 00000000..ba3715a7 --- /dev/null +++ b/main/lib/idds/tests/auth_test_script.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2021 - 2022 + + +""" +Test authentication. +""" + +try: + from urllib import urlencode # noqa F401 +except ImportError: + from urllib.parse import urlencode # noqa F401 + raw_input = input + +import datetime +import sys +import time + +import unittest2 as unittest +# from nose.tools import assert_equal +from idds.common.utils import setup_logging +from idds.common.authentication import OIDCAuthentication + + +setup_logging(__name__) + + +class TestAuthentication: + + def test_oidc_authentication(self): + vo = 'iamdev' + + oidc = OIDCAuthentication() + allow_vos = oidc.get_allow_vos() + print("allow_vos") + print(allow_vos) + assert(vo in allow_vos) + auth_config = oidc.get_auth_config(vo) + print("auth_config") + print(auth_config) + assert('vo' in auth_config) + assert(auth_config['vo'] == vo) + + endpoint_config = oidc.get_endpoint_config(auth_config) + print("endpoint_config") + print(endpoint_config) + assert('token_endpoint' in endpoint_config) + + status, sign_url = oidc.get_oidc_sign_url(vo) + print("sign_url") + print(sign_url) + assert('user_code' in sign_url) + print(("Please go to {0} and sign in. " + "Waiting until authentication is completed").format(sign_url['verification_uri_complete'])) + + print('Ready to get ID token?') + while True: + sys.stdout.write("[y/n] \n") + choice = raw_input().lower() + if choice == 'y': + break + elif choice == 'n': + print('aborted') + return + + if 'interval' in sign_url: + interval = sign_url['interval'] + else: + interval = 5 + + if 'expires_in' in sign_url: + expires_in = sign_url['expires_in'] + else: + expires_in = 60 + + token = None + start_time = datetime.datetime.utcnow() + while datetime.datetime.utcnow() - start_time < datetime.timedelta(seconds=expires_in): + try: + status, output = oidc.get_id_token(vo, sign_url['device_code']) + if status: + # print(output) + token = output + break + else: + if type(output) in [dict] and 'error' in output and output['error'] == 'authorization_pending': + time.sleep(interval) + else: + print(output) + break + except Exception as error: + print(error) + break + + if not token: + print("Failed to get a token") + else: + print(token) + assert('id_token' in token) + + status, new_token = oidc.refresh_id_token(vo, token['refresh_token']) + # print(new_token) + assert('id_token' in new_token) + + print("verifying the token") + status, decoded_token, username = oidc.verify_id_token(vo, token['id_token']) + if not status: + print("Failed to verify the token: %s" % decoded_token) + else: + print(username) + print(decoded_token) + + +if __name__ == '__main__': + test = TestAuthentication() + test.test_oidc_authentication() diff --git a/main/lib/idds/tests/find_dependencies.py b/main/lib/idds/tests/find_dependencies.py new file mode 100644 index 00000000..18f94e35 --- /dev/null +++ b/main/lib/idds/tests/find_dependencies.py @@ -0,0 +1,75 @@ +import sys +import datetime + +from idds.common.utils import json_dumps # noqa F401 +from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking # noqa F401 +from idds.core.requests import get_requests # noqa F401 +from idds.core.messages import retrieve_messages # noqa F401 +from idds.core.transforms import get_transforms # noqa F401 +from idds.core.workprogress import get_workprogresses # noqa F401 +from idds.core.processings import get_processings # noqa F401 +from idds.core import transforms as core_transforms # noqa F401 +from idds.core import catalog as core_catalog # noqa F401 +from idds.orm.contents import get_input_contents +from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401 + + +def get_input_dep(contents, request_id, transform_id, coll_id, scope, name): + # print(request_id, transform_id, coll_id, scope, name) + map_id = None + for content in contents: + if content['transform_id'] == transform_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # print(content) + print("output name: %s, status: %s" % (content['name'], content['status'])) + map_id = content['map_id'] + # elif coll_id and content['coll_id'] == coll_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output: + # # print(content) + # map_id = content['map_id'] + + print(map_id) + deps = [] + for content in contents: + if content['transform_id'] == transform_id and content['map_id'] == map_id and content['content_relation_type'] == ContentRelationType.InputDependency: + # print(content) + print("Input dependency name: %s, status: %s" % (content['name'], content['status'])) + deps.append({'request_id': content['request_id'], + 'transform_id': content['transform_id'], + 'coll_id': content['coll_id'], + 'scope': content['scope'], + 'name': content['name']}) + return deps + + +def get_transform_id(collections, coll_id): + for coll in collections: + if coll['coll_id'] == coll_id: + return coll['transform_id'] + + +def get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=1): + deps = get_input_dep(contents, request_id, transform_id, coll_id, scope, name) + print("Step: %s" % step) + print("(%s, %s, %s, %s, %s) depents on %s" % (request_id, transform_id, coll_id, scope, name, deps)) + step += 1 + for dep in deps: + coll_id = dep['coll_id'] + transform_id = get_transform_id(collections, coll_id) + dep['transform_id'] = transform_id + get_dep_link(collections=collections, contents=contents, step=step, **dep) + + +def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1): + collections = core_catalog.get_collections(request_id=request_id) + contents = core_catalog.get_contents(request_id=request_id) + get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=step) + + +if __name__ == '__main__': + request_id = 1689 + transform_id = 14713 + coll_id = None + scope = 'pseudo_dataset' + name = '94248742-a255-4541-ab38-ab69364a4c88_transformPreSourceTable_23224_72+qgraphNodeId:94248742-a255-4541-ab38-ab69364a4c88+qgraphId:1657127232.749359-46373' + + # get_input_dep(request_id, transform_id, scope, name) + get_dep_links(request_id, transform_id, coll_id, scope, name) diff --git a/main/tools/env/config_monitor.py b/main/tools/env/config_monitor.py new file mode 100644 index 00000000..336cca99 --- /dev/null +++ b/main/tools/env/config_monitor.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0OA +# +# Authors: +# - Wen Guan, , 2022 + + +import argparse +import logging + + +def config_api_host(conf_file_template="data/conf.js.template", conf_file='data/conf.js', hostname=None): + with open(conf_file_template, 'r') as f: + template = f.read() + template = template.format(api_host_name=hostname) + with open(conf_file, 'w') as f: + f.write(template) + + +logging.getLogger().setLevel(logging.INFO) +parser = argparse.ArgumentParser(description="config iDDS monitor") +parser.add_argument('-s', '--source', default=None, help='Source config file path') +parser.add_argument('-d', '--destination', default=None, help='Destination file path') +parser.add_argument('--host', default=None, help='idds host name') +args = parser.parse_args() + + +config_api_host(conf_file_template=args.source, conf_file=args.destination, hostname=args.host) diff --git a/main/tools/env/install_packages.sh b/main/tools/env/install_packages.sh new file mode 100755 index 00000000..70d615b8 --- /dev/null +++ b/main/tools/env/install_packages.sh @@ -0,0 +1,6 @@ +$'#!/bin/bash +set -m +for package in common main client workflow doma atlas website monitor ; +do + python3 -m pip install `ls $package/dist/*.tar.gz` +done diff --git a/monitor/data/conf.js b/monitor/data/conf.js index 39e1faff..f8739468 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus786.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus786.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus786.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus786.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus7107.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus7107.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus7107.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus7107.cern.ch:443/idds/monitor/null/null/false/false/true" }