Merge pull request #342 from HSF/dev

Dev

Dockerfile

@@ -39,6 +39,14 @@ RUN yum install --nogpgcheck -y postgresql
 RUN  yum clean all && rm -rf /var/cache/yum
 
 
+# install Oracle Instant Client and tnsnames.ora
+RUN wget https://download.oracle.com/otn_software/linux/instantclient/oracle-instantclient-basic-linuxx64.rpm -P /tmp/ && \
+    yum install -y /tmp/oracle-instantclient-basic-linuxx64.rpm&& \
+    wget https://download.oracle.com/otn_software/linux/instantclient/oracle-instantclient-sqlplus-linuxx64.rpm -P /tmp/ && \
+    yum install -y /tmp/oracle-instantclient-sqlplus-linuxx64.rpm && \
+    wget https://download.oracle.com/otn_software/linux/instantclient/oracle-instantclient-sqlplus-linuxx64.rpm -P /tmp/ && \
+    yum install -y /tmp/oracle-instantclient-devel-linuxx64.rpm
+
 # RUN curl http://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo -o /etc/yum.repos.d/EGI-trustanchors.repo/
 RUN curl https://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo -o /etc/yum.repos.d/EGI-trustanchors.repo
 
@@ -91,7 +99,7 @@ RUN source /etc/profile.d/conda.sh; conda activate /opt/idds
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade pip
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade setuptools
 
-RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle  unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
+RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle oracledb unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade psycopg2-binary
 RUN source /etc/profile.d/conda.sh; conda activate /opt/idds; python3 -m pip install --no-cache-dir --upgrade rucio-clients-atlas rucio-clients panda-client-light
 

environment.yml

@@ -11,6 +11,7 @@ dependencies:
   #- web.py                           # web service
   - stomp.py                         # Messaging broker client
   - cx-Oracle                        # Oralce client
+  - oracledb
   - unittest2        # unit test tool
   - pep8             # checks for PEP8 code style compliance
   - flake8           # Wrapper around PyFlakes&pep8

main/lib/idds/agents/carrier/plugins/base.py

@@ -8,10 +8,14 @@
 # Authors:
 # - Wen Guan, <[email protected]>, 2024
 
+import logging
 import json
 
 from idds.common.constants import WorkflowType
-from idds.common.utils import encode_base64
+from idds.common.utils import encode_base64, setup_logging
+
+
+setup_logging(__name__)
 
 
 class BaseSubmitter(object):
@@ -100,6 +104,7 @@ def get_task_params(self, work):
             task_param_map['maxFailure'] = work.max_attempt
 
         if work.enable_separate_log:
+            logging.debug(f"BaseSubmitter enable_separate_log: {work.enable_separate_log}")
             task_param_map['log'] = {"dataset": "PandaJob_iworkflow/",   # "PandaJob_#{pandaid}/"
                                      "destination": "local",
                                      "param_type": "log",

main/lib/idds/orm/base/session.py

@@ -143,6 +143,8 @@ def get_engine(echo=True):
             except:  # noqa: B901
                 pass
         params['execution_options'] = {'schema_translate_map': {None: DEFAULT_SCHEMA_NAME}}
+        if 'oracledb' in sql_connection:
+            params['thick_mode'] = True
         _ENGINE = create_engine(sql_connection, **params)
 
         if 'mysql' in sql_connection:

main/lib/idds/orm/base/utils.py

@@ -125,6 +125,7 @@ def destroy_everything(echo=True):
 def dump_schema():
     """ Creates a schema dump to a specific database. """
     engine = session.get_dump_engine()
+    models.unregister_models(engine)
     models.register_models(engine)
 
 

main/lib/idds/orm/processings.py

@@ -16,7 +16,7 @@
 import datetime
 
 import sqlalchemy
-from sqlalchemy import func
+from sqlalchemy import func, select
 from sqlalchemy.exc import DatabaseError, IntegrityError
 from sqlalchemy.sql.expression import asc
 
@@ -160,25 +160,24 @@ def get_processing_by_id_status(processing_id, status=None, locking=False, sessi
     """
 
     try:
-        query = session.query(models.Processing)\
-                       .filter(models.Processing.processing_id == processing_id)
+        query = select(models.Processing).filter(models.Processing.processing_id == processing_id)
 
         if status:
             if not isinstance(status, (list, tuple)):
                 status = [status]
             if len(status) == 1:
                 status = [status[0], status[0]]
-            query = query.filter(models.Processing.status.in_(status))
+            query = query.where(models.Processing.status.in_(status))
 
         if locking:
-            query = query.filter(models.Processing.locking == ProcessingLocking.Idle)
+            query = query.where(models.Processing.locking == ProcessingLocking.Idle)
             query = query.with_for_update(skip_locked=True)
 
-        ret = query.first()
+        ret = session.execute(query).fetchone()
         if not ret:
             return None
         else:
-            return ret.to_dict()
+            return ret[0].to_dict()
     except sqlalchemy.orm.exc.NoResultFound as error:
         raise exceptions.NoObject('processing processing_id: %s cannot be found: %s' % (processing_id, error))