Merge pull request #302 from HSF/dev

Dev

common/lib/idds/common/utils.py

@@ -7,9 +7,10 @@
 #
 # Authors:
 # - Wen Guan, <[email protected]>, 2019 - 2024
-
+# - Lino Oscar Gerlach, <[email protected]>, 2024
 
 import base64
+import contextlib
 import errno
 import datetime
 import importlib
@@ -942,3 +943,32 @@ def idds_mask(dict_):
         else:
             ret[k] = dict_[k]
     return ret
+
+
+@contextlib.contextmanager
+def modified_environ(*remove, **update):
+    """
+    Temporarily updates the ``os.environ`` dictionary in-place.
+    The ``os.environ`` dictionary is updated in-place so that the modification
+    is sure to work in all situations.
+    :param remove: Environment variables to remove.
+    :param update: Dictionary of environment variables and values to add/update.
+    """
+    env = os.environ
+    update = update or {}
+    remove = remove or []
+
+    # List of environment variables being updated or removed.
+    stomped = (set(update.keys()) | set(remove)) & set(env.keys())
+    # Environment variables and values to restore on exit.
+    update_after = {k: env[k] for k in stomped}
+    # Environment variables and values to remove on exit.
+    remove_after = frozenset(k for k in update if k not in env)
+
+    try:
+        env.update(update)
+        [env.pop(k, None) for k in remove]
+        yield
+    finally:
+        env.update(update_after)
+        [env.pop(k) for k in remove_after]

main/etc/sql/postgresql.sql

@@ -181,8 +181,8 @@ CREATE TABLE doma_idds.contents_ext (
 	memory_leak VARCHAR(10), 
 	memory_leak_x2 VARCHAR(10), 
 	job_label VARCHAR(20), 
-	CONSTRAINT "CONTENTS_EXT_PK" PRIMARY KEY (content_id, request_id)
-) PARTITION BY RANGE (request_id) ;
+	CONSTRAINT "CONTENTS_EXT_PK" PRIMARY KEY (content_id)
+);
 
 CREATE INDEX "CONTENTS_EXT_RTW_IDX" ON doma_idds.contents_ext (request_id, transform_id, workload_id);
 
@@ -500,13 +500,13 @@ CREATE TABLE doma_idds.contents (
 	accessed_at TIMESTAMP WITHOUT TIME ZONE, 
 	expired_at TIMESTAMP WITHOUT TIME ZONE, 
 	content_metadata VARCHAR(1000), 
-	CONSTRAINT "CONTENTS_PK_TEST" PRIMARY KEY (content_id, request_id),
-	CONSTRAINT "CONTENT_ID_UQ_TEST" UNIQUE (transform_id, coll_id, request_id, map_id, sub_map_id, dep_sub_map_id, content_relation_type, name_md5, scope_name_md5, min_id, max_id),
-    CONSTRAINT "CONTENTS_TRANSFORM_ID_FK" FOREIGN KEY(transform_id) REFERENCES doma_idds.transforms (transform_id),
+	CONSTRAINT "CONTENTS_PK" PRIMARY KEY (content_id), 
+	CONSTRAINT "CONTENT_ID_UQ" UNIQUE (transform_id, coll_id, map_id, sub_map_id, dep_sub_map_id, content_relation_type, name_md5, scope_name_md5, min_id, max_id), 
+	CONSTRAINT "CONTENTS_TRANSFORM_ID_FK" FOREIGN KEY(transform_id) REFERENCES doma_idds.transforms (transform_id), 
 	CONSTRAINT "CONTENTS_COLL_ID_FK" FOREIGN KEY(coll_id) REFERENCES doma_idds.collections (coll_id), 
 	CONSTRAINT "CONTENTS_STATUS_ID_NN" CHECK (status IS NOT NULL), 
 	CONSTRAINT "CONTENTS_COLL_ID_NN" CHECK (coll_id IS NOT NULL)
-) PARTITION BY RANGE (request_id) ;
+);
 
 CREATE INDEX "CONTENTS_STATUS_UPDATED_IDX" ON doma_idds.contents (status, locking, updated_at, created_at);
 
@@ -518,7 +518,8 @@ CREATE INDEX "CONTENTS_REQ_TF_COLL_IDX" ON doma_idds.contents (request_id, trans
 
 CREATE INDEX "CONTENTS_TF_IDX" ON doma_idds.contents (transform_id, request_id, coll_id, map_id, content_relation_type);
 
-CREATE INDEX "CONTENTS_ID_NAME_IDX" ON doma_idds.contents (coll_id, scope, md5('name'), status);
+CREATE INDEX "CONTENTS_ID_NAME_IDX" ON doma_idds.contents (coll_id, scope, md5('name');
+, status);
 
 
         SET search_path TO doma_idds;

main/tools/env/setup_panda.sh

@@ -56,6 +56,37 @@ elif [ "$instance" == "usdf" ]; then
 
     # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/
     export PANDA_CONFIG_ROOT=~/.panda/
+elif [ "$instance" == "new" ]; then
+    export PANDA_AUTH=oidc
+    export PANDA_URL_SSL=https://ai-idds-05.cern.ch:25443/server/panda
+    export PANDA_URL=http://ai-idds-05.cern.ch:25080/server/panda
+    export PANDAMON_URL=https://panda-doma.cern.ch
+    # export PANDA_AUTH_VO=panda_dev
+    export PANDA_AUTH_VO=Rubin:production
+
+    export PANDACACHE_URL=$PANDA_URL_SSL
+
+    export PANDA_SYS=/afs/cern.ch/user/w/wguan/workdisk/iDDS/.conda/iDDS/
+    # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/
+    export PANDA_CONFIG_ROOT=~/.panda/
+
+    # export IDDS_HOST=https://aipanda015.cern.ch:443/idds
+
+    # dev
+    # export IDDS_HOST=https://aipanda104.cern.ch:443/idds
+
+    # doma
+    export IDDS_HOST=https://aipanda105.cern.ch:443/idds
+
+    # export IDDS_BROKERS=atlas-test-mb.cern.ch:61013
+    # export IDDS_BROKER_DESTINATION=/topic/doma.idds
+    # export IDDS_BROKER_USERNAME=domaidds
+    # export IDDS_BROKER_PASSWORD=1d25yeft6krJ1HFH
+    # export IDDS_BROKER_TIMEOUT=360
+
+    PANDA_QUEUE=BNL_OSG_2
+    PANDA_WORKING_GROUP=EIC
+    PANDA_VO=wlcg
 else
     export PANDA_AUTH=oidc
     export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda

monitor/data/conf.js

@@ -1,9 +1,9 @@
 
 var appConfig = {
-    'iddsAPI_request': "https://lxplus926.cern.ch:443/idds/monitor_request/null/null",
-    'iddsAPI_transform': "https://lxplus926.cern.ch:443/idds/monitor_transform/null/null",
-    'iddsAPI_processing': "https://lxplus926.cern.ch:443/idds/monitor_processing/null/null",
-    'iddsAPI_request_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/true/false/false",
-    'iddsAPI_transform_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/false/true/false",
-    'iddsAPI_processing_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/false/false/true"
+    'iddsAPI_request': "https://lxplus935.cern.ch:443/idds/monitor_request/null/null",
+    'iddsAPI_transform': "https://lxplus935.cern.ch:443/idds/monitor_transform/null/null",
+    'iddsAPI_processing': "https://lxplus935.cern.ch:443/idds/monitor_processing/null/null",
+    'iddsAPI_request_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/true/false/false",
+    'iddsAPI_transform_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/false/true/false",
+    'iddsAPI_processing_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/false/false/true"
 }

website/data/index.html

@@ -61,7 +61,7 @@ <h2>Intelligent</h2>
 					<div class="inner">
 						<header>
 							<p><b>Fine grained data transformation</b></p>
-                                                        <p><b>Remote data transformation/reduction</b></p>
+                                                        <p><b>Remtoe data transformation/reduction</b></p>
                                                         <p><b>On-demand production of analysis format data transformation</b></p>
 							<h2>Transformation</h2>
 						</header>

workflow/lib/idds/iworkflow/work.py

@@ -7,6 +7,7 @@
 #
 # Authors:
 # - Wen Guan, <[email protected]>, 2023 - 2024
+# - Lino Oscar Gerlach, <[email protected]>, 2024
 
 import base64
 import datetime
@@ -21,7 +22,7 @@
 from idds.common import exceptions
 from idds.common.constants import WorkflowType, TransformStatus
 from idds.common.imports import get_func_name
-from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64
+from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64, modified_environ
 from .asyncresult import AsyncResult, MapResult
 from .base import Base, Context
 from .workflow import WorkflowCanvas
@@ -779,9 +780,8 @@ def load(self, func_name):
 
         :raise Exception
         """
-        os.environ['IDDS_IWORKFLOW_LOAD_WORK'] = 'true'
-        func = super(Work, self).load(func_name)
-        del os.environ['IDDS_IWORKFLOW_LOAD_WORK']
+        with modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'):
+            func = super(Work, self).load(func_name)
 
         return func
 
@@ -922,7 +922,7 @@ def work(func=None, *, map_results=False, lazy=False, init_env=None):
     if func is None:
         return functools.partial(work, map_results=map_results, lazy=lazy, init_env=init_env)
 
-    if 'IDDS_IWORKFLOW_LOAD_WORK' in os.environ:
+    if 'IDDS_IGNORE_WORK_DECORATOR' in os.environ:
         return func
 
     @functools.wraps(func)

workflow/lib/idds/iworkflow/workflow.py

@@ -7,6 +7,7 @@
 #
 # Authors:
 # - Wen Guan, <[email protected]>, 2023 - 2024
+# - Lino Oscar Gerlach, <[email protected]>, 2024
 
 import base64
 import collections
@@ -23,7 +24,7 @@
 
 # from idds.common import exceptions
 from idds.common.constants import WorkflowType
-from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64
+from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64, modified_environ
 from .asyncresult import AsyncResult
 from .base import Base, Context
 
@@ -638,9 +639,7 @@ def get_idds_env(self):
     def get_idds_server(self):
         if 'IDDS_HOST' in self._idds_env:
             return self._idds_env['IDDS_HOST']
-        if os.environ.get('IDDS_HOST', None):
-            return os.environ.get('IDDS_HOST', None)
-        return None
+        return os.environ.get('IDDS_HOST', None)
 
     def prepare_with_idds(self):
         """
@@ -1059,9 +1058,8 @@ def load(self, func_name):
 
         :raise Exception
         """
-        os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] = 'true'
-        func = super(Workflow, self).load(func_name)
-        del os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW']
+        with modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'):
+            func = super(Workflow, self).load(func_name)
 
         return func
 
@@ -1161,7 +1159,7 @@ def workflow(func=None, *, local=False, service='idds', source_dir=None, primary
         return functools.partial(workflow, local=local, service=service, source_dir=source_dir, primary=primary, queue=queue, site=site, cloud=cloud,
                                  max_walltime=max_walltime, distributed=distributed, init_env=init_env)
 
-    if 'IDDS_IWORKFLOW_LOAD_WORKFLOW' in os.environ:
+    if 'IDDS_IGNORE_WORKFLOW_DECORATOR' in os.environ:
         return func
 
     @functools.wraps(func)

workflow/tools/env/environment.yml

@@ -5,5 +5,5 @@ dependencies:
 - pip:
   - anytree
   - networkx
-  - stomp.py==8.0.1
+  - stomp.py<=8.0.1
   - idds-common==2.0.9

workflow/tools/make/environment.yaml

@@ -11,7 +11,7 @@ dependencies:
   - packaging
   - anytree
   - networkx
-  - stomp.py==8.0.1
+  - stomp.py<=8.0.1
   - panda-client
   - cffi
   - charset_normalizer