diff --git a/common/lib/idds/common/utils.py b/common/lib/idds/common/utils.py index 1a73cde2..87bcdab0 100644 --- a/common/lib/idds/common/utils.py +++ b/common/lib/idds/common/utils.py @@ -7,9 +7,10 @@ # # Authors: # - Wen Guan, , 2019 - 2024 - +# - Lino Oscar Gerlach, , 2024 import base64 +import contextlib import errno import datetime import importlib @@ -942,3 +943,32 @@ def idds_mask(dict_): else: ret[k] = dict_[k] return ret + + +@contextlib.contextmanager +def modified_environ(*remove, **update): + """ + Temporarily updates the ``os.environ`` dictionary in-place. + The ``os.environ`` dictionary is updated in-place so that the modification + is sure to work in all situations. + :param remove: Environment variables to remove. + :param update: Dictionary of environment variables and values to add/update. + """ + env = os.environ + update = update or {} + remove = remove or [] + + # List of environment variables being updated or removed. + stomped = (set(update.keys()) | set(remove)) & set(env.keys()) + # Environment variables and values to restore on exit. + update_after = {k: env[k] for k in stomped} + # Environment variables and values to remove on exit. + remove_after = frozenset(k for k in update if k not in env) + + try: + env.update(update) + [env.pop(k, None) for k in remove] + yield + finally: + env.update(update_after) + [env.pop(k) for k in remove_after] diff --git a/main/etc/sql/postgres_partition.sql b/main/etc/sql/postgres_partition.sql deleted file mode 100644 index 42eb4843..00000000 --- a/main/etc/sql/postgres_partition.sql +++ /dev/null @@ -1,15 +0,0 @@ -SELECT partman.create_parent( -p_parent_table => 'doma_idds.contents', -p_control => 'request_id', -p_type => 'native', -p_interval=> '1000', -p_premake => 3 -); - -SELECT partman.create_parent( -p_parent_table => 'doma_idds.contents_ext', -p_control => 'request_id', -p_type => 'native', -p_interval=> '1000', -p_premake => 3 -); \ No newline at end of file diff --git a/main/etc/sql/postgresql.sql b/main/etc/sql/postgresql.sql index c8d1f1d1..792a94cf 100644 --- a/main/etc/sql/postgresql.sql +++ b/main/etc/sql/postgresql.sql @@ -181,8 +181,8 @@ CREATE TABLE doma_idds.contents_ext ( memory_leak VARCHAR(10), memory_leak_x2 VARCHAR(10), job_label VARCHAR(20), - CONSTRAINT "CONTENTS_EXT_PK" PRIMARY KEY (content_id, request_id) -) PARTITION BY RANGE (request_id) ; + CONSTRAINT "CONTENTS_EXT_PK" PRIMARY KEY (content_id) +); CREATE INDEX "CONTENTS_EXT_RTW_IDX" ON doma_idds.contents_ext (request_id, transform_id, workload_id); @@ -500,13 +500,13 @@ CREATE TABLE doma_idds.contents ( accessed_at TIMESTAMP WITHOUT TIME ZONE, expired_at TIMESTAMP WITHOUT TIME ZONE, content_metadata VARCHAR(1000), - CONSTRAINT "CONTENTS_PK_TEST" PRIMARY KEY (content_id, request_id), - CONSTRAINT "CONTENT_ID_UQ_TEST" UNIQUE (transform_id, coll_id, request_id, map_id, sub_map_id, dep_sub_map_id, content_relation_type, name_md5, scope_name_md5, min_id, max_id), - CONSTRAINT "CONTENTS_TRANSFORM_ID_FK" FOREIGN KEY(transform_id) REFERENCES doma_idds.transforms (transform_id), + CONSTRAINT "CONTENTS_PK" PRIMARY KEY (content_id), + CONSTRAINT "CONTENT_ID_UQ" UNIQUE (transform_id, coll_id, map_id, sub_map_id, dep_sub_map_id, content_relation_type, name_md5, scope_name_md5, min_id, max_id), + CONSTRAINT "CONTENTS_TRANSFORM_ID_FK" FOREIGN KEY(transform_id) REFERENCES doma_idds.transforms (transform_id), CONSTRAINT "CONTENTS_COLL_ID_FK" FOREIGN KEY(coll_id) REFERENCES doma_idds.collections (coll_id), CONSTRAINT "CONTENTS_STATUS_ID_NN" CHECK (status IS NOT NULL), CONSTRAINT "CONTENTS_COLL_ID_NN" CHECK (coll_id IS NOT NULL) -) PARTITION BY RANGE (request_id) ; +); CREATE INDEX "CONTENTS_STATUS_UPDATED_IDX" ON doma_idds.contents (status, locking, updated_at, created_at); @@ -518,7 +518,8 @@ CREATE INDEX "CONTENTS_REQ_TF_COLL_IDX" ON doma_idds.contents (request_id, trans CREATE INDEX "CONTENTS_TF_IDX" ON doma_idds.contents (transform_id, request_id, coll_id, map_id, content_relation_type); -CREATE INDEX "CONTENTS_ID_NAME_IDX" ON doma_idds.contents (coll_id, scope, md5('name'), status); +CREATE INDEX "CONTENTS_ID_NAME_IDX" ON doma_idds.contents (coll_id, scope, md5('name'); +, status); SET search_path TO doma_idds; diff --git a/main/tools/env/setup_panda.sh b/main/tools/env/setup_panda.sh index a379dbea..7d7bc097 100644 --- a/main/tools/env/setup_panda.sh +++ b/main/tools/env/setup_panda.sh @@ -56,6 +56,37 @@ elif [ "$instance" == "usdf" ]; then # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ export PANDA_CONFIG_ROOT=~/.panda/ +elif [ "$instance" == "new" ]; then + export PANDA_AUTH=oidc + export PANDA_URL_SSL=https://ai-idds-05.cern.ch:25443/server/panda + export PANDA_URL=http://ai-idds-05.cern.ch:25080/server/panda + export PANDAMON_URL=https://panda-doma.cern.ch + # export PANDA_AUTH_VO=panda_dev + export PANDA_AUTH_VO=Rubin:production + + export PANDACACHE_URL=$PANDA_URL_SSL + + export PANDA_SYS=/afs/cern.ch/user/w/wguan/workdisk/iDDS/.conda/iDDS/ + # export PANDA_CONFIG_ROOT=/afs/cern.ch/user/w/wguan/workdisk/iDDS/main/etc/panda/ + export PANDA_CONFIG_ROOT=~/.panda/ + + # export IDDS_HOST=https://aipanda015.cern.ch:443/idds + + # dev + # export IDDS_HOST=https://aipanda104.cern.ch:443/idds + + # doma + export IDDS_HOST=https://aipanda105.cern.ch:443/idds + + # export IDDS_BROKERS=atlas-test-mb.cern.ch:61013 + # export IDDS_BROKER_DESTINATION=/topic/doma.idds + # export IDDS_BROKER_USERNAME=domaidds + # export IDDS_BROKER_PASSWORD=1d25yeft6krJ1HFH + # export IDDS_BROKER_TIMEOUT=360 + + PANDA_QUEUE=BNL_OSG_2 + PANDA_WORKING_GROUP=EIC + PANDA_VO=wlcg else export PANDA_AUTH=oidc export PANDA_URL_SSL=https://pandaserver-doma.cern.ch:25443/server/panda diff --git a/monitor/data/conf.js b/monitor/data/conf.js index dd4ac352..c5e4e602 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus926.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus926.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus926.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus926.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus935.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus935.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus935.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus935.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/website/data/index.html b/website/data/index.html index 01434d84..5535ef1f 100644 --- a/website/data/index.html +++ b/website/data/index.html @@ -61,7 +61,7 @@

Intelligent

Fine grained data transformation

-

Remote data transformation/reduction

+

Remtoe data transformation/reduction

On-demand production of analysis format data transformation

Transformation

diff --git a/workflow/lib/idds/iworkflow/work.py b/workflow/lib/idds/iworkflow/work.py index f15e9cb3..55e47e91 100644 --- a/workflow/lib/idds/iworkflow/work.py +++ b/workflow/lib/idds/iworkflow/work.py @@ -7,6 +7,7 @@ # # Authors: # - Wen Guan, , 2023 - 2024 +# - Lino Oscar Gerlach, , 2024 import base64 import datetime @@ -21,7 +22,7 @@ from idds.common import exceptions from idds.common.constants import WorkflowType, TransformStatus from idds.common.imports import get_func_name -from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64 +from idds.common.utils import setup_logging, json_dumps, json_loads, encode_base64, modified_environ from .asyncresult import AsyncResult, MapResult from .base import Base, Context from .workflow import WorkflowCanvas @@ -779,9 +780,8 @@ def load(self, func_name): :raise Exception """ - os.environ['IDDS_IWORKFLOW_LOAD_WORK'] = 'true' - func = super(Work, self).load(func_name) - del os.environ['IDDS_IWORKFLOW_LOAD_WORK'] + with modified_environ(IDDS_IGNORE_WORK_DECORATOR='true'): + func = super(Work, self).load(func_name) return func @@ -922,7 +922,7 @@ def work(func=None, *, map_results=False, lazy=False, init_env=None): if func is None: return functools.partial(work, map_results=map_results, lazy=lazy, init_env=init_env) - if 'IDDS_IWORKFLOW_LOAD_WORK' in os.environ: + if 'IDDS_IGNORE_WORK_DECORATOR' in os.environ: return func @functools.wraps(func) diff --git a/workflow/lib/idds/iworkflow/workflow.py b/workflow/lib/idds/iworkflow/workflow.py index cf4eb719..9d718ae5 100644 --- a/workflow/lib/idds/iworkflow/workflow.py +++ b/workflow/lib/idds/iworkflow/workflow.py @@ -7,6 +7,7 @@ # # Authors: # - Wen Guan, , 2023 - 2024 +# - Lino Oscar Gerlach, , 2024 import base64 import collections @@ -23,7 +24,7 @@ # from idds.common import exceptions from idds.common.constants import WorkflowType -from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64 +from idds.common.utils import setup_logging, create_archive_file, json_dumps, json_loads, encode_base64, modified_environ from .asyncresult import AsyncResult from .base import Base, Context @@ -638,9 +639,7 @@ def get_idds_env(self): def get_idds_server(self): if 'IDDS_HOST' in self._idds_env: return self._idds_env['IDDS_HOST'] - if os.environ.get('IDDS_HOST', None): - return os.environ.get('IDDS_HOST', None) - return None + return os.environ.get('IDDS_HOST', None) def prepare_with_idds(self): """ @@ -1059,9 +1058,8 @@ def load(self, func_name): :raise Exception """ - os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] = 'true' - func = super(Workflow, self).load(func_name) - del os.environ['IDDS_IWORKFLOW_LOAD_WORKFLOW'] + with modified_environ(IDDS_IGNORE_WORKFLOW_DECORATOR='true'): + func = super(Workflow, self).load(func_name) return func @@ -1161,7 +1159,7 @@ def workflow(func=None, *, local=False, service='idds', source_dir=None, primary return functools.partial(workflow, local=local, service=service, source_dir=source_dir, primary=primary, queue=queue, site=site, cloud=cloud, max_walltime=max_walltime, distributed=distributed, init_env=init_env) - if 'IDDS_IWORKFLOW_LOAD_WORKFLOW' in os.environ: + if 'IDDS_IGNORE_WORKFLOW_DECORATOR' in os.environ: return func @functools.wraps(func) diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index b59b2e54..c2ba6e1a 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -5,5 +5,5 @@ dependencies: - pip: - anytree - networkx - - stomp.py==8.0.1 + - stomp.py<=8.0.1 - idds-common==2.0.9 diff --git a/workflow/tools/make/environment.yaml b/workflow/tools/make/environment.yaml index f1c8273a..4c78df39 100644 --- a/workflow/tools/make/environment.yaml +++ b/workflow/tools/make/environment.yaml @@ -11,7 +11,7 @@ dependencies: - packaging - anytree - networkx - - stomp.py==8.0.1 + - stomp.py<=8.0.1 - panda-client - cffi - charset_normalizer