Merge pull request #247 from HSF/dev

boost memory
HSF · Nov 9, 2023 · 4b0c96c · 4b0c96c
2 parents 072135a + 10ed23a
commit 4b0c96c
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 2 deletions.
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -49,3 +49,4 @@ jobs:
         python main/tools/pypi/update_version.py ${version_tag}
         python setup.py sdist bdist_wheel
         twine upload  */dist/idds*-${version_tag}.tar.gz
+        twine upload  */dist/idds*-${version_tag}*.whl
diff --git a/doma/lib/idds/doma/workflowv2/domapandawork.py b/doma/lib/idds/doma/workflowv2/domapandawork.py
@@ -553,8 +553,9 @@ def create_processing(self, input_output_maps=[]):
             task_param_map['ramCount'] = self.task_rss / self.core_count if self.core_count else self.task_rss
             # task_param_map['ramUnit'] = 'MB'
             task_param_map['ramUnit'] = 'MBPerCoreFixed'
-        if self.task_rss_retry_offset and self.task_rss_retry_step:
+        if self.task_rss_retry_offset:
             task_param_map['retryRamOffset'] = self.task_rss_retry_offset
+        if self.task_rss_retry_step:
             task_param_map['retryRamStep'] = self.task_rss_retry_step
         if self.task_rss_max:
             # todo: until PanDA supports it

diff --git a/main/lib/idds/tests/panda_client_submit_test.py b/main/lib/idds/tests/panda_client_submit_test.py
@@ -43,7 +43,7 @@
 taskParamMap['ramUnit'] = 'MBPerCoreFixed'
 
 taskParamMap['retryRamOffset'] = 2000
-taskParamMap['retryRamStep'] = 400
+taskParamMap['retryRamStep'] = 0.3
 
 """
 taskParamMap['log'] = {'dataset': logDatasetName,

diff --git a/main/tools/panda/increase_memory b/main/tools/panda/increase_memory
@@ -6,3 +6,7 @@ insert into retryerrors(retryerror_id, errorsource, errorcode, active, retryacti
 insert into retryerrors(retryerror_id, errorsource, errorcode, errordiag, active, retryaction, description) values(3, 'pilotErrorCode', 1305, '.*Unable to allocate.*', 'Y', 2, 'increase memory');
 
 
+# add rule to increase memory x times
+insert into retryactions(retryaction_id, retry_action, active, retry_description) values (1, 'increase_memory_xtimes', 'Y', 'Job ran out of memory. Increase memory setting for next retry.');
+insert into retryerrors(retryerror_id, errorsource, errorcode, errordiag, active, retryaction, description) values(1, 'taskBufferErrorCode', 300, '.*The worker was finished while the job was starting.*', 'Y', 1, 'increase memory');
+