Skip to content

Commit

Permalink
Make sure we set file size also for purged outputs
Browse files Browse the repository at this point in the history 
Fixes one of the integration tests:
```
Exception: HistoryDatasetAssociation <galaxy.model.HistoryDatasetAssociation(None) at 0x7f50ddcf56a0> in state ok with null file size, this is not valid
ERROR    galaxy.jobs.runners:__init__.py:177 (1) Unhandled exception calling finish_job
Traceback (most recent call last):
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/pulsar.py", line 702, in finish_job
    job_wrapper.finish(
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/__init__.py", line 2054, in finish
    self.sa_session.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/scoping.py", line 597, in commit
    return self._proxied.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2017, in commit
    trans.commit(_to_root=True)
  File "<string>", line 2, in commit
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1302, in commit
    self._prepare_impl()
  File "<string>", line 2, in _prepare_impl
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1277, in _prepare_impl
    self.session.flush()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4341, in flush
    self._flush(objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4369, in _flush
    self.dispatch.before_flush(self, flush_context, objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/event/attr.py", line 378, in __call__
    fn(*args, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/base.py", line 184, in before_flush
    obj.__strict_check_before_flush__()
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/__init__.py", line 5223, in __strict_check_before_flush__
    raise Exception(
Exception: HistoryDatasetAssociation <galaxy.model.HistoryDatasetAssociation(None) at 0x7f50ddcf56a0> in state ok with null file size, this is not valid

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/__init__.py", line 174, in run_next
    method(arg)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/pulsar.py", line 713, in finish_job
    job_wrapper.fail("Unable to finish job", exception=True)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/__init__.py", line 1468, in fail
    self.sa_session.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/scoping.py", line 597, in commit
    return self._proxied.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2017, in commit
    trans.commit(_to_root=True)
  File "<string>", line 2, in commit
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1302, in commit
    self._prepare_impl()
  File "<string>", line 2, in _prepare_impl
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1277, in _prepare_impl
    self.session.flush()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4341, in flush
    self._flush(objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4369, in _flush
    self.dispatch.before_flush(self, flush_context, objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/event/attr.py", line 378, in __call__
    fn(*args, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/base.py", line 184, in before_flush
    obj.__strict_check_before_flush__()
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/__init__.py", line 5223, in __strict_check_before_flush__
    raise Exception(
Exception: HistoryDatasetAssociation <galaxy.model.HistoryDatasetAssociation(None) at 0x7f50ddcf56a0> in state ok with null file size, this is not valid
```

More fallout from galaxyproject#18653
  • Loading branch information
@mvdbeek
mvdbeek committed Aug 12, 2024
1 parent 591f36c commit c71f944
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 8 deletions.
10 changes: 5 additions & 5 deletions lib/galaxy/jobs/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2001,13 +2001,13 @@ def fail(message=job.info, exception=None):
# Once datasets are collected, set the total dataset size (includes extra files)
for dataset_assoc in job.output_datasets:
dataset = dataset_assoc.dataset.dataset
if not dataset.purged:
# assume all datasets in a job get written to the same objectstore
quota_source_info = dataset.quota_source_info
collected_bytes += dataset.set_total_size()
else:
# assume all datasets in a job get written to the same objectstore
quota_source_info = dataset.quota_source_info
collected_bytes += dataset.set_total_size()
if dataset.purged:
# Purge, in case job wrote directly to object store
dataset.full_delete()
collected_bytes = 0

user = job.user
if user and collected_bytes > 0 and quota_source_info is not None and quota_source_info.use:
Expand Down
6 changes: 3 additions & 3 deletions lib/galaxy/model/store/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -665,9 +665,9 @@ def handle_dataset_object_edit(dataset_instance, dataset_attrs):
assert file_source_root
dataset_extra_files_path = os.path.join(file_source_root, dataset_extra_files_path)
persist_extra_files(self.object_store, dataset_extra_files_path, dataset_instance)
# Don't trust serialized file size
dataset_instance.dataset.file_size = None
dataset_instance.dataset.set_total_size() # update the filesize record in the database
# Only trust file size if the dataset is purged. If we keep the data we should check the file size.
dataset_instance.dataset.file_size = None
dataset_instance.dataset.set_total_size() # update the filesize record in the database

if dataset_instance.deleted:
dataset_instance.dataset.deleted = True
Expand Down

0 comments on commit c71f944

Please sign in to comment.