WIP: untar directory to extra_files_path

galaxyproject · Nov 1, 2023 · 4368c9d · 4368c9d
1 parent 91d4dc9
commit 4368c9d
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 1 deletion.
diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
@@ -1211,6 +1211,12 @@ def _archive_main_file(
         error, msg, messagetype = False, "", ""
         return (error, msg, messagetype)
 
+    @staticmethod
+    def to_directory(path: str, extra_files_path: str, remove_source: bool = False):
+        compression_utils.CompressedFile(path).extract(extra_files_path)
+        if remove_source:
+            os.remove(path)
+
 
 class GenericAsn1(Text):
     """Class for generic ASN.1 text format"""

diff --git a/lib/galaxy/model/deferred.py b/lib/galaxy/model/deferred.py
@@ -33,6 +33,7 @@
 from galaxy.objectstore import (
     ObjectStore,
     ObjectStorePopulator,
+    persist_extra_files,
 )
 
 log = logging.getLogger(__name__)
@@ -128,7 +129,15 @@ def ensure_materialized(
                     sa_session.commit()
             object_store_populator.set_dataset_object_store_id(materialized_dataset)
             path = self._stream_source(target_source, datatype=dataset_instance.datatype)
+            if dataset_instance.ext == "directory":
+                dataset_instance.datatype.to_directory(path, materialized_dataset.extra_files_path)
+                persist_extra_files(
+                    object_store=object_store,
+                    src_extra_files_path=materialized_dataset.extra_files_path,
+                    primary_data=materialized_dataset,
+                )
             object_store.update_from_file(materialized_dataset, file_name=path)
+
         else:
             transient_path_mapper = self._transient_path_mapper
             assert transient_path_mapper

diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py
@@ -317,11 +317,11 @@ def _resolve_item_with_primary(item):
             elif not link_data_only:
                 path = upload_config.ensure_in_working_directory(path, purge_source, in_place)
 
+            extra_files_path = f"{path}_extra"
             extra_files = item.get("extra_files")
             if extra_files:
                 # TODO: optimize to just copy the whole directory to extra files instead.
                 assert not upload_config.link_data_only, "linking composite dataset files not yet implemented"
-                extra_files_path = f"{path}_extra"
                 staged_extra_files = extra_files_path
                 os.mkdir(extra_files_path)
 
@@ -361,6 +361,10 @@ def walk_extra_files(items, prefix=""):
                 assert path
                 datatype.groom_dataset_content(path)
 
+            if hasattr(datatype, "to_directory"):
+                datatype.to_directory(path, extra_files_path)
+                staged_extra_files = extra_files_path
+
             if len(transform) > 0:
                 source_dict["transform"] = transform
         elif not error_message: