debugging refactored upload

xnat_ingest/cli/upload.py

@@ -12,8 +12,10 @@
 from frametree.core.frameset import FrameSet
 from frametree.xnat import Xnat
 from xnat.exceptions import XNATResponseError
+from fileformats.application import Json
 from xnat_ingest.cli.base import cli
 from xnat_ingest.session import ImagingSession
+from xnat_ingest.resource import ImagingResource
 from xnat_ingest.utils import (
     logger,
     LogFile,
@@ -360,20 +362,39 @@ def do_upload() -> None:
                             for fspath in resource.fileset.fspaths:
                                 xresource.upload(str(fspath), fspath.name)
                         else:
+                            # Temporarily move the manifest file out of the way so it
+                            # doesn't get uploaded
+                            manifest_file = (
+                                resource.fileset.parent / ImagingResource.MANIFEST_FNAME
+                            )
+                            moved_manifest_file = (
+                                resource.fileset.parent.parent
+                                / ImagingResource.MANIFEST_FNAME
+                            )
+                            if manifest_file.exists():
+                                manifest_file.rename(moved_manifest_file)
+                            # Upload the contents of the resource to XNAT
                             xresource.upload_dir(resource.fileset.parent, method=method)
+                            # Move the manifest file back again
+                            if moved_manifest_file.exists():
+                                moved_manifest_file.rename(manifest_file)
                         logger.debug("retrieving checksums for %s", xresource)
                         remote_checksums = get_xnat_checksums(xresource)
                         logger.debug("calculating checksums for %s", xresource)
                         calc_checksums = calculate_checksums(resource.fileset)
                         if remote_checksums != calc_checksums:
+                            extra_keys = set(remote_checksums) - set(calc_checksums)
+                            missing_keys = set(calc_checksums) - set(remote_checksums)
                             mismatching = [
                                 k
-                                for k, v in remote_checksums.items()
-                                if v != calc_checksums[k]
+                                for k, v in calc_checksums.items()
+                                if v != remote_checksums[k]
                             ]
                             raise RuntimeError(
                                 "Checksums do not match after upload of "
-                                f"'{resource.path}' resource. "
+                                f"'{resource.path}' resource.\n"
+                                f"Extra keys were {extra_keys}\n"
+                                f"Missing keys were {missing_keys}\n"
                                 f"Mismatching files were {mismatching}"
                             )
                         logger.info(f"Uploaded '{resource.path}' in '{session.name}'")

xnat_ingest/session.py

@@ -16,7 +16,7 @@
 from frametree.core.frameset import FrameSet  # type: ignore[import-untyped]
 from frametree.core.exceptions import FrameTreeDataMatchError  # type: ignore[import-untyped]
 from .exceptions import ImagingSessionParseError, StagingError
-from .utils import AssociatedFiles
+from .utils import AssociatedFiles, invalid_path_chars_re
 from .scan import ImagingScan
 from .resource import ImagingResource
 
@@ -356,7 +356,7 @@ def get_id(field_type: str, field_name: str) -> str:
                 if index is not None:
                     value = value[index]
                 value_str = str(value)
-                value_str = cls.id_escape_re.sub("_", value_str)
+                value_str = invalid_path_chars_re.sub("_", value_str)
                 return value_str
 
             if not project_id:

xnat_ingest/upload_helpers.py

@@ -284,10 +284,10 @@ def get_xnat_resource(resource: ImagingResource, xsession: ty.Any) -> ty.Any:
         return None
     logger.debug(
         "Creating resource %s in %s",
-        resource_name,
+        resource.name,
         resource.scan.path,
     )
-    xresource = xscan.create_resource(resource_name)
+    xresource = xscan.create_resource(resource.name)
     return xresource
 
 

xnat_ingest/utils.py

@@ -406,3 +406,5 @@ def glob_to_re(glob_pattern: str) -> str:
 )
 
 _str_templ_replacement = re.compile(r"\{[\w\.]+\}")
+
+invalid_path_chars_re = re.compile(r'[<>:"/\\|?*\x00-\x1F]')