From d4ba8eb6c2088a5f7712ef777e2945c88f71d0d4 Mon Sep 17 00:00:00 2001
From: Tom Close <tom.g.close@gmail.com>
Date: Fri, 27 Sep 2024 08:53:11 +1000
Subject: [PATCH] debugging refactored upload

---
 xnat_ingest/cli/upload.py     | 27 ++++++++++++++++++++++++---
 xnat_ingest/session.py        |  4 ++--
 xnat_ingest/upload_helpers.py |  4 ++--
 xnat_ingest/utils.py          |  2 ++
 4 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/xnat_ingest/cli/upload.py b/xnat_ingest/cli/upload.py
index 1293a4a..11f3621 100644
--- a/xnat_ingest/cli/upload.py
+++ b/xnat_ingest/cli/upload.py
@@ -12,8 +12,10 @@
 from frametree.core.frameset import FrameSet
 from frametree.xnat import Xnat
 from xnat.exceptions import XNATResponseError
+from fileformats.application import Json
 from xnat_ingest.cli.base import cli
 from xnat_ingest.session import ImagingSession
+from xnat_ingest.resource import ImagingResource
 from xnat_ingest.utils import (
     logger,
     LogFile,
@@ -360,20 +362,39 @@ def do_upload() -> None:
                             for fspath in resource.fileset.fspaths:
                                 xresource.upload(str(fspath), fspath.name)
                         else:
+                            # Temporarily move the manifest file out of the way so it
+                            # doesn't get uploaded
+                            manifest_file = (
+                                resource.fileset.parent / ImagingResource.MANIFEST_FNAME
+                            )
+                            moved_manifest_file = (
+                                resource.fileset.parent.parent
+                                / ImagingResource.MANIFEST_FNAME
+                            )
+                            if manifest_file.exists():
+                                manifest_file.rename(moved_manifest_file)
+                            # Upload the contents of the resource to XNAT
                             xresource.upload_dir(resource.fileset.parent, method=method)
+                            # Move the manifest file back again
+                            if moved_manifest_file.exists():
+                                moved_manifest_file.rename(manifest_file)
                         logger.debug("retrieving checksums for %s", xresource)
                         remote_checksums = get_xnat_checksums(xresource)
                         logger.debug("calculating checksums for %s", xresource)
                         calc_checksums = calculate_checksums(resource.fileset)
                         if remote_checksums != calc_checksums:
+                            extra_keys = set(remote_checksums) - set(calc_checksums)
+                            missing_keys = set(calc_checksums) - set(remote_checksums)
                             mismatching = [
                                 k
-                                for k, v in remote_checksums.items()
-                                if v != calc_checksums[k]
+                                for k, v in calc_checksums.items()
+                                if v != remote_checksums[k]
                             ]
                             raise RuntimeError(
                                 "Checksums do not match after upload of "
-                                f"'{resource.path}' resource. "
+                                f"'{resource.path}' resource.\n"
+                                f"Extra keys were {extra_keys}\n"
+                                f"Missing keys were {missing_keys}\n"
                                 f"Mismatching files were {mismatching}"
                             )
                         logger.info(f"Uploaded '{resource.path}' in '{session.name}'")
diff --git a/xnat_ingest/session.py b/xnat_ingest/session.py
index 6f45e5d..4479895 100644
--- a/xnat_ingest/session.py
+++ b/xnat_ingest/session.py
@@ -16,7 +16,7 @@
 from frametree.core.frameset import FrameSet  # type: ignore[import-untyped]
 from frametree.core.exceptions import FrameTreeDataMatchError  # type: ignore[import-untyped]
 from .exceptions import ImagingSessionParseError, StagingError
-from .utils import AssociatedFiles
+from .utils import AssociatedFiles, invalid_path_chars_re
 from .scan import ImagingScan
 from .resource import ImagingResource
 
@@ -356,7 +356,7 @@ def get_id(field_type: str, field_name: str) -> str:
                 if index is not None:
                     value = value[index]
                 value_str = str(value)
-                value_str = cls.id_escape_re.sub("_", value_str)
+                value_str = invalid_path_chars_re.sub("_", value_str)
                 return value_str
 
             if not project_id:
diff --git a/xnat_ingest/upload_helpers.py b/xnat_ingest/upload_helpers.py
index dac8324..73665b7 100644
--- a/xnat_ingest/upload_helpers.py
+++ b/xnat_ingest/upload_helpers.py
@@ -284,10 +284,10 @@ def get_xnat_resource(resource: ImagingResource, xsession: ty.Any) -> ty.Any:
         return None
     logger.debug(
         "Creating resource %s in %s",
-        resource_name,
+        resource.name,
         resource.scan.path,
     )
-    xresource = xscan.create_resource(resource_name)
+    xresource = xscan.create_resource(resource.name)
     return xresource
 
 
diff --git a/xnat_ingest/utils.py b/xnat_ingest/utils.py
index 4b75f44..061adf5 100644
--- a/xnat_ingest/utils.py
+++ b/xnat_ingest/utils.py
@@ -406,3 +406,5 @@ def glob_to_re(glob_pattern: str) -> str:
 )
 
 _str_templ_replacement = re.compile(r"\{[\w\.]+\}")
+
+invalid_path_chars_re = re.compile(r'[<>:"/\\|?*\x00-\x1F]')