diff --git a/docs/src/.overrides/assets/stylesheets/extra.css b/docs/src/.overrides/assets/stylesheets/extra.css index 17eb6b5..70875fc 100644 --- a/docs/src/.overrides/assets/stylesheets/extra.css +++ b/docs/src/.overrides/assets/stylesheets/extra.css @@ -92,7 +92,6 @@ html a[title="YouTube"].md-social__link svg { /* --md-footer-fg-color: var(--dj-white); */ } -[data-md-color-scheme="slate"] td, -th { +[data-md-color-scheme="slate"] .jupyter-wrapper .Table Td { color: var(--dj-black) } diff --git a/element_interface/dandi.py b/element_interface/dandi.py index 22486b3..ef9fd13 100644 --- a/element_interface/dandi.py +++ b/element_interface/dandi.py @@ -59,11 +59,6 @@ def upload_to_dandi( shell=shell, ) - subprocess.run( - ["dandi", "organize", "-d", dandiset_directory, data_directory, "-f", "dry"], - shell=shell, # without this param, subprocess interprets first arg as file/dir - ) - subprocess.run( [ "dandi", diff --git a/element_interface/prairie_view_loader.py b/element_interface/prairie_view_loader.py index 6dcda90..f43f7ca 100644 --- a/element_interface/prairie_view_loader.py +++ b/element_interface/prairie_view_loader.py @@ -90,7 +90,7 @@ def get_prairieview_filenames( f".//Sequence/Frame{plane_search}/File{channel_search}" ) - fnames = [f.attrib["filename"] for f in frames] + fnames = np.unique([f.attrib["filename"] for f in frames]).tolist() return fnames if not return_pln_chn else (fnames, plane_idx, channel) def write_single_bigtiff( @@ -114,46 +114,90 @@ def write_single_bigtiff( if output_tiff_fullpath.exists() and not overwrite: return output_tiff_fullpath - if not caiman_compatible: - with tifffile.TiffWriter( - output_tiff_fullpath, - bigtiff=True, - ) as tiff_writer: - try: - for input_file in tiff_names: - with tifffile.TiffFile( - self.prairieview_dir / input_file - ) as tffl: - assert len(tffl.pages) == 1 - tiff_writer.write( - tffl.pages[0].asarray(), - metadata={ - "axes": "YX", - "'fps'": self.meta["frame_rate"], - }, - ) - except Exception as e: - raise Exception(f"Error in processing tiff file {input_file}: {e}") - else: - combined_data = [] + if self.meta["is_multipage"]: + # For multi-page tiff - the pages are organized as: + # (channel x slice x frame) - each page is (height x width) + # - TODO: verify this is the case for Bruker multi-page tiff + # This implementation is partially based on the reference code from `scanreader` package - https://github.com/atlab/scanreader + # See: https://github.com/atlab/scanreader/blob/2a021a85fca011c17e553d0e1c776998d3f2b2d8/scanreader/scans.py#L337 + slice_step = self.meta["num_channels"] + frame_step = self.meta["num_channels"] * self.meta["num_planes"] + slice_idx = self.meta["plane_indices"].index(plane_idx) + channel_idx = self.meta["channels"].index(channel) + + page_indices = [frame_idx * frame_step + slice_idx * slice_step + channel_idx + for frame_idx in range(self.meta["num_frames"])] + + combined_data = np.empty([self.meta["num_frames"], + self.meta["height_in_pixels"], + self.meta["width_in_pixels"]], + dtype=int) + start_page = 0 try: for input_file in tiff_names: with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl: - assert len(tffl.pages) == 1 - combined_data.append(tffl.pages[0].asarray()) + # Get indices in this tiff file and in output array + final_page_in_file = start_page + len(tffl.pages) + is_page_in_file = lambda page: page in range(start_page, final_page_in_file) + pages_in_file = filter(is_page_in_file, page_indices) + file_indices = [page - start_page for page in pages_in_file] + global_indices = [is_page_in_file(page) for page in page_indices] + + # Read from this tiff file (if needed) + if len(file_indices) > 0: + # this line looks a bit ugly but is memory efficient. Do not separate + combined_data[global_indices] = tffl.asarray(key=file_indices) + start_page += len(tffl.pages) except Exception as e: raise Exception(f"Error in processing tiff file {input_file}: {e}") - combined_data = np.dstack(combined_data).transpose( - 2, 0, 1 - ) # (frame x height x width) - tifffile.imwrite( output_tiff_fullpath, combined_data, metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]}, bigtiff=True, ) + else: + if not caiman_compatible: + with tifffile.TiffWriter( + output_tiff_fullpath, + bigtiff=True, + ) as tiff_writer: + try: + for input_file in tiff_names: + with tifffile.TiffFile( + self.prairieview_dir / input_file + ) as tffl: + assert len(tffl.pages) == 1 + tiff_writer.write( + tffl.pages[0].asarray(), + metadata={ + "axes": "YX", + "'fps'": self.meta["frame_rate"], + }, + ) + except Exception as e: + raise Exception(f"Error in processing tiff file {input_file}: {e}") + else: + combined_data = [] + try: + for input_file in tiff_names: + with tifffile.TiffFile(self.prairieview_dir / input_file) as tffl: + assert len(tffl.pages) == 1 + combined_data.append(tffl.pages[0].asarray()) + except Exception as e: + raise Exception(f"Error in processing tiff file {input_file}: {e}") + + combined_data = np.dstack(combined_data).transpose( + 2, 0, 1 + ) # (frame x height x width) + + tifffile.imwrite( + output_tiff_fullpath, + combined_data, + metadata={"axes": "TYX", "'fps'": self.meta["frame_rate"]}, + bigtiff=True, + ) return output_tiff_fullpath @@ -167,6 +211,7 @@ def _extract_prairieview_metadata(xml_filepath: str): bidirectional_scan = False # Does not support bidirectional roi = 0 + is_multipage = xml_root.find(".//Sequence/Frame/File/[@page]") is not None recording_start_time = xml_root.find(".//Sequence/[@cycle='1']").attrib.get("time") # Get all channels and find unique values @@ -310,6 +355,7 @@ def _extract_prairieview_metadata(xml_filepath: str): frame_period=frame_period, bidirectional=bidirectional_scan, bidirectional_z=bidirection_z, + is_multipage=is_multipage, scan_datetime=scan_datetime, usecs_per_line=usec_per_line, scan_duration=total_scan_duration,