Add 8 bit test for NDStorage; Fix other Java backend acqs; improve installer

java/pom.xml

@@ -65,7 +65,7 @@
       <dependency>
          <groupId>org.micro-manager.ndtiffstorage</groupId>
          <artifactId>NDTiffStorage</artifactId>
-     <version>2.18.0</version>
+     <version>2.18.2</version>
       </dependency>
   </dependencies>
 

pycromanager/_version.py

@@ -1,2 +1,2 @@
-version_info = (0, 34, 4)
+version_info = (0, 34, 5)
 __version__ = ".".join(map(str, version_info))

pycromanager/acquisition/java_backend_acquisitions.py

@@ -202,7 +202,8 @@ def _notification_handler_fn(acquisition, notification_push_port, connected_even
             if AcqNotification.is_image_saved_notification(notification): # it was saved to RAM, not disk
                 if not notification.is_data_sink_finished_notification():
                     # check if NDTiff data storage used
-                    if acquisition._directory is not None:
+                    if acquisition._directory is not None or isinstance(acquisition, MagellanAcquisition) or \
+                            isinstance(acquisition, XYTiledAcquisition):
                         index_entry = notification.payload.encode('ISO-8859-1')
                         axes = acquisition._dataset.add_index_entry(index_entry)
                         # swap the notification.payload from the byte array of index information to axes
@@ -300,12 +301,7 @@ def __init__(
             warnings.warn('Could not create acquisition notification handler. '
                           'Update Micro-Manager and Pyrcro-Manager to the latest versions to fix this')
 
-        # Start remote acquisition
-        # Acquistition.start is now deprecated, so this can be removed later
-        # Acquisitions now get started automatically when the first events submitted
-        # but Magellan acquisitons (and probably others that generate their own events)
-        # will need some new method to submit events only after image processors etc have been added
-        self._acq.start()
+
         self._dataset_disk_location = (
             self._acq.get_data_sink().get_storage().get_disk_location()
             if self._acq.get_data_sink() is not None
@@ -321,7 +317,7 @@ def __init__(
         # when images are written to disk/RAM storage
         storage_java_class = data_sink.get_storage()
         summary_metadata = storage_java_class.get_summary_metadata()
-        if directory is not None:
+        if directory is not None or isinstance(self, MagellanAcquisition) or isinstance(self, XYTiledAcquisition):
             # NDTiff dataset saved to disk on Java side
             self._dataset = Dataset(dataset_path=self._dataset_disk_location, summary_metadata=summary_metadata)
         else:
@@ -364,10 +360,6 @@ def await_completion(self):
             if hasattr(self, '_event_thread'):
                 self._event_thread.join()
 
-            # need to do this so its _Bridge can be garbage collected and a reference to the JavaBackendAcquisition
-            # does not prevent Bridge cleanup and process exiting
-            self._remote_acq = None
-
             # Wait on all the other threads to shut down properly
             if hasattr(self, '_storage_monitor_thread'):
                 self._storage_monitor_thread.join()
@@ -633,6 +625,7 @@ def __init__(
         l = locals()
         named_args = {arg_name: l[arg_name] for arg_name in arg_names}
         super().__init__(**named_args)
+        self._acq.start()
 
     def _create_remote_acquisition(self, port, **kwargs):
         core = ZMQRemoteMMCoreJ(port=self._port, timeout=self._timeout)
@@ -648,7 +641,7 @@ def _create_remote_acquisition(self, port, **kwargs):
             x_overlap = self.tile_overlap
             y_overlap = self.tile_overlap
 
-        self._remote_acq = acq_factory.create_tiled_acquisition(
+        self._acq = acq_factory.create_tiled_acquisition(
             kwargs['directory'],
             kwargs['name'],
             show_viewer,
@@ -710,6 +703,7 @@ def __init__(
         l = locals()
         named_args = {arg_name: l[arg_name] for arg_name in arg_names}
         super().__init__(**named_args)
+        self._acq.start()
 
     def _create_remote_acquisition(self, port, **kwargs):
         if type(self.tile_overlap) is tuple:
@@ -720,7 +714,7 @@ def _create_remote_acquisition(self, port, **kwargs):
 
         ui_class = JavaClass('org.micromanager.explore.ExploreAcqUIAndStorage')
         ui = ui_class.create(kwargs['directory'], kwargs['name'], x_overlap, y_overlap, self.z_step_um, self.channel_group)
-        self._remote_acq = ui.get_acquisition()
+        self._acq = ui.get_acquisition()
 
     def _start_events(self, **kwargs):
         pass # These come from the user
@@ -767,6 +761,7 @@ def __init__(
         l = locals()
         named_args = {arg_name: l[arg_name] for arg_name in arg_names}
         super().__init__(**named_args)
+        self._acq.start()
 
     def _start_events(self, **kwargs):
         pass # Magellan handles this on Java side
@@ -777,7 +772,7 @@ def _create_event_queue(self, **kwargs):
     def _create_remote_acquisition(self, **kwargs):
         magellan_api = Magellan()
         if self.magellan_acq_index is not None:
-            self._remote_acq = magellan_api.create_acquisition(self.magellan_acq_index, False)
+            self._acq = magellan_api.create_acquisition(self.magellan_acq_index, False)
         elif self.magellan_explore:
-            self._remote_acq = magellan_api.create_explore_acquisition(False)
+            self._acq = magellan_api.create_explore_acquisition(False)
         self._event_queue = None

pycromanager/install.py

@@ -14,6 +14,26 @@
 MM_DOWNLOAD_URL_MAC = MM_DOWNLOAD_URL_BASE + '/nightly/2.0/Mac'
 MM_DOWNLOAD_URL_WINDOWS = MM_DOWNLOAD_URL_BASE + '/nightly/2.0/Windows'
 
+def _get_download_url(ci_build=False):
+    """
+    Get the download URL for the latest nightly build of Micro-Manager
+
+    Returns
+    -------
+    str
+        The URL to the latest nightly build
+    """
+    platform = _get_platform()
+    if platform == 'Windows':
+        url = MM_DOWNLOAD_URL_WINDOWS
+    elif platform == 'Mac':
+        url = MM_DOWNLOAD_URL_MAC
+    else:
+        raise ValueError(f"Unsupported OS: {platform}")
+    if ci_build:
+        url = url.replace('nightly', 'ci')
+    return url
+
 def _get_platform():
     """
     Get the platform of the system
@@ -30,18 +50,12 @@ def _get_platform():
     else:
         raise ValueError(f"Unsupported OS: {sys.platform}")
 
-def _find_versions():
+def _find_versions(ci_build=False):
     """
-    Find all available versions of Micro-Manager nightly builds
+    Find all available versions of Micro-Manager  builds
     """
-    platform = _get_platform()
     # Get the webpage
-    if platform == 'Windows':
-        webpage = requests.get(MM_DOWNLOAD_URL_WINDOWS)
-    elif platform == 'Mac':
-        webpage = requests.get(MM_DOWNLOAD_URL_MAC)
-    else:
-        raise ValueError(f"Unsupported OS: {platform}")
+    webpage = requests.get(_get_download_url(ci_build))
     return re.findall(r'class="rowDefault" href="([^"]+)', webpage.text)
 
 def find_existing_mm_install():
@@ -63,14 +77,18 @@ def find_existing_mm_install():
     else:
         raise ValueError(f"Unsupported OS: {platform}")
 
-def download_and_install(destination='auto', mm_install_log_path=None):
+def download_and_install(destination='auto', mm_install_log_path=None, ci_build=False):
     """
     Download and install the latest nightly build of Micro-Manager
 
     Parameters
     ----------
     destination : str
         The directory to install Micro-Manager to. If 'auto', it will install to the user's home directory.
+    mm_install_log_path : str
+        Path to save the installation log to
+    ci_build : bool
+        If True, download the latest CI build instead of nightly build
 
     Returns
     -------
@@ -80,14 +98,15 @@ def download_and_install(destination='auto', mm_install_log_path=None):
     windows = _get_platform() == 'Windows'
     platform = 'Windows' if windows else 'Mac'
     installer = 'mm_installer.exe' if windows else 'mm_installer.dmg'
-    latest_version = MM_DOWNLOAD_URL_BASE + _find_versions()[0]
+    latest_version = _get_download_url(ci_build) + '/' + _find_versions(ci_build)[0].split('/')[-1]
     # make a progress bar that updates every 0.5 seconds
     def bar(curr, total, width):
         if not hasattr(bar, 'last_update'):
             bar.last_update = 0
         if curr / total*100 - bar.last_update > 0.5:
             print(f"\rDownloading installer: {curr / total*100:.2f}%", end='')
             bar.last_update = curr / total*100
+    print('Downloading: ', latest_version)
     wget.download(latest_version, out=installer, bar=bar)
 
     if windows:

pycromanager/test/test_acquisition.py

@@ -469,21 +469,25 @@ def test_abort_with_no_events(launch_mm_headless, setup_data_folder):
     with Acquisition(setup_data_folder, 'test_abort_with_no_events', show_display=False) as acq:
         acq.abort()
     assert not mmc.is_sequence_running()
-
-def test_abort_from_external(launch_mm_headless, setup_data_folder):
-    """
-    Simulates the acquisition being shutdown from a remote source (e.g. Xing out the viewer)
-    """
-    with pytest.raises(AcqAlreadyCompleteException):
-        with Acquisition(setup_data_folder, 'test_abort_from_external', show_display=False) as acq:
-            events = multi_d_acquisition_events(num_time_points=6)
-            acq.acquire(events[0])
-            # this simulates an abort from the java side unbeknownst to python side
-            # it comes from a new thread so it is non-blocking to the port
-            acq._acq.abort()
-            for event in events[1:]:
-                acq.acquire(event)
-                time.sleep(5)
+    acq.get_dataset().close()
+
+# def test_abort_from_external(launch_mm_headless, setup_data_folder):
+#     """
+#     Simulates the acquisition being shutdown from a remote source (e.g. Xing out the viewer)
+#     """
+#     with pytest.raises(AcqAlreadyCompleteException):
+#         try:
+#             with Acquisition(setup_data_folder, 'test_abort_from_external', show_display=False) as acq:
+#                 events = multi_d_acquisition_events(num_time_points=6)
+#                 acq.acquire(events[0])
+#                 # this simulates an abort from the java side unbeknownst to python side
+#                 # it comes from a new thread so it is non-blocking to the port
+#                 acq._acq.abort()
+#                 for event in events[1:]:
+#                     acq.acquire(event)
+#                     time.sleep(5)
+#         finally:
+#             acq.get_dataset().close()
 
 def test_abort_sequenced_zstack(launch_mm_headless, setup_data_folder):
     """
@@ -666,5 +670,25 @@ def test_empty_axes(launch_mm_headless, setup_data_folder):
     dataset = acq.get_dataset()
     try:
         assert dataset.read_image() is not None and dataset.read_image().max() > 0
+    finally:
+        dataset.close()
+
+
+def test_8bit(launch_mm_headless, setup_data_folder):
+    """
+    Test that images with empty axes are correctly saved
+    """
+    events = multi_d_acquisition_events(10)
+    core = Core()
+    core.set_property('Camera', 'BitDepth', '8')
+    core.set_property('Camera', 'PixelType', '8bit')
+
+    with Acquisition(setup_data_folder, 'test_8_bit', show_display=False) as acq:
+        acq.acquire(events)
+
+    dataset = acq.get_dataset()
+    try:
+        image_coordinates = events[0]['axes']
+        assert dataset.read_image(**image_coordinates) is not None and dataset.read_image(**image_coordinates).max() > 0
     finally:
         dataset.close()

requirements.txt

@@ -1,8 +1,9 @@
 numpy
 dask[array]>=2022.2.0
 pyzmq
-ndstorage>=0.1.5
+ndstorage>=0.1.6
 docstring-inheritance
 pymmcore
 sortedcontainers
 pyjavaz>=1.2.1
+wget