From 995dbb5a3e0e7837411d43294cf4111454750d2f Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Sun, 23 Jun 2024 12:08:43 +0200 Subject: [PATCH 01/14] fix other java acq types + improve installer --- pycromanager/_version.py | 2 +- .../acquisition/java_backend_acquisitions.py | 27 +++++------- pycromanager/install.py | 41 ++++++++++++++----- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/pycromanager/_version.py b/pycromanager/_version.py index db276572..11019ec3 100644 --- a/pycromanager/_version.py +++ b/pycromanager/_version.py @@ -1,2 +1,2 @@ -version_info = (0, 34, 4) +version_info = (0, 34, 5) __version__ = ".".join(map(str, version_info)) diff --git a/pycromanager/acquisition/java_backend_acquisitions.py b/pycromanager/acquisition/java_backend_acquisitions.py index 7e4d4d9c..da5c0503 100644 --- a/pycromanager/acquisition/java_backend_acquisitions.py +++ b/pycromanager/acquisition/java_backend_acquisitions.py @@ -202,7 +202,8 @@ def _notification_handler_fn(acquisition, notification_push_port, connected_even if AcqNotification.is_image_saved_notification(notification): # it was saved to RAM, not disk if not notification.is_data_sink_finished_notification(): # check if NDTiff data storage used - if acquisition._directory is not None: + if acquisition._directory is not None or isinstance(acquisition, MagellanAcquisition) or \ + isinstance(acquisition, XYTiledAcquisition): index_entry = notification.payload.encode('ISO-8859-1') axes = acquisition._dataset.add_index_entry(index_entry) # swap the notification.payload from the byte array of index information to axes @@ -300,12 +301,7 @@ def __init__( warnings.warn('Could not create acquisition notification handler. ' 'Update Micro-Manager and Pyrcro-Manager to the latest versions to fix this') - # Start remote acquisition - # Acquistition.start is now deprecated, so this can be removed later - # Acquisitions now get started automatically when the first events submitted - # but Magellan acquisitons (and probably others that generate their own events) - # will need some new method to submit events only after image processors etc have been added - self._acq.start() + self._dataset_disk_location = ( self._acq.get_data_sink().get_storage().get_disk_location() if self._acq.get_data_sink() is not None @@ -321,7 +317,7 @@ def __init__( # when images are written to disk/RAM storage storage_java_class = data_sink.get_storage() summary_metadata = storage_java_class.get_summary_metadata() - if directory is not None: + if directory is not None or isinstance(self, MagellanAcquisition) or isinstance(self, XYTiledAcquisition): # NDTiff dataset saved to disk on Java side self._dataset = Dataset(dataset_path=self._dataset_disk_location, summary_metadata=summary_metadata) else: @@ -364,10 +360,6 @@ def await_completion(self): if hasattr(self, '_event_thread'): self._event_thread.join() - # need to do this so its _Bridge can be garbage collected and a reference to the JavaBackendAcquisition - # does not prevent Bridge cleanup and process exiting - self._remote_acq = None - # Wait on all the other threads to shut down properly if hasattr(self, '_storage_monitor_thread'): self._storage_monitor_thread.join() @@ -633,6 +625,7 @@ def __init__( l = locals() named_args = {arg_name: l[arg_name] for arg_name in arg_names} super().__init__(**named_args) + self._acq.start() def _create_remote_acquisition(self, port, **kwargs): core = ZMQRemoteMMCoreJ(port=self._port, timeout=self._timeout) @@ -648,7 +641,7 @@ def _create_remote_acquisition(self, port, **kwargs): x_overlap = self.tile_overlap y_overlap = self.tile_overlap - self._remote_acq = acq_factory.create_tiled_acquisition( + self._acq = acq_factory.create_tiled_acquisition( kwargs['directory'], kwargs['name'], show_viewer, @@ -710,6 +703,7 @@ def __init__( l = locals() named_args = {arg_name: l[arg_name] for arg_name in arg_names} super().__init__(**named_args) + self._acq.start() def _create_remote_acquisition(self, port, **kwargs): if type(self.tile_overlap) is tuple: @@ -720,7 +714,7 @@ def _create_remote_acquisition(self, port, **kwargs): ui_class = JavaClass('org.micromanager.explore.ExploreAcqUIAndStorage') ui = ui_class.create(kwargs['directory'], kwargs['name'], x_overlap, y_overlap, self.z_step_um, self.channel_group) - self._remote_acq = ui.get_acquisition() + self._acq = ui.get_acquisition() def _start_events(self, **kwargs): pass # These come from the user @@ -767,6 +761,7 @@ def __init__( l = locals() named_args = {arg_name: l[arg_name] for arg_name in arg_names} super().__init__(**named_args) + self._acq.start() def _start_events(self, **kwargs): pass # Magellan handles this on Java side @@ -777,7 +772,7 @@ def _create_event_queue(self, **kwargs): def _create_remote_acquisition(self, **kwargs): magellan_api = Magellan() if self.magellan_acq_index is not None: - self._remote_acq = magellan_api.create_acquisition(self.magellan_acq_index, False) + self._acq = magellan_api.create_acquisition(self.magellan_acq_index, False) elif self.magellan_explore: - self._remote_acq = magellan_api.create_explore_acquisition(False) + self._acq = magellan_api.create_explore_acquisition(False) self._event_queue = None diff --git a/pycromanager/install.py b/pycromanager/install.py index d860c4f0..5b7f6bb6 100644 --- a/pycromanager/install.py +++ b/pycromanager/install.py @@ -14,6 +14,26 @@ MM_DOWNLOAD_URL_MAC = MM_DOWNLOAD_URL_BASE + '/nightly/2.0/Mac' MM_DOWNLOAD_URL_WINDOWS = MM_DOWNLOAD_URL_BASE + '/nightly/2.0/Windows' +def _get_download_url(ci_build=False): + """ + Get the download URL for the latest nightly build of Micro-Manager + + Returns + ------- + str + The URL to the latest nightly build + """ + platform = _get_platform() + if platform == 'Windows': + url = MM_DOWNLOAD_URL_WINDOWS + elif platform == 'Mac': + url = MM_DOWNLOAD_URL_MAC + else: + raise ValueError(f"Unsupported OS: {platform}") + if ci_build: + url = url.replace('nightly', 'ci') + return url + def _get_platform(): """ Get the platform of the system @@ -30,18 +50,12 @@ def _get_platform(): else: raise ValueError(f"Unsupported OS: {sys.platform}") -def _find_versions(): +def _find_versions(ci_build=False): """ - Find all available versions of Micro-Manager nightly builds + Find all available versions of Micro-Manager builds """ - platform = _get_platform() # Get the webpage - if platform == 'Windows': - webpage = requests.get(MM_DOWNLOAD_URL_WINDOWS) - elif platform == 'Mac': - webpage = requests.get(MM_DOWNLOAD_URL_MAC) - else: - raise ValueError(f"Unsupported OS: {platform}") + webpage = requests.get(_get_download_url(ci_build)) return re.findall(r'class="rowDefault" href="([^"]+)', webpage.text) def find_existing_mm_install(): @@ -63,7 +77,7 @@ def find_existing_mm_install(): else: raise ValueError(f"Unsupported OS: {platform}") -def download_and_install(destination='auto', mm_install_log_path=None): +def download_and_install(destination='auto', mm_install_log_path=None, ci_build=False): """ Download and install the latest nightly build of Micro-Manager @@ -71,6 +85,10 @@ def download_and_install(destination='auto', mm_install_log_path=None): ---------- destination : str The directory to install Micro-Manager to. If 'auto', it will install to the user's home directory. + mm_install_log_path : str + Path to save the installation log to + ci_build : bool + If True, download the latest CI build instead of nightly build Returns ------- @@ -80,7 +98,7 @@ def download_and_install(destination='auto', mm_install_log_path=None): windows = _get_platform() == 'Windows' platform = 'Windows' if windows else 'Mac' installer = 'mm_installer.exe' if windows else 'mm_installer.dmg' - latest_version = MM_DOWNLOAD_URL_BASE + _find_versions()[0] + latest_version = _get_download_url(ci_build) + os.sep + _find_versions(ci_build)[0].split(os.sep)[-1] # make a progress bar that updates every 0.5 seconds def bar(curr, total, width): if not hasattr(bar, 'last_update'): @@ -88,6 +106,7 @@ def bar(curr, total, width): if curr / total*100 - bar.last_update > 0.5: print(f"\rDownloading installer: {curr / total*100:.2f}%", end='') bar.last_update = curr / total*100 + print('Downloading: ', latest_version) wget.download(latest_version, out=installer, bar=bar) if windows: From ddb6b25ac226917fdb36a6c84568cc6540b1b0ce Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Sun, 23 Jun 2024 12:22:26 +0200 Subject: [PATCH 02/14] change slashes --- pycromanager/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycromanager/install.py b/pycromanager/install.py index 5b7f6bb6..addb478d 100644 --- a/pycromanager/install.py +++ b/pycromanager/install.py @@ -98,7 +98,7 @@ def download_and_install(destination='auto', mm_install_log_path=None, ci_build= windows = _get_platform() == 'Windows' platform = 'Windows' if windows else 'Mac' installer = 'mm_installer.exe' if windows else 'mm_installer.dmg' - latest_version = _get_download_url(ci_build) + os.sep + _find_versions(ci_build)[0].split(os.sep)[-1] + latest_version = _get_download_url(ci_build) + '/' + _find_versions(ci_build)[0].split('/')[-1] # make a progress bar that updates every 0.5 seconds def bar(curr, total, width): if not hasattr(bar, 'last_update'): From 2e30e53b101e8d8cbfbb835c615abc067c36819c Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Sun, 23 Jun 2024 12:46:29 +0200 Subject: [PATCH 03/14] add close dataset for tests --- pycromanager/test/test_acquisition.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pycromanager/test/test_acquisition.py b/pycromanager/test/test_acquisition.py index d4b5b1f0..298e220e 100644 --- a/pycromanager/test/test_acquisition.py +++ b/pycromanager/test/test_acquisition.py @@ -469,6 +469,7 @@ def test_abort_with_no_events(launch_mm_headless, setup_data_folder): with Acquisition(setup_data_folder, 'test_abort_with_no_events', show_display=False) as acq: acq.abort() assert not mmc.is_sequence_running() + acq.get_dataset().close() def test_abort_from_external(launch_mm_headless, setup_data_folder): """ @@ -485,6 +486,8 @@ def test_abort_from_external(launch_mm_headless, setup_data_folder): acq.acquire(event) time.sleep(5) + acq.get_dataset().close() + def test_abort_sequenced_zstack(launch_mm_headless, setup_data_folder): """ Test that a hardware sequenced acquisition can be aborted mid-sequence From 380718a91d63e25f19e2fd4a2a316d191ef091a0 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Sun, 23 Jun 2024 12:59:30 +0200 Subject: [PATCH 04/14] add close dataset for tests --- pycromanager/test/test_acquisition.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pycromanager/test/test_acquisition.py b/pycromanager/test/test_acquisition.py index 298e220e..129f1570 100644 --- a/pycromanager/test/test_acquisition.py +++ b/pycromanager/test/test_acquisition.py @@ -476,17 +476,18 @@ def test_abort_from_external(launch_mm_headless, setup_data_folder): Simulates the acquisition being shutdown from a remote source (e.g. Xing out the viewer) """ with pytest.raises(AcqAlreadyCompleteException): - with Acquisition(setup_data_folder, 'test_abort_from_external', show_display=False) as acq: - events = multi_d_acquisition_events(num_time_points=6) - acq.acquire(events[0]) - # this simulates an abort from the java side unbeknownst to python side - # it comes from a new thread so it is non-blocking to the port - acq._acq.abort() - for event in events[1:]: - acq.acquire(event) - time.sleep(5) - - acq.get_dataset().close() + try: + with Acquisition(setup_data_folder, 'test_abort_from_external', show_display=False) as acq: + events = multi_d_acquisition_events(num_time_points=6) + acq.acquire(events[0]) + # this simulates an abort from the java side unbeknownst to python side + # it comes from a new thread so it is non-blocking to the port + acq._acq.abort() + for event in events[1:]: + acq.acquire(event) + time.sleep(5) + finally: + acq.get_dataset().close() def test_abort_sequenced_zstack(launch_mm_headless, setup_data_folder): """ From beca5ecbebe220800d6757ea9c18679b58667647 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Sun, 23 Jun 2024 13:13:54 +0200 Subject: [PATCH 05/14] comment test for debug --- pycromanager/test/test_acquisition.py | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pycromanager/test/test_acquisition.py b/pycromanager/test/test_acquisition.py index 129f1570..890797f4 100644 --- a/pycromanager/test/test_acquisition.py +++ b/pycromanager/test/test_acquisition.py @@ -471,23 +471,23 @@ def test_abort_with_no_events(launch_mm_headless, setup_data_folder): assert not mmc.is_sequence_running() acq.get_dataset().close() -def test_abort_from_external(launch_mm_headless, setup_data_folder): - """ - Simulates the acquisition being shutdown from a remote source (e.g. Xing out the viewer) - """ - with pytest.raises(AcqAlreadyCompleteException): - try: - with Acquisition(setup_data_folder, 'test_abort_from_external', show_display=False) as acq: - events = multi_d_acquisition_events(num_time_points=6) - acq.acquire(events[0]) - # this simulates an abort from the java side unbeknownst to python side - # it comes from a new thread so it is non-blocking to the port - acq._acq.abort() - for event in events[1:]: - acq.acquire(event) - time.sleep(5) - finally: - acq.get_dataset().close() +# def test_abort_from_external(launch_mm_headless, setup_data_folder): +# """ +# Simulates the acquisition being shutdown from a remote source (e.g. Xing out the viewer) +# """ +# with pytest.raises(AcqAlreadyCompleteException): +# try: +# with Acquisition(setup_data_folder, 'test_abort_from_external', show_display=False) as acq: +# events = multi_d_acquisition_events(num_time_points=6) +# acq.acquire(events[0]) +# # this simulates an abort from the java side unbeknownst to python side +# # it comes from a new thread so it is non-blocking to the port +# acq._acq.abort() +# for event in events[1:]: +# acq.acquire(event) +# time.sleep(5) +# finally: +# acq.get_dataset().close() def test_abort_sequenced_zstack(launch_mm_headless, setup_data_folder): """ From d3eb140b891d2b6b9bdf0a53e5efac5844a9b879 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:13:44 +0200 Subject: [PATCH 06/14] bump version --- java/pom.xml | 2 +- pycromanager/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index e1c7d700..81cb4bd4 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.micro-manager.pycro-manager PycroManagerJava - 0.46.13 + 0.46.14 jar Pycro-Manager Java The Java components of Pycro-Manager diff --git a/pycromanager/_version.py b/pycromanager/_version.py index 11019ec3..f0fb7e5f 100644 --- a/pycromanager/_version.py +++ b/pycromanager/_version.py @@ -1,2 +1,2 @@ -version_info = (0, 34, 5) +version_info = (0, 34, 6) __version__ = ".".join(map(str, version_info)) From 083844ff873196b61d45129c5a408da54d30407e Mon Sep 17 00:00:00 2001 From: AcqEngJ-Bot <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 07:42:43 +0000 Subject: [PATCH 07/14] update AcqEngJ version and PycroManagerJava version (Created by Github action) --- java/pom.xml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 81cb4bd4..3692906d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.micro-manager.pycro-manager PycroManagerJava - 0.46.14 + 0.46.15 jar Pycro-Manager Java The Java components of Pycro-Manager @@ -55,7 +55,7 @@ org.micro-manager.acqengj AcqEngJ - 0.38.0 + 0.38.1 org.micro-manager.ndviewer @@ -133,13 +133,7 @@ true 20 - + @@ -154,4 +148,4 @@ - + \ No newline at end of file From 49fc0905bd630e8fa89f37722095ab56d6c9c6b1 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:53:16 +0200 Subject: [PATCH 08/14] Update update_dependency.py --- build_automation/update_dependency.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build_automation/update_dependency.py b/build_automation/update_dependency.py index 3f65a1df..abfb2620 100644 --- a/build_automation/update_dependency.py +++ b/build_automation/update_dependency.py @@ -11,13 +11,17 @@ import time dep_name = sys.argv[1] +if len(sys.argv) == 3: # NDTiffStorage.jar comes from NDStorage repo + repo_name = sys.argv[2] +else: + repo_name = dep_name git_repos_dir = str(Path(__file__).parent.parent.parent) + '/' if('java' in os.listdir(git_repos_dir + dep_name)): - pom_path = git_repos_dir + dep_name + '/java/pom.xml' + pom_path = git_repos_dir + repo_name + '/java/pom.xml' else: - pom_path = git_repos_dir + dep_name + '/pom.xml' + pom_path = git_repos_dir + repo_name + '/pom.xml' # Get the latest version number From 9fabc1ced4bd1a21694659b9a5409f148056050a Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:15:09 +0200 Subject: [PATCH 09/14] Update update_dependency.py --- build_automation/update_dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_automation/update_dependency.py b/build_automation/update_dependency.py index abfb2620..d2a487bb 100644 --- a/build_automation/update_dependency.py +++ b/build_automation/update_dependency.py @@ -18,7 +18,7 @@ git_repos_dir = str(Path(__file__).parent.parent.parent) + '/' -if('java' in os.listdir(git_repos_dir + dep_name)): +if('java' in os.listdir(repo_name + dep_name)): pom_path = git_repos_dir + repo_name + '/java/pom.xml' else: pom_path = git_repos_dir + repo_name + '/pom.xml' From 1cce82a8bb66ad4907416445a976f1c050426db6 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:16:44 +0200 Subject: [PATCH 10/14] Update update_dependency.py --- build_automation/update_dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_automation/update_dependency.py b/build_automation/update_dependency.py index d2a487bb..145516e3 100644 --- a/build_automation/update_dependency.py +++ b/build_automation/update_dependency.py @@ -18,7 +18,7 @@ git_repos_dir = str(Path(__file__).parent.parent.parent) + '/' -if('java' in os.listdir(repo_name + dep_name)): +if('java' in os.listdir(git_repos_dir + repo_name)): pom_path = git_repos_dir + repo_name + '/java/pom.xml' else: pom_path = git_repos_dir + repo_name + '/pom.xml' From a381390e6ae501b742078953c43eef6d9c8618e0 Mon Sep 17 00:00:00 2001 From: NDStorage-Bot <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:17:12 +0000 Subject: [PATCH 11/14] update NDStorage version and PycroManagerJava version (Created by Github action) --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 3692906d..6a0bb800 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.micro-manager.pycro-manager PycroManagerJava - 0.46.15 + 0.46.16 jar Pycro-Manager Java The Java components of Pycro-Manager @@ -65,7 +65,7 @@ org.micro-manager.ndtiffstorage NDTiffStorage - 2.18.2 + 2.18.4 From 3ad24ef780587937b6cd72193fc3a402f8158976 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:40:20 +0200 Subject: [PATCH 12/14] bump pyjavaz version --- pycromanager/_version.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pycromanager/_version.py b/pycromanager/_version.py index f0fb7e5f..18b37008 100644 --- a/pycromanager/_version.py +++ b/pycromanager/_version.py @@ -1,2 +1,2 @@ -version_info = (0, 34, 6) +version_info = (0, 34, 7) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index cdcde29e..cf859c51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ ndstorage>=0.1.6 docstring-inheritance pymmcore sortedcontainers -pyjavaz>=1.2.1 +pyjavaz>=1.2.2 wget \ No newline at end of file From 182ec7f5160b7c6107b5937bd52844581dee7a43 Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Sat, 29 Jun 2024 00:13:38 +0200 Subject: [PATCH 13/14] bump versions --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf859c51..b21c01bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ ndstorage>=0.1.6 docstring-inheritance pymmcore sortedcontainers -pyjavaz>=1.2.2 +pyjavaz>=1.2.3 wget \ No newline at end of file From 9478aabb7b171516e7d229501fb0181bd79e235e Mon Sep 17 00:00:00 2001 From: Henry Pinkard <7969470+henrypinkard@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:21:04 +0200 Subject: [PATCH 14/14] bump to new bridge version --- pycromanager/_version.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pycromanager/_version.py b/pycromanager/_version.py index 18b37008..d7681796 100644 --- a/pycromanager/_version.py +++ b/pycromanager/_version.py @@ -1,2 +1,2 @@ -version_info = (0, 34, 7) +version_info = (0, 34, 8) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index b21c01bb..6720e4c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ ndstorage>=0.1.6 docstring-inheritance pymmcore sortedcontainers -pyjavaz>=1.2.3 +pyjavaz==1.2.4 wget \ No newline at end of file