diff --git a/pycromanager/__init__.py b/pycromanager/__init__.py index a6aafabf..41a09d6f 100644 --- a/pycromanager/__init__.py +++ b/pycromanager/__init__.py @@ -1,12 +1,13 @@ name = "pycromanager" -from pycromanager.acquisition.java_backend_acquisitions import JavaBackendAcquisition, MagellanAcquisition, XYTiledAcquisition, ExploreAcquisition +from pycromanager.acquisition.java_backend_acquisitions import (JavaBackendAcquisition, MagellanAcquisition, + XYTiledAcquisition, ExploreAcquisition) from pycromanager.acquisition.acquisition_superclass import multi_d_acquisition_events from pycromanager.acquisition.acq_constructor import Acquisition -from pycromanager.headless import start_headless, stop_headless from pycromanager.mm_java_classes import Studio, Magellan -from pycromanager.core import Core -from pyjavaz import JavaObject, JavaClass, PullSocket, PushSocket from pycromanager.acquisition.acq_eng_py.main.acq_notification import AcqNotification -from ndstorage import Dataset + +from pycromanager.headless import start_headless, stop_headless +from mmpycorex import download_and_install_mm, find_existing_mm_install, Core + from ._version import __version__, version_info diff --git a/pycromanager/acquisition/acq_constructor.py b/pycromanager/acquisition/acq_constructor.py index 6217a494..6e7fe80d 100644 --- a/pycromanager/acquisition/acq_constructor.py +++ b/pycromanager/acquisition/acq_constructor.py @@ -1,8 +1,8 @@ -from pycromanager.headless import _PYMMCORES from pycromanager.acquisition.java_backend_acquisitions import JavaBackendAcquisition from pycromanager.acquisition.python_backend_acquisitions import PythonBackendAcquisition from pycromanager.acquisition.acquisition_superclass import Acquisition as PycromanagerAcquisitionBase from inspect import signature +from mmpycorex import is_pymmcore_active # This is a convenience class that automatically selects the appropriate acquisition # type based on backend is running. It is subclassed from the base acquisition class @@ -29,7 +29,7 @@ def __new__(cls, dict(signature(Acquisition.__init__).parameters.items())[arg_name].default) for arg_name in arg_names } - if _PYMMCORES: + if is_pymmcore_active(): # Python backend detected, so create a python backend acquisition specific_arg_names = [k for k in signature(PythonBackendAcquisition.__init__).parameters.keys() if k != 'self'] for name in specific_arg_names: diff --git a/pycromanager/acq_future.py b/pycromanager/acquisition/acq_future.py similarity index 100% rename from pycromanager/acq_future.py rename to pycromanager/acquisition/acq_future.py diff --git a/pycromanager/acquisition/acquisition_superclass.py b/pycromanager/acquisition/acquisition_superclass.py index ccf00db9..c79b717f 100644 --- a/pycromanager/acquisition/acquisition_superclass.py +++ b/pycromanager/acquisition/acquisition_superclass.py @@ -5,19 +5,16 @@ import copy import types import numpy as np -from typing import Union, List, Iterable +from typing import List, Iterable import warnings from abc import ABCMeta, abstractmethod from docstring_inheritance import NumpyDocstringInheritanceMeta import queue import weakref -from pycromanager.acq_future import AcqNotification, AcquisitionFuture -import os +from pycromanager.acquisition.acq_future import AcqNotification, AcquisitionFuture import threading from inspect import signature -from typing import Generator from types import GeneratorType -import time from queue import Queue from typing import Generator, Dict, Union diff --git a/pycromanager/acquisition/java_backend_acquisitions.py b/pycromanager/acquisition/java_backend_acquisitions.py index da5c0503..38c48f34 100644 --- a/pycromanager/acquisition/java_backend_acquisitions.py +++ b/pycromanager/acquisition/java_backend_acquisitions.py @@ -1,16 +1,13 @@ """ The Pycro-manager Acquisiton system """ -import json import logging import warnings -import weakref import numpy as np import multiprocessing import threading from inspect import signature -import time from pyjavaz import deserialize_array from pyjavaz import PullSocket, PushSocket, JavaObject, JavaClass from pyjavaz import DEFAULT_BRIDGE_PORT as DEFAULT_PORT @@ -19,11 +16,10 @@ from ndstorage import Dataset import os.path -import queue from docstring_inheritance import NumpyDocstringInheritanceMeta from pycromanager.acquisition.acquisition_superclass import Acquisition import traceback -from pycromanager.acq_future import AcqNotification, AcquisitionFuture +from pycromanager.acquisition.acq_future import AcqNotification import json logger = logging.getLogger(__name__) @@ -337,7 +333,7 @@ def __init__( import napari except: raise Exception('Napari must be installed in order to use this feature') - from pycromanager.napari_util import start_napari_signalling + from pycromanager.acquisition.napari_util import start_napari_signalling assert isinstance(napari_viewer, napari.Viewer), 'napari_viewer must be an instance of napari.Viewer' self._napari_viewer = napari_viewer start_napari_signalling(self._napari_viewer, self.get_dataset()) diff --git a/pycromanager/napari_util.py b/pycromanager/acquisition/napari_util.py similarity index 100% rename from pycromanager/napari_util.py rename to pycromanager/acquisition/napari_util.py diff --git a/pycromanager/acquisition/python_backend_acquisitions.py b/pycromanager/acquisition/python_backend_acquisitions.py index 070adc93..3a513661 100644 --- a/pycromanager/acquisition/python_backend_acquisitions.py +++ b/pycromanager/acquisition/python_backend_acquisitions.py @@ -3,7 +3,7 @@ from pycromanager.acquisition.acq_eng_py.main.AcqEngPy_Acquisition import Acquisition as pymmcore_Acquisition from pycromanager.acquisition.acquisition_superclass import _validate_acq_events, Acquisition from pycromanager.acquisition.acq_eng_py.main.acquisition_event import AcquisitionEvent -from pycromanager.acq_future import AcqNotification +from pycromanager.acquisition.acq_future import AcqNotification import threading from inspect import signature import traceback @@ -110,7 +110,7 @@ def post_notification(notification): import napari except: raise Exception('Napari must be installed in order to use this feature') - from pycromanager.napari_util import start_napari_signalling + from pycromanager.acquisition.napari_util import start_napari_signalling assert isinstance(napari_viewer, napari.Viewer), 'napari_viewer must be an instance of napari.Viewer' self._napari_viewer = napari_viewer start_napari_signalling(self._napari_viewer, self.get_dataset()) diff --git a/pycromanager/core.py b/pycromanager/core.py deleted file mode 100644 index 85bf512c..00000000 --- a/pycromanager/core.py +++ /dev/null @@ -1,15 +0,0 @@ -from pycromanager.mm_java_classes import ZMQRemoteMMCoreJ -import pymmcore -from pycromanager.headless import _PYMMCORES - -class Core(): - """ - Return a remote Java ZMQ Core, or a local Python Core, if the start_headless has been called with a Python backend - """ - - def __new__(cls, **kwargs): - if _PYMMCORES: - return _PYMMCORES[0] - else: - return ZMQRemoteMMCoreJ(**kwargs) - diff --git a/pycromanager/headless.py b/pycromanager/headless.py index 84b3658a..63d7f9cb 100644 --- a/pycromanager/headless.py +++ b/pycromanager/headless.py @@ -1,127 +1,11 @@ -import logging -import subprocess -import platform -import atexit -import threading -import types -import os - +from mmpycorex import create_core_instance, terminate_core_instances +from mmpycorex import Core from pycromanager.acquisition.acq_eng_py.internal.engine import Engine -from pymmcore import CMMCore +from pyjavaz import DEFAULT_BRIDGE_PORT +import atexit import pymmcore -from pyjavaz import DEFAULT_BRIDGE_PORT, server_terminated - -import re - -logger = logging.getLogger(__name__) - -class TaggedImage: - - def __init__(self, tags, pix): - self.tags = tags - self.pix = pix - -def _camel_to_snake(name): - """ - Convert camelCase string to snake_case - """ - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - -def _create_pymmcore_instance(): - """ - Make a subclass of CMMCore with two differences: - - 1. All methods are converted to snake_case - 2. add convenience methods to match the MMCoreJ API: - """ - - # Create a new dictionary for the class attributes - new_attributes = {} - - # Iterate through the original attributes - for attr_name, attr_value in vars(CMMCore).items(): - # If it's a dunder method, skip it (we don't want to override these) - if attr_name.startswith("__") and attr_name.endswith("__"): - continue - # If the attribute is callable (i.e., a method), convert its name to snake_case and add it - if callable(attr_value): - new_attr_name = _camel_to_snake(attr_name) - new_attributes[new_attr_name] = attr_value - - # Create and return a new class that subclasses the original class and has the new attributes - clz = type(CMMCore.__name__ + "SnakeCase", (CMMCore,), new_attributes) - - instance = clz() - - def pop_next_tagged_image(self): - md = pymmcore.Metadata() - pix = self.pop_next_image_md(0, 0, md) - tags = {key: md.GetSingleTag(key).GetValue() for key in md.GetKeys()} - return TaggedImage(tags, pix) - - def get_tagged_image(core, cam_index, camera, height, width, binning=None, pixel_type=None, roi_x_start=None, - roi_y_start=None): - """ - Different signature than the Java version because of difference in metadata handling in the swig layers - """ - pix = core.get_image() - md = pymmcore.Metadata() - # most of the same tags from pop_next_tagged_image, which may not be the same as the MMCoreJ version of this function - tags = {'Camera': camera, 'Height': height, 'Width': width, 'PixelType': pixel_type, - 'CameraChannelIndex': cam_index} - # Could optionally add these for completeness but there might be a performance hit - if binning is not None: - tags['Binning'] = binning - if roi_x_start is not None: - tags['ROI-X-start'] = roi_x_start - if roi_y_start is not None: - tags['ROI-Y-start'] = roi_y_start - - return TaggedImage(tags, pix) - - instance.get_tagged_image = types.MethodType(get_tagged_image, instance) - instance.pop_next_tagged_image = types.MethodType(pop_next_tagged_image, instance) - - # attach TaggedImage class - instance.TaggedImage = TaggedImage - return instance - - -_JAVA_HEADLESS_SUBPROCESSES = [] -_PYMMCORES = [] - -def stop_headless(debug=False): - - for p in _JAVA_HEADLESS_SUBPROCESSES: - port = p.port - if debug: - logger.debug('Stopping headless process with pid {}'.format(p.pid)) - p.terminate() - server_terminated(port) - if debug: - logger.debug('Waiting for process with pid {} to terminate'.format(p.pid)) - p.wait() # wait for process to terminate - if debug: - logger.debug('Process with pid {} terminated'.format(p.pid)) - _JAVA_HEADLESS_SUBPROCESSES.clear() - if debug: - logger.debug('Stopping {} pymmcore instances'.format(len(_PYMMCORES))) - for c in _PYMMCORES: - if debug: - logger.debug('Stopping pymmcore instance') - c.unloadAllDevices() - if debug: - logger.debug('Unloaded all devices') - Engine.get_instance().shutdown() - if debug: - logger.debug('Engine shut down') - _PYMMCORES.clear() - if debug: - logger.debug('Headless stopped') +import types -# make sure any Java processes are cleaned up when Python exits -atexit.register(stop_headless) def start_headless( mm_app_path: str, config_file: str=None, java_loc: str=None, @@ -129,9 +13,9 @@ def start_headless( buffer_size_mb: int=1024, max_memory_mb: int=2000, port: int=DEFAULT_BRIDGE_PORT, debug=False): """ - Start a Java process that contains the neccessary libraries for pycro-manager to run, - so that it can be run independently of the Micro-Manager GUI/application. This calls - will create and initialize MMCore with the configuration file provided. + Start an instance of the Micro-Manager core and acquisition engine in headless mode. This can be + either a Python (i.e. pymmcore) or Java (i.e. MMCoreJ) backend. If a Python backend is used, + the core will be started in the same process. On windows plaforms, the Java Runtime Environment will be grabbed automatically as it is installed along with the Micro-Manager application. @@ -161,68 +45,17 @@ def start_headless( debug : bool Print debug messages """ - + create_core_instance( + mm_app_path=mm_app_path, config_file=config_file, java_loc=java_loc, + python_backend=python_backend, core_log_path=core_log_path, + buffer_size_mb=buffer_size_mb, max_memory_mb=max_memory_mb, + port=port, debug=debug) if python_backend: - mmc = _create_pymmcore_instance() - mmc.set_device_adapter_search_paths([mm_app_path]) - if config_file is not None and config_file != "": - mmc.load_system_configuration(config_file) - mmc.set_circular_buffer_memory_footprint(buffer_size_mb) - _PYMMCORES.append(mmc) # Store so it doesn't get garbage collected - Engine(mmc) - else: - classpath = mm_app_path + '/plugins/Micro-Manager/*' - if java_loc is None: - if platform.system() == "Windows": - # windows comes with its own JRE - java_loc = mm_app_path + "/jre/bin/javaw.exe" - else: - java_loc = "java" - if debug: - logger.debug(f'Java location: {java_loc}') - #print classpath - logger.debug(f'Classpath: {classpath}') - # print stuff in the classpath directory - logger.debug('Contents of classpath directory:') - for f in os.listdir(classpath.split('*')[0]): - logger.debug(f) - - # This starts Java process and instantiates essential objects (core, - # acquisition engine, ZMQServer) - process = subprocess.Popen( - [ - java_loc, - "-classpath", - classpath, - "-Dsun.java2d.dpiaware=false", - f"-Xmx{max_memory_mb}m", - # This is used by MM desktop app but breaks things on MacOS...Don't think its neccessary - # "-XX:MaxDirectMemorySize=1000", - "org.micromanager.remote.HeadlessLauncher", - str(port), - config_file if config_file is not None else '', - str(buffer_size_mb), - core_log_path, - ], cwd=mm_app_path, stdout=subprocess.PIPE - ) - process.port = port - _JAVA_HEADLESS_SUBPROCESSES.append(process) - - started = False - output = True - # Some drivers output various status messages which need to be skipped over to look for the STARTED token. - while output and not started: - output = process.stdout.readline() - started = "STARTED" in output.decode('utf-8') - if not started: - raise Exception('Error starting headless mode') - if debug: - logger.debug('Headless mode started') - def loggerFunction(): - while process in _JAVA_HEADLESS_SUBPROCESSES: - line = process.stdout.readline().decode('utf-8') - if line.strip() != '': - logger.debug(line) - threading.Thread(target=loggerFunction).start() + Engine(Core()) +def stop_headless(debug=False): + terminate_core_instances(debug=debug) + Engine.get_instance().shutdown() +# make sure any Java processes are cleaned up when Python exits +atexit.register(stop_headless) \ No newline at end of file diff --git a/pycromanager/install.py b/pycromanager/install.py deleted file mode 100644 index addb478d..00000000 --- a/pycromanager/install.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -Utility functions for installing micro-manager -""" -import sys -import os -import re -import requests -import wget -import subprocess -import shutil - -MM_DOWNLOAD_URL_BASE = 'https://download.micro-manager.org' - -MM_DOWNLOAD_URL_MAC = MM_DOWNLOAD_URL_BASE + '/nightly/2.0/Mac' -MM_DOWNLOAD_URL_WINDOWS = MM_DOWNLOAD_URL_BASE + '/nightly/2.0/Windows' - -def _get_download_url(ci_build=False): - """ - Get the download URL for the latest nightly build of Micro-Manager - - Returns - ------- - str - The URL to the latest nightly build - """ - platform = _get_platform() - if platform == 'Windows': - url = MM_DOWNLOAD_URL_WINDOWS - elif platform == 'Mac': - url = MM_DOWNLOAD_URL_MAC - else: - raise ValueError(f"Unsupported OS: {platform}") - if ci_build: - url = url.replace('nightly', 'ci') - return url - -def _get_platform(): - """ - Get the platform of the system - - Returns - ------- - str - "Windows" or "Mac" - """ - if sys.platform.startswith('win'): - return 'Windows' - elif sys.platform.startswith('darwin'): - return 'Mac' - else: - raise ValueError(f"Unsupported OS: {sys.platform}") - -def _find_versions(ci_build=False): - """ - Find all available versions of Micro-Manager builds - """ - # Get the webpage - webpage = requests.get(_get_download_url(ci_build)) - return re.findall(r'class="rowDefault" href="([^"]+)', webpage.text) - -def find_existing_mm_install(): - """ - Check if Micro-Manager is installed in the default auto-download paths - - Returns - ------- - str - The path to the installed Micro-Manager directory, or None if not found - """ - platform = _get_platform() - if platform == 'Windows': - if os.path.isdir(r'C:\Program Files\Micro-Manager'): - return r'C:\Program Files\Micro-Manager' - elif platform == 'Mac': - if os.path.isdir(str(os.path.expanduser('~')) + '/Micro-Manager'): - return str(os.path.expanduser('~')) + '/Micro-Manager' - else: - raise ValueError(f"Unsupported OS: {platform}") - -def download_and_install(destination='auto', mm_install_log_path=None, ci_build=False): - """ - Download and install the latest nightly build of Micro-Manager - - Parameters - ---------- - destination : str - The directory to install Micro-Manager to. If 'auto', it will install to the user's home directory. - mm_install_log_path : str - Path to save the installation log to - ci_build : bool - If True, download the latest CI build instead of nightly build - - Returns - ------- - str - The path to the installed Micro-Manager directory - """ - windows = _get_platform() == 'Windows' - platform = 'Windows' if windows else 'Mac' - installer = 'mm_installer.exe' if windows else 'mm_installer.dmg' - latest_version = _get_download_url(ci_build) + '/' + _find_versions(ci_build)[0].split('/')[-1] - # make a progress bar that updates every 0.5 seconds - def bar(curr, total, width): - if not hasattr(bar, 'last_update'): - bar.last_update = 0 - if curr / total*100 - bar.last_update > 0.5: - print(f"\rDownloading installer: {curr / total*100:.2f}%", end='') - bar.last_update = curr / total*100 - print('Downloading: ', latest_version) - wget.download(latest_version, out=installer, bar=bar) - - if windows: - if destination == 'auto': - destination = r'C:\Program Files\Micro-Manager' - cmd = f"{installer} /SP /VERYSILENT /SUPRESSMSGBOXES /CURRENTUSER /DIR=\"{destination}\"" - - if mm_install_log_path: - cmd += f" /LOG={mm_install_log_path}" - subprocess.run(cmd, shell=True) - - return destination - else: - if destination == 'auto': - destination = str(os.path.expanduser('~')) + '/Micro-Manager' - try: - # unmount if already mounted - subprocess.run(['hdiutil', 'detach', '/Volumes/Micro-Manager']) - except: - pass - process = subprocess.run(['hdiutil', 'attach', '-nobrowse', str(installer)]) - latest_build = [name for name in os.listdir('/Volumes/Micro-Manager') if 'Micro-Manager' in name][0] - shutil.copytree('/Volumes/Micro-Manager/' + latest_build, destination, dirs_exist_ok=True) - # unmount - subprocess.run(['hdiutil', 'detach', '/Volumes/Micro-Manager']) - # delete this installer - os.remove(installer) - return destination - - # For issues with M1 Macs: https://github.com/conda-forge/miniforge/issues/165#issuecomment-860233092 diff --git a/pycromanager/mm_java_classes.py b/pycromanager/mm_java_classes.py index ac5bf87a..3808cc42 100644 --- a/pycromanager/mm_java_classes.py +++ b/pycromanager/mm_java_classes.py @@ -1,104 +1,8 @@ """ Classes that wrap instance of known java objects for ease of use """ -from pyjavaz import JavaObject, PullSocket, DEFAULT_BRIDGE_PORT, DEFAULT_BRIDGE_TIMEOUT -import threading - -class _CoreCallback: - """ - A class for recieving callbacks from the core, which are mostly used - for the case where some hardware has changed - See (https://github.com/micro-manager/mmCoreAndDevices/blob/main/MMCore/CoreCallback.cpp) - - """ - - def __init__(self, callback_fn=None, bridge_port=DEFAULT_BRIDGE_PORT): - self._closed = False - self._thread = threading.Thread( - target=self._callback_recieving_fn, - name="CoreCallback", - args=(bridge_port, self), - ) - self.callback_fn = callback_fn - self._thread.start() - - def _callback_recieving_fn(self, bridge_port, core_callback): - callback_java = JavaObject( - "org.micromanager.remote.RemoteCoreCallback", args=(ZMQRemoteMMCoreJ(port=bridge_port),) - ) - - port = callback_java.get_push_port() - pull_socket = PullSocket(port) - callback_java.start_push() - - while True: - message = pull_socket.receive(timeout=100) - if message is not None: - core_callback._set_value(message) - if core_callback._closed: - callback_java.shutdown() - break - - def _set_value(self, value): - """ - Call the callback function - :return: - - """ - function_name = value["name"] - function_args = value["arguments"] if "arguments" in value else tuple() - - if self.callback_fn is not None: - self.callback_fn(function_name, *function_args) - - def __del__(self): - self._closed = True - self._thread.join() - - -class ZMQRemoteMMCoreJ(JavaObject): - """ - Remote instance of Micro-Manager Core - """ - - def __new__( - cls, convert_camel_case=True, port=DEFAULT_BRIDGE_PORT, new_socket=False, debug=False, timeout=1000, - ): - """ - Parameters - ---------- - convert_camel_case: bool - If True, methods for Java objects that are passed across the bridge - will have their names converted from camel case to underscores. i.e. class.methodName() - becomes class.method_name() - port: int - The port of the Bridge used to create the object - new_socket: bool - If True, will create new java object on a new port so that blocking calls will not interfere - with the bridges main port - debug: - print debug messages - timeout: - timeout for underlying bridge - """ - try: - return JavaObject("mmcorej.CMMCore", new_socket=new_socket, - port=port, timeout=timeout, convert_camel_case=convert_camel_case, debug=debug) - except Exception as e: - raise Exception(f"Couldn't create Core. Is Micro-Manager running and is the ZMQ server on {port} option enabled?") - - def get_core_callback(self, callback_fn=None, bridge_port=DEFAULT_BRIDGE_PORT): - """ - Get a CoreCallback function that will fire callback_fn with (name, *args) each - time MMCore emits a callback signal - - callback_fn: Callable - a function that takes (name, *args) - bridge_port: int - port of the Core instance to get callbacks from - """ - return _CoreCallback(callback_fn=callback_fn, bridge_port=bridge_port) - +from pyjavaz import JavaObject, DEFAULT_BRIDGE_PORT, DEFAULT_BRIDGE_TIMEOUT +from mmpycorex.core import ZMQRemoteMMCoreJ # Don't delete this, its called by other code class Magellan(JavaObject): """ diff --git a/pycromanager/test/conftest.py b/pycromanager/test/conftest.py index e59fd501..1c5992fc 100644 --- a/pycromanager/test/conftest.py +++ b/pycromanager/test/conftest.py @@ -1,21 +1,15 @@ import os import sys import shutil -import subprocess import warnings import pytest -import wget -import requests import re -import time import glob -import pycromanager -from pycromanager import start_headless -from pycromanager.headless import stop_headless +from pycromanager import start_headless, stop_headless import socket -from pycromanager.install import download_and_install, find_existing_mm_install +from mmpycorex import download_and_install_mm, find_existing_mm_install def is_port_in_use(port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -66,7 +60,7 @@ def install_mm(): yield find_existing_mm_install() else: # Download an install latest nightly build - mm_install_dir = download_and_install(destination='auto') + mm_install_dir = download_and_install_mm(destination='auto') #### Replace with newer versions of Java libraries #### # find pycro-manager/java path diff --git a/requirements.txt b/requirements.txt index 6720e4c8..8633f756 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ numpy dask[array]>=2022.2.0 -pyzmq ndstorage>=0.1.6 docstring-inheritance -pymmcore -sortedcontainers -pyjavaz==1.2.4 -wget \ No newline at end of file +mmpycorex +sortedcontainers \ No newline at end of file diff --git a/scripts/string_axes.py b/scripts/string_axes.py index 6cbe26b0..7f361fde 100644 --- a/scripts/string_axes.py +++ b/scripts/string_axes.py @@ -3,8 +3,8 @@ """ -from pycromanager import JavaBackendAcquisition, multi_d_acquisition_events -from pycromanager.headless import multi_d_acquisition_events_new +from pycromanager import JavaBackendAcquisition +from launcher import multi_d_acquisition_events_new with JavaBackendAcquisition(directory="/Users/henrypinkard/tmp", name="NDTiff3.2_monochrome", debug=False) as acq: