From 22b1ea6ab8afb64882d208a913ed3107c8af300d Mon Sep 17 00:00:00 2001 From: rtuck99 Date: Mon, 7 Oct 2024 11:11:08 +0100 Subject: [PATCH] 470 Full experiment plan to replace execute request (#538) * Implementation of grand unified experiment plan * Add load_centre_collect to the experiment registry * Additional system test for full experiment plan * Make ruff happpy * Changes following PR review comments * Make pyright happy --- .../hyperion/reference/param_hierarchy.puml | 24 +- .../experiment_plans/experiment_registry.py | 9 + .../flyscan_xray_centre_plan.py | 85 +++-- .../load_centre_collect_full_plan.py | 46 +++ .../callbacks/common/callback_util.py | 18 + .../callbacks/ispyb_callback_base.py | 6 +- .../hyperion/parameters/components.py | 18 +- .../parameters/load_centre_collect.py | 50 +++ tests/conftest.py | 47 ++- tests/system_tests/conftest.py | 226 ++++++++++++ .../hyperion/external_interaction/conftest.py | 198 ++++++++++- .../test_ispyb_dev_connection.py | 336 +----------------- .../test_load_centre_collect_full_plan.py | 254 +++++++++++++ .../example_load_centre_collect_params.json | 53 +++ .../good_test_load_centre_collect_params.json | 45 +++ .../good_test_robot_load_params.json | 1 + .../hyperion/experiment_plans/conftest.py | 93 ++++- .../test_flyscan_xray_centre_plan.py | 76 ++-- .../test_load_centre_collect_full_plan.py | 224 ++++++++++++ .../test_pin_centre_then_xray_centre_plan.py | 4 + .../test_robot_load_and_change_energy.py | 65 ++-- .../test_robot_load_then_centre.py | 68 ---- 22 files changed, 1388 insertions(+), 558 deletions(-) create mode 100644 src/mx_bluesky/hyperion/experiment_plans/load_centre_collect_full_plan.py create mode 100644 src/mx_bluesky/hyperion/parameters/load_centre_collect.py create mode 100644 tests/system_tests/conftest.py create mode 100644 tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py create mode 100644 tests/test_data/parameter_json_files/example_load_centre_collect_params.json create mode 100644 tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json create mode 100644 tests/unit_tests/hyperion/experiment_plans/test_load_centre_collect_full_plan.py diff --git a/docs/developer/hyperion/reference/param_hierarchy.puml b/docs/developer/hyperion/reference/param_hierarchy.puml index 64e3f9ef3..3272a1960 100644 --- a/docs/developer/hyperion/reference/param_hierarchy.puml +++ b/docs/developer/hyperion/reference/param_hierarchy.puml @@ -8,7 +8,9 @@ package Mixins { class WithSample class WithScan class WithOavCentring + class WithOptionalEnergyChange class WithSnapshot + class WithVisit class OptionalXyzStarts class XyzStarts class OptionalGonioAngleStarts @@ -22,49 +24,51 @@ package Experiments { class DiffractionExperimentWithSample class GridCommon class GridScanWithEdgeDetect + class LoadCentreCollect class PinTipCentreThenXrayCentre class RotationScan class MultiRotationScan + class RobotLoadAndEnergyChange class RobotLoadThenCentre class SpecifiedGridScan class ThreeDGridScan } class HyperionParameters -note bottom: Base class for all experiment parameter models - -class TemporaryIspybExtras -note bottom: To be removed - +note top: Base class for all experiment parameter models BaseModel <|-- HyperionParameters BaseModel <|-- SplitScan BaseModel <|-- OptionalGonioAngleStarts BaseModel <|-- OptionalXyzStarts -BaseModel <|-- TemporaryIspybExtras BaseModel <|-- WithOavCentring +BaseModel <|-- WithOptionalEnergyChange BaseModel <|-- WithSnapshot BaseModel <|-- WithSample BaseModel <|-- WithScan +BaseModel <|-- WithVisit BaseModel <|-- XyzStarts -RotationScan *-- TemporaryIspybExtras -MultiRotationScan *-- TemporaryIspybExtras OptionalGonioAngleStarts <|-- RotationScanPerSweep OptionalXyzStarts <|-- RotationScanPerSweep DiffractionExperimentWithSample <|-- RotationExperiment HyperionParameters <|-- DiffractionExperiment WithSnapshot <|-- DiffractionExperiment +WithOptionalEnergyChange <|-- DiffractionExperiment +WithVisit <|-- DiffractionExperiment DiffractionExperiment <|-- DiffractionExperimentWithSample WithSample <|-- DiffractionExperimentWithSample DiffractionExperimentWithSample <|-- GridCommon GridCommon <|-- GridScanWithEdgeDetect GridCommon <|-- PinTipCentreThenXrayCentre GridCommon <|-- RobotLoadThenCentre +RobotLoadThenCentre *-- RobotLoadAndEnergyChange +RobotLoadThenCentre *-- PinTipCentreThenXrayCentre GridCommon <|-- SpecifiedGridScan WithScan <|-- SpecifiedGridScan SpecifiedGridScan <|-- ThreeDGridScan SplitScan <|-- ThreeDGridScan +WithOptionalEnergyChange <|-- ThreeDGridScan WithOavCentring <|-- GridCommon WithScan <|-- RotationScan RotationScanPerSweep <|-- RotationScan @@ -75,4 +79,8 @@ SplitScan <|-- MultiRotationScan XyzStarts <|-- SpecifiedGridScan OptionalGonioAngleStarts <|-- GridCommon OptionalGonioAngleStarts <|-- RotationScan +HyperionParameters <|-- RobotLoadAndEnergyChange +WithSample <|-- RobotLoadAndEnergyChange +WithSnapshot <|-- RobotLoadAndEnergyChange +WithOptionalEnergyChange <|-- RobotLoadAndEnergyChange @enduml diff --git a/src/mx_bluesky/hyperion/experiment_plans/experiment_registry.py b/src/mx_bluesky/hyperion/experiment_plans/experiment_registry.py index 860608624..c07a9f05c 100644 --- a/src/mx_bluesky/hyperion/experiment_plans/experiment_registry.py +++ b/src/mx_bluesky/hyperion/experiment_plans/experiment_registry.py @@ -7,12 +7,14 @@ import mx_bluesky.hyperion.experiment_plans.rotation_scan_plan as rotation_scan_plan from mx_bluesky.hyperion.experiment_plans import ( grid_detect_then_xray_centre_plan, + load_centre_collect_full_plan, pin_centre_then_xray_centre_plan, robot_load_then_centre_plan, ) from mx_bluesky.hyperion.external_interaction.callbacks.common.callback_util import ( CallbacksFactory, create_gridscan_callbacks, + create_load_centre_collect_callbacks, create_robot_load_and_centre_callbacks, create_rotation_callbacks, ) @@ -22,6 +24,7 @@ RobotLoadThenCentre, ThreeDGridScan, ) +from mx_bluesky.hyperion.parameters.load_centre_collect import LoadCentreCollect from mx_bluesky.hyperion.parameters.rotation import MultiRotationScan, RotationScan @@ -42,6 +45,7 @@ class ExperimentRegistryEntry(TypedDict): | MultiRotationScan | PinTipCentreThenXrayCentre | RobotLoadThenCentre + | LoadCentreCollect ] callbacks_factory: CallbacksFactory @@ -77,6 +81,11 @@ class ExperimentRegistryEntry(TypedDict): "param_type": MultiRotationScan, "callbacks_factory": create_rotation_callbacks, }, + "load_centre_collect_full_plan": { + "setup": load_centre_collect_full_plan.create_devices, + "param_type": LoadCentreCollect, + "callbacks_factory": create_load_centre_collect_callbacks, + }, } diff --git a/src/mx_bluesky/hyperion/experiment_plans/flyscan_xray_centre_plan.py b/src/mx_bluesky/hyperion/experiment_plans/flyscan_xray_centre_plan.py index 130a574d8..cb2b154a1 100755 --- a/src/mx_bluesky/hyperion/experiment_plans/flyscan_xray_centre_plan.py +++ b/src/mx_bluesky/hyperion/experiment_plans/flyscan_xray_centre_plan.py @@ -77,6 +77,12 @@ class SmargonSpeedException(Exception): pass +class CrystalNotFoundException(WarningException): + """Raised if grid detection completed normally but no crystal was found.""" + + pass + + @dataclasses.dataclass class FlyScanXRayCentreComposite: """All devices which are directly or indirectly required by this plan""" @@ -190,47 +196,50 @@ def run_gridscan_and_move( LOGGER.info("Grid scan finished, getting results.") - with TRACER.start_span("wait_for_zocalo"): - yield from bps.trigger_and_read( - [fgs_composite.zocalo], name=ZOCALO_READING_PLAN_NAME - ) - LOGGER.info("Zocalo triggered and read, interpreting results.") - xray_centre, bbox_size = yield from get_processing_result(fgs_composite.zocalo) - LOGGER.info(f"Got xray centre: {xray_centre}, bbox size: {bbox_size}") - if xray_centre is not None: - xray_centre = parameters.FGS_params.grid_position_to_motor_position( - xray_centre + try: + with TRACER.start_span("wait_for_zocalo"): + yield from bps.trigger_and_read( + [fgs_composite.zocalo], name=ZOCALO_READING_PLAN_NAME ) - else: - xray_centre = initial_xyz - LOGGER.warning("No X-ray centre received") - if bbox_size is not None: - with TRACER.start_span("change_aperture"): - yield from set_aperture_for_bbox_size( - fgs_composite.aperture_scatterguard, bbox_size + LOGGER.info("Zocalo triggered and read, interpreting results.") + xray_centre, bbox_size = yield from get_processing_result( + fgs_composite.zocalo + ) + LOGGER.info(f"Got xray centre: {xray_centre}, bbox size: {bbox_size}") + if xray_centre is not None: + xray_centre = parameters.FGS_params.grid_position_to_motor_position( + xray_centre ) - else: - LOGGER.warning("No bounding box size received") - - # once we have the results, go to the appropriate position - LOGGER.info("Moving to centre of mass.") - with TRACER.start_span("move_to_result"): - x, y, z = xray_centre - yield from move_x_y_z(fgs_composite.sample_motors, x, y, z, wait=True) - - if parameters.FGS_params.set_stub_offsets: - LOGGER.info("Recentring smargon co-ordinate system to this point.") - yield from bps.mv( - fgs_composite.sample_motors.stub_offsets, StubPosition.CURRENT_AS_CENTER - ) - - # Turn off dev/shm streaming to avoid filling disk, see https://github.com/DiamondLightSource/hyperion/issues/1395 - LOGGER.info("Turning off Eiger dev/shm streaming") - yield from bps.abs_set(fgs_composite.eiger.odin.fan.dev_shm_enable, 0) + else: + LOGGER.warning("No X-ray centre received") + raise CrystalNotFoundException() + if bbox_size is not None: + with TRACER.start_span("change_aperture"): + yield from set_aperture_for_bbox_size( + fgs_composite.aperture_scatterguard, bbox_size + ) + else: + LOGGER.warning("No bounding box size received") + + # once we have the results, go to the appropriate position + LOGGER.info("Moving to centre of mass.") + with TRACER.start_span("move_to_result"): + x, y, z = xray_centre + yield from move_x_y_z(fgs_composite.sample_motors, x, y, z, wait=True) + + if parameters.FGS_params.set_stub_offsets: + LOGGER.info("Recentring smargon co-ordinate system to this point.") + yield from bps.mv( + fgs_composite.sample_motors.stub_offsets, StubPosition.CURRENT_AS_CENTER + ) + finally: + # Turn off dev/shm streaming to avoid filling disk, see https://github.com/DiamondLightSource/hyperion/issues/1395 + LOGGER.info("Turning off Eiger dev/shm streaming") + yield from bps.abs_set(fgs_composite.eiger.odin.fan.dev_shm_enable, 0) - # Wait on everything before returning to GDA (particularly apertures), can be removed - # when we do not return to GDA here - yield from bps.wait() + # Wait on everything before returning to GDA (particularly apertures), can be removed + # when we do not return to GDA here + yield from bps.wait() @bpp.set_run_key_decorator(CONST.PLAN.GRIDSCAN_MAIN) diff --git a/src/mx_bluesky/hyperion/experiment_plans/load_centre_collect_full_plan.py b/src/mx_bluesky/hyperion/experiment_plans/load_centre_collect_full_plan.py new file mode 100644 index 000000000..62830471e --- /dev/null +++ b/src/mx_bluesky/hyperion/experiment_plans/load_centre_collect_full_plan.py @@ -0,0 +1,46 @@ +import dataclasses + +from blueapi.core import BlueskyContext +from dodal.devices.oav.oav_parameters import OAVParameters + +from mx_bluesky.hyperion.experiment_plans.robot_load_then_centre_plan import ( + RobotLoadThenCentreComposite, + robot_load_then_centre, +) +from mx_bluesky.hyperion.experiment_plans.rotation_scan_plan import ( + RotationScanComposite, + multi_rotation_scan, +) +from mx_bluesky.hyperion.parameters.load_centre_collect import LoadCentreCollect +from mx_bluesky.hyperion.utils.context import device_composite_from_context + + +@dataclasses.dataclass +class LoadCentreCollectComposite(RobotLoadThenCentreComposite, RotationScanComposite): + """Composite that provides access to the required devices.""" + + pass + + +def create_devices(context: BlueskyContext) -> LoadCentreCollectComposite: + """Create the necessary devices for the plan.""" + return device_composite_from_context(context, LoadCentreCollectComposite) + + +def load_centre_collect_full_plan( + composite: LoadCentreCollectComposite, + params: LoadCentreCollect, + oav_params: OAVParameters | None = None, +): + """Attempt a complete data collection experiment, consisting of the following: + * Load the sample if necessary + * Move to the specified goniometer start angles + * Perform optical centring, then X-ray centring + * If X-ray centring finds a diffracting centre then move to that centre and + * do a collection with the specified parameters. + """ + if not oav_params: + oav_params = OAVParameters(context="xrayCentring") + yield from robot_load_then_centre(composite, params.robot_load_then_centre) + + yield from multi_rotation_scan(composite, params.multi_rotation_scan, oav_params) diff --git a/src/mx_bluesky/hyperion/external_interaction/callbacks/common/callback_util.py b/src/mx_bluesky/hyperion/external_interaction/callbacks/common/callback_util.py index d5be89bf5..9f5baff27 100644 --- a/src/mx_bluesky/hyperion/external_interaction/callbacks/common/callback_util.py +++ b/src/mx_bluesky/hyperion/external_interaction/callbacks/common/callback_util.py @@ -44,3 +44,21 @@ def create_rotation_callbacks() -> ( tuple[RotationNexusFileCallback, RotationISPyBCallback] ): return (RotationNexusFileCallback(), RotationISPyBCallback(emit=ZocaloCallback())) + + +def create_load_centre_collect_callbacks() -> ( + tuple[ + GridscanNexusFileCallback, + GridscanISPyBCallback, + RobotLoadISPyBCallback, + RotationNexusFileCallback, + RotationISPyBCallback, + ] +): + return ( + GridscanNexusFileCallback(), + GridscanISPyBCallback(emit=ZocaloCallback()), + RobotLoadISPyBCallback(), + RotationNexusFileCallback(), + RotationISPyBCallback(emit=ZocaloCallback()), + ) diff --git a/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py index fcd4a822e..f8bc99b6f 100644 --- a/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/mx_bluesky/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -110,9 +110,9 @@ def _handle_ispyb_hardware_read(self, doc) -> Sequence[ScanDataInfo]: slitgap_vertical=doc["data"]["s4_slit_gaps_ygap"], ) hwscan_position_info = DataCollectionPositionInfo( - pos_x=doc["data"]["smargon-x"], - pos_y=doc["data"]["smargon-y"], - pos_z=doc["data"]["smargon-z"], + pos_x=float(doc["data"]["smargon-x"]), + pos_y=float(doc["data"]["smargon-y"]), + pos_z=float(doc["data"]["smargon-z"]), ) scan_data_infos = self.populate_info_for_update( hwscan_data_collection_info, hwscan_position_info, self.params diff --git a/src/mx_bluesky/hyperion/parameters/components.py b/src/mx_bluesky/hyperion/parameters/components.py index 2ecbe6d13..e0b092a64 100644 --- a/src/mx_bluesky/hyperion/parameters/components.py +++ b/src/mx_bluesky/hyperion/parameters/components.py @@ -147,6 +147,15 @@ class WithOptionalEnergyChange(BaseModel): class WithVisit(BaseModel): visit: str = Field(min_length=1) + zocalo_environment: str = Field(default=CONST.ZOCALO_ENV) + beamline: str = Field(default=CONST.I03.BEAMLINE, pattern=r"BL\d{2}[BIJS]") + det_dist_to_beam_converter_path: str = Field( + default=CONST.PARAM.DETECTOR.BEAM_XY_LUT_PATH + ) + insertion_prefix: str = Field( + default=CONST.I03.INSERTION_PREFIX, pattern=r"SR\d{2}[BIJS]" + ) + detector_distance_mm: float | None = Field(default=None, gt=0) class DiffractionExperiment( @@ -157,16 +166,7 @@ class DiffractionExperiment( file_name: str exposure_time_s: float = Field(gt=0) comment: str = Field(default="") - beamline: str = Field(default=CONST.I03.BEAMLINE, pattern=r"BL\d{2}[BIJS]") - insertion_prefix: str = Field( - default=CONST.I03.INSERTION_PREFIX, pattern=r"SR\d{2}[BIJS]" - ) - det_dist_to_beam_converter_path: str = Field( - default=CONST.PARAM.DETECTOR.BEAM_XY_LUT_PATH - ) - zocalo_environment: str = Field(default=CONST.ZOCALO_ENV) trigger_mode: TriggerMode = Field(default=TriggerMode.FREE_RUN) - detector_distance_mm: float | None = Field(default=None, gt=0) run_number: int | None = Field(default=None, ge=0) selected_aperture: ApertureValue | None = Field(default=None) transmission_frac: float = Field(default=0.1) diff --git a/src/mx_bluesky/hyperion/parameters/load_centre_collect.py b/src/mx_bluesky/hyperion/parameters/load_centre_collect.py new file mode 100644 index 000000000..f47132655 --- /dev/null +++ b/src/mx_bluesky/hyperion/parameters/load_centre_collect.py @@ -0,0 +1,50 @@ +from typing import TypeVar + +from pydantic import BaseModel, model_validator + +from mx_bluesky.hyperion.parameters.components import ( + HyperionParameters, + WithSample, + WithVisit, +) +from mx_bluesky.hyperion.parameters.gridscan import ( + RobotLoadThenCentre, +) +from mx_bluesky.hyperion.parameters.rotation import MultiRotationScan + +T = TypeVar("T", bound=BaseModel) + + +def construct_from_values(parent_context: dict, key: str, t: type[T]) -> T: + values = dict(parent_context) + values |= values[key] + return t(**values) + + +class LoadCentreCollect(HyperionParameters, WithVisit, WithSample): + """Experiment parameters to perform the combined robot load, + pin-tip centre and rotation scan operations.""" + + robot_load_then_centre: RobotLoadThenCentre + multi_rotation_scan: MultiRotationScan + + @model_validator(mode="before") + @classmethod + def validate_model(cls, values): + allowed_keys = ( + LoadCentreCollect.model_fields.keys() + | RobotLoadThenCentre.model_fields.keys() + | MultiRotationScan.model_fields.keys() + ) + disallowed_keys = values.keys() - allowed_keys + assert ( + disallowed_keys == set() + ), f"Unexpected fields found in LoadCentreCollect {disallowed_keys}" + + values["robot_load_then_centre"] = construct_from_values( + values, "robot_load_then_centre", RobotLoadThenCentre + ) + values["multi_rotation_scan"] = construct_from_values( + values, "multi_rotation_scan", MultiRotationScan + ) + return values diff --git a/tests/conftest.py b/tests/conftest.py index 4c88a84ae..05f4ed0e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,11 +5,13 @@ import sys import threading from collections.abc import Callable, Generator, Sequence +from contextlib import ExitStack from functools import partial from typing import Any from unittest.mock import MagicMock, patch import bluesky.plan_stubs as bps +import numpy import numpy as np import pytest from bluesky.run_engine import RunEngine @@ -41,6 +43,7 @@ from dodal.devices.synchrotron import Synchrotron, SynchrotronMode from dodal.devices.thawer import Thawer from dodal.devices.undulator import Undulator +from dodal.devices.util.test_utils import patch_motor from dodal.devices.util.test_utils import patch_motor as oa_patch_motor from dodal.devices.webcam import Webcam from dodal.devices.xbpm_feedback import XBPMFeedback @@ -280,6 +283,10 @@ def smargon(RE: RunEngine) -> Generator[Smargon, None, None]: set_mock_value(smargon.z.user_readback, 0.0) set_mock_value(smargon.x.high_limit_travel, 2) set_mock_value(smargon.x.low_limit_travel, -2) + set_mock_value(smargon.y.high_limit_travel, 2) + set_mock_value(smargon.y.low_limit_travel, -2) + set_mock_value(smargon.z.high_limit_travel, 2) + set_mock_value(smargon.z.low_limit_travel, -2) with ( patch_async_motor(smargon.omega), @@ -414,7 +421,12 @@ def dcm(RE): dcm = i03.dcm(fake_with_ophyd_sim=True) set_mock_value(dcm.energy_in_kev.user_readback, 12.7) set_mock_value(dcm.pitch_in_mrad.user_readback, 1) - return dcm + with ( + oa_patch_motor(dcm.roll_in_mrad), + oa_patch_motor(dcm.pitch_in_mrad), + oa_patch_motor(dcm.offset_in_mm), + ): + yield dcm @pytest.fixture @@ -423,7 +435,9 @@ def vfm(RE): vfm.bragg_to_lat_lookup_table_path = ( "tests/test_data/test_beamline_vfm_lat_converter.txt" ) - return vfm + with ExitStack() as stack: + stack.enter_context(patch_motor(vfm.x_mm)) + yield vfm @pytest.fixture @@ -441,7 +455,15 @@ def lower_gonio(RE): def vfm_mirror_voltages(): voltages = i03.vfm_mirror_voltages(fake_with_ophyd_sim=True) voltages.voltage_lookup_table_path = "tests/test_data/test_mirror_focus.json" - yield voltages + with ExitStack() as stack: + [ + stack.enter_context(context_mgr) + for context_mgr in [ + patch.object(vc, "set") for vc in voltages.voltage_channels.values() + ] + ] + + yield voltages beamline_utils.clear_devices() @@ -603,7 +625,7 @@ def fake_create_rotation_devices( xbpm_feedback: XBPMFeedback, ): set_mock_value(smargon.omega.max_velocity, 131) - oav.zoom_controller.onst.sim_put("1.0x") # type: ignore + oav.zoom_controller.zrst.sim_put("1.0x") # type: ignore oav.zoom_controller.fvst.sim_put("5.0x") # type: ignore return RotationScanComposite( @@ -913,3 +935,20 @@ def assert_none_matching( predicate: Callable[[Msg], bool], ): assert not list(filter(predicate, messages)) + + +def pin_tip_edge_data(): + tip_x_px = 100 + tip_y_px = 200 + microns_per_pixel = 2.87 # from zoom levels .xml + grid_width_px = int(400 / microns_per_pixel) + target_grid_height_px = 70 + top_edge_data = ([0] * tip_x_px) + ( + [(tip_y_px - target_grid_height_px // 2)] * grid_width_px + ) + bottom_edge_data = [0] * tip_x_px + [ + (tip_y_px + target_grid_height_px // 2) + ] * grid_width_px + top_edge_array = numpy.array(top_edge_data, dtype=numpy.uint32) + bottom_edge_array = numpy.array(bottom_edge_data, dtype=numpy.uint32) + return tip_x_px, tip_y_px, top_edge_array, bottom_edge_array diff --git a/tests/system_tests/conftest.py b/tests/system_tests/conftest.py new file mode 100644 index 000000000..face5c4b9 --- /dev/null +++ b/tests/system_tests/conftest.py @@ -0,0 +1,226 @@ +import re +from decimal import Decimal +from unittest.mock import MagicMock, patch + +import pytest +from dodal.beamlines import i03 +from dodal.devices.oav.oav_parameters import OAVConfigParams +from ophyd_async.core import AsyncStatus, set_mock_value +from requests import Response + +# Map all the case-sensitive column names from their normalised versions +DATA_COLLECTION_COLUMN_MAP = { + s.lower(): s + for s in [ + "dataCollectionId", + "BLSAMPLEID", + "SESSIONID", + "experimenttype", + "dataCollectionNumber", + "startTime", + "endTime", + "runStatus", + "axisStart", + "axisEnd", + "axisRange", + "overlap", + "numberOfImages", + "startImageNumber", + "numberOfPasses", + "exposureTime", + "imageDirectory", + "imagePrefix", + "imageSuffix", + "imageContainerSubPath", + "fileTemplate", + "wavelength", + "resolution", + "detectorDistance", + "xBeam", + "yBeam", + "comments", + "printableForReport", + "CRYSTALCLASS", + "slitGapVertical", + "slitGapHorizontal", + "transmission", + "synchrotronMode", + "xtalSnapshotFullPath1", + "xtalSnapshotFullPath2", + "xtalSnapshotFullPath3", + "xtalSnapshotFullPath4", + "rotationAxis", + "phiStart", + "kappaStart", + "omegaStart", + "chiStart", + "resolutionAtCorner", + "detector2Theta", + "DETECTORMODE", + "undulatorGap1", + "undulatorGap2", + "undulatorGap3", + "beamSizeAtSampleX", + "beamSizeAtSampleY", + "centeringMethod", + "averageTemperature", + "ACTUALSAMPLEBARCODE", + "ACTUALSAMPLESLOTINCONTAINER", + "ACTUALCONTAINERBARCODE", + "ACTUALCONTAINERSLOTINSC", + "actualCenteringPosition", + "beamShape", + "dataCollectionGroupId", + "POSITIONID", + "detectorId", + "FOCALSPOTSIZEATSAMPLEX", + "POLARISATION", + "FOCALSPOTSIZEATSAMPLEY", + "APERTUREID", + "screeningOrigId", + "flux", + "strategySubWedgeOrigId", + "blSubSampleId", + "processedDataFile", + "datFullPath", + "magnification", + "totalAbsorbedDose", + "binning", + "particleDiameter", + "boxSize", + "minResolution", + "minDefocus", + "maxDefocus", + "defocusStepSize", + "amountAstigmatism", + "extractSize", + "bgRadius", + "voltage", + "objAperture", + "c1aperture", + "c2aperture", + "c3aperture", + "c1lens", + "c2lens", + "c3lens", + "startPositionId", + "endPositionId", + "flux", + "bestWilsonPlotPath", + "totalExposedDose", + "nominalMagnification", + "nominalDefocus", + "imageSizeX", + "imageSizeY", + "pixelSizeOnImage", + "phasePlate", + "dataCollectionPlanId", + ] +} + + +@pytest.fixture +def undulator_for_system_test(undulator): + set_mock_value(undulator.current_gap, 1.11) + return undulator + + +@pytest.fixture +def oav_for_system_test(test_config_files): + parameters = OAVConfigParams( + test_config_files["zoom_params_file"], test_config_files["display_config"] + ) + oav = i03.oav(fake_with_ophyd_sim=True, params=parameters) + oav.zoom_controller.zrst.set("1.0x") + oav.zoom_controller.onst.set("7.5x") + oav.cam.array_size.array_size_x.sim_put(1024) + oav.cam.array_size.array_size_y.sim_put(768) + + unpatched_method = oav.parameters.load_microns_per_pixel + + def patch_lmpp(zoom, xsize, ysize): + unpatched_method(zoom, 1024, 768) + + # Grid snapshots + oav.grid_snapshot.x_size.sim_put(1024) # type: ignore + oav.grid_snapshot.y_size.sim_put(768) # type: ignore + oav.grid_snapshot.top_left_x.set(50) + oav.grid_snapshot.top_left_y.set(100) + oav.grid_snapshot.box_width.set(0.1 * 1000 / 1.25) # size in pixels + unpatched_snapshot_trigger = oav.grid_snapshot.trigger + + def mock_grid_snapshot_trigger(): + oav.grid_snapshot.last_path_full_overlay.set("test_1_y") + oav.grid_snapshot.last_path_outer.set("test_2_y") + oav.grid_snapshot.last_saved_path.set("test_3_y") + return unpatched_snapshot_trigger() + + # Plain snapshots + def next_snapshot(): + next_snapshot_idx = 1 + while True: + yield f"/tmp/snapshot{next_snapshot_idx}.png" + next_snapshot_idx += 1 + + empty_response = MagicMock(spec=Response) + empty_response.content = b"" + with ( + patch( + "dodal.devices.areadetector.plugins.MJPG.requests.get", + return_value=empty_response, + ), + patch("dodal.devices.areadetector.plugins.MJPG.Image.open"), + patch.object(oav.grid_snapshot, "post_processing"), + patch.object( + oav.grid_snapshot, "trigger", side_effect=mock_grid_snapshot_trigger + ), + patch.object( + oav.parameters, + "load_microns_per_pixel", + new=MagicMock(side_effect=patch_lmpp), + ), + patch.object(oav.snapshot.last_saved_path, "get") as mock_last_saved_path, + ): + it_next_snapshot = next_snapshot() + + @AsyncStatus.wrap + async def mock_rotation_snapshot_trigger(): + mock_last_saved_path.side_effect = lambda: next(it_next_snapshot) + + with patch.object( + oav.snapshot, + "trigger", + side_effect=mock_rotation_snapshot_trigger, + ): + oav.parameters.load_microns_per_pixel(1.0, 1024, 768) + yield oav + + +def compare_actual_and_expected( + id, expected_values, fetch_datacollection_attribute, column_map: dict | None = None +): + results = "\n" + for k, v in expected_values.items(): + actual = fetch_datacollection_attribute( + id, column_map[k.lower()] if column_map else k + ) + if isinstance(actual, Decimal): + actual = float(actual) + if isinstance(v, float): + actual_v = actual == pytest.approx(v) + else: + actual_v = actual == v + if not actual_v: + results += f"expected {k} {v} == {actual}\n" + assert results == "\n", results + + +def compare_comment( + fetch_datacollection_attribute, data_collection_id, expected_comment +): + actual_comment = fetch_datacollection_attribute( + data_collection_id, DATA_COLLECTION_COLUMN_MAP["comments"] + ) + match = re.search(" Zocalo processing took", actual_comment) + truncated_comment = actual_comment[: match.start()] if match else actual_comment + assert truncated_comment == expected_comment diff --git a/tests/system_tests/hyperion/external_interaction/conftest.py b/tests/system_tests/hyperion/external_interaction/conftest.py index d1a99d397..f03ae9b0f 100644 --- a/tests/system_tests/hyperion/external_interaction/conftest.py +++ b/tests/system_tests/hyperion/external_interaction/conftest.py @@ -1,19 +1,46 @@ import os -from collections.abc import Callable +from collections.abc import Callable, Sequence from functools import partial from typing import Any +from unittest.mock import patch import ispyb.sqlalchemy +import numpy import pytest +from dodal.devices.aperturescatterguard import ApertureScatterguard +from dodal.devices.attenuator import Attenuator +from dodal.devices.backlight import Backlight +from dodal.devices.dcm import DCM +from dodal.devices.detector.detector_motion import DetectorMotion +from dodal.devices.eiger import EigerDetector +from dodal.devices.flux import Flux +from dodal.devices.oav.oav_detector import OAV +from dodal.devices.robot import BartRobot +from dodal.devices.s4_slit_gaps import S4SlitGaps +from dodal.devices.smargon import Smargon +from dodal.devices.synchrotron import Synchrotron, SynchrotronMode +from dodal.devices.undulator import Undulator +from dodal.devices.xbpm_feedback import XBPMFeedback +from dodal.devices.zebra import Zebra +from dodal.devices.zebra_controlled_shutter import ZebraShutter from ispyb.sqlalchemy import DataCollection, DataCollectionGroup, GridInfo, Position +from ophyd.sim import NullStatus +from ophyd_async.core import AsyncStatus, set_mock_value from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from mx_bluesky.hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( + GridDetectThenXRayCentreComposite, +) +from mx_bluesky.hyperion.experiment_plans.rotation_scan_plan import ( + RotationScanComposite, +) from mx_bluesky.hyperion.external_interaction.ispyb.ispyb_store import StoreInIspyb from mx_bluesky.hyperion.parameters.constants import CONST from mx_bluesky.hyperion.parameters.gridscan import ThreeDGridScan +from mx_bluesky.hyperion.utils.utils import convert_angstrom_to_eV -from ....conftest import raw_params_from_file +from ....conftest import fake_read, pin_tip_edge_data, raw_params_from_file TEST_RESULT_LARGE = [ { @@ -62,6 +89,14 @@ def get_current_datacollection_comment(Session: Callable, dcid: int) -> str: return current_comment +def get_datacollections(Session: Callable, dcg_id: int) -> Sequence[int]: + with Session.begin() as session: # type: ignore + query = session.query(DataCollection.dataCollectionId).filter( + DataCollection.dataCollectionGroupId == dcg_id + ) + return [row[0] for row in query.all()] + + def get_current_datacollection_attribute( Session: Callable, dcid: int, attr: str ) -> str: @@ -105,7 +140,7 @@ def get_current_datacollectiongroup_attribute( ): with Session() as session: query = session.query(DataCollectionGroup).filter( - DataCollection.dataCollectionGroupId == dcg_id + DataCollectionGroup.dataCollectionGroupId == dcg_id ) first_result = query.first() return getattr(first_result, attr) @@ -123,6 +158,13 @@ def fetch_comment(sqlalchemy_sessionmaker) -> Callable: return partial(get_current_datacollection_comment, sqlalchemy_sessionmaker) +@pytest.fixture +def fetch_datacollection_ids_for_group_id( + sqlalchemy_sessionmaker, +) -> Callable[[int], Sequence]: + return partial(get_datacollections, sqlalchemy_sessionmaker) + + @pytest.fixture def fetch_datacollection_attribute(sqlalchemy_sessionmaker) -> Callable: return partial(get_current_datacollection_attribute, sqlalchemy_sessionmaker) @@ -167,3 +209,153 @@ def dummy_ispyb_3d(dummy_params) -> StoreInIspyb: @pytest.fixture def zocalo_env(): os.environ["ZOCALO_CONFIG"] = "/dls_sw/apps/zocalo/live/configuration.yaml" + + +@pytest.fixture +def grid_detect_then_xray_centre_composite( + fast_grid_scan, + backlight, + smargon, + undulator_for_system_test, + synchrotron, + s4_slit_gaps, + attenuator, + xbpm_feedback, + detector_motion, + zocalo, + aperture_scatterguard, + zebra, + eiger, + robot, + oav_for_system_test, + dcm, + flux, + ophyd_pin_tip_detection, + sample_shutter, +): + composite = GridDetectThenXRayCentreComposite( + zebra_fast_grid_scan=fast_grid_scan, + pin_tip_detection=ophyd_pin_tip_detection, + backlight=backlight, + panda_fast_grid_scan=None, # type: ignore + smargon=smargon, + undulator=undulator_for_system_test, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + attenuator=attenuator, + xbpm_feedback=xbpm_feedback, + detector_motion=detector_motion, + zocalo=zocalo, + aperture_scatterguard=aperture_scatterguard, + zebra=zebra, + eiger=eiger, + panda=None, # type: ignore + robot=robot, + oav=oav_for_system_test, + dcm=dcm, + flux=flux, + sample_shutter=sample_shutter, + ) + + @AsyncStatus.wrap + async def mock_pin_tip_detect(): + tip_x_px, tip_y_px, top_edge_array, bottom_edge_array = pin_tip_edge_data() + set_mock_value( + ophyd_pin_tip_detection.triggered_top_edge, + top_edge_array, + ) + + set_mock_value( + ophyd_pin_tip_detection.triggered_bottom_edge, + bottom_edge_array, + ) + set_mock_value( + zocalo.bbox_sizes, numpy.array([[10, 10, 10]], dtype=numpy.uint64) + ) + set_mock_value(ophyd_pin_tip_detection.triggered_tip, (tip_x_px, tip_y_px)) + + @AsyncStatus.wrap + async def mock_zocalo_complete(): + await zocalo._put_results(TEST_RESULT_MEDIUM, {"dcid": 0, "dcgid": 0}) + + with ( + patch.object(eiger, "wait_on_arming_if_started"), + # xsize, ysize will always be wrong since computed as 0 before we get here + # patch up load_microns_per_pixel connect to receive non-zero values + patch.object( + ophyd_pin_tip_detection, "trigger", side_effect=mock_pin_tip_detect + ), + patch.object(fast_grid_scan, "kickoff", return_value=NullStatus()), + patch.object(fast_grid_scan, "complete", return_value=NullStatus()), + patch.object(zocalo, "trigger", side_effect=mock_zocalo_complete), + ): + yield composite + + +@pytest.fixture +def composite_for_rotation_scan( + eiger: EigerDetector, + smargon: Smargon, + zebra: Zebra, + detector_motion: DetectorMotion, + backlight: Backlight, + attenuator: Attenuator, + flux: Flux, + undulator_for_system_test: Undulator, + aperture_scatterguard: ApertureScatterguard, + synchrotron: Synchrotron, + s4_slit_gaps: S4SlitGaps, + dcm: DCM, + robot: BartRobot, + oav_for_system_test: OAV, + sample_shutter: ZebraShutter, + xbpm_feedback: XBPMFeedback, +): + set_mock_value(smargon.omega.max_velocity, 131) + oav_for_system_test.zoom_controller.zrst.sim_put("1.0x") # type: ignore + oav_for_system_test.zoom_controller.fvst.sim_put("5.0x") # type: ignore + + fake_create_rotation_devices = RotationScanComposite( + attenuator=attenuator, + backlight=backlight, + dcm=dcm, + detector_motion=detector_motion, + eiger=eiger, + flux=flux, + smargon=smargon, + undulator=undulator_for_system_test, + aperture_scatterguard=aperture_scatterguard, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + zebra=zebra, + robot=robot, + oav=oav_for_system_test, + sample_shutter=sample_shutter, + xbpm_feedback=xbpm_feedback, + ) + + energy_ev = convert_angstrom_to_eV(0.71) + set_mock_value( + fake_create_rotation_devices.dcm.energy_in_kev.user_readback, + energy_ev / 1000, # pyright: ignore + ) + set_mock_value( + fake_create_rotation_devices.synchrotron.synchrotron_mode, + SynchrotronMode.USER, + ) + set_mock_value( + fake_create_rotation_devices.synchrotron.top_up_start_countdown, + -1, + ) + fake_create_rotation_devices.s4_slit_gaps.xgap.user_readback.sim_put( # pyright: ignore + 0.123 + ) + fake_create_rotation_devices.s4_slit_gaps.ygap.user_readback.sim_put( # pyright: ignore + 0.234 + ) + + with ( + patch("bluesky.preprocessors.__read_and_stash_a_motor", fake_read), + patch("bluesky.plan_stubs.wait"), + ): + yield fake_create_rotation_devices diff --git a/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py index 575950bf5..531df412d 100644 --- a/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/hyperion/external_interaction/test_ispyb_dev_connection.py @@ -1,23 +1,15 @@ from __future__ import annotations import os -import re from collections.abc import Callable, Sequence from copy import deepcopy -from decimal import Decimal from typing import Any, Literal -from unittest.mock import MagicMock, patch -import numpy import pytest from bluesky.run_engine import RunEngine -from dodal.devices.oav.oav_detector import OAV from dodal.devices.oav.oav_parameters import OAVParameters from dodal.devices.synchrotron import SynchrotronMode -from ophyd.sim import NullStatus -from ophyd_async.core import AsyncStatus, set_mock_value -from mx_bluesky.hyperion.experiment_plans import oav_grid_detection_plan from mx_bluesky.hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( GridDetectThenXRayCentreComposite, grid_detect_then_xray_centre, @@ -57,9 +49,12 @@ ThreeDGridScan, ) from mx_bluesky.hyperion.parameters.rotation import RotationScan -from mx_bluesky.hyperion.utils.utils import convert_angstrom_to_eV -from ....conftest import fake_read +from ...conftest import ( + DATA_COLLECTION_COLUMN_MAP, + compare_actual_and_expected, + compare_comment, +) from .conftest import raw_params_from_file EXPECTED_DATACOLLECTION_FOR_ROTATION = { @@ -67,122 +62,12 @@ "beamSizeAtSampleX": 0.02, "beamSizeAtSampleY": 0.02, "exposureTime": 0.023, - "undulatorGap1": 1.12, + "undulatorGap1": 1.11, "synchrotronMode": SynchrotronMode.USER.value, "slitGapHorizontal": 0.123, "slitGapVertical": 0.234, } -# Map all the case-sensitive column names from their normalised versions -DATA_COLLECTION_COLUMN_MAP = { - s.lower(): s - for s in [ - "dataCollectionId", - "BLSAMPLEID", - "SESSIONID", - "experimenttype", - "dataCollectionNumber", - "startTime", - "endTime", - "runStatus", - "axisStart", - "axisEnd", - "axisRange", - "overlap", - "numberOfImages", - "startImageNumber", - "numberOfPasses", - "exposureTime", - "imageDirectory", - "imagePrefix", - "imageSuffix", - "imageContainerSubPath", - "fileTemplate", - "wavelength", - "resolution", - "detectorDistance", - "xBeam", - "yBeam", - "comments", - "printableForReport", - "CRYSTALCLASS", - "slitGapVertical", - "slitGapHorizontal", - "transmission", - "synchrotronMode", - "xtalSnapshotFullPath1", - "xtalSnapshotFullPath2", - "xtalSnapshotFullPath3", - "xtalSnapshotFullPath4", - "rotationAxis", - "phiStart", - "kappaStart", - "omegaStart", - "chiStart", - "resolutionAtCorner", - "detector2Theta", - "DETECTORMODE", - "undulatorGap1", - "undulatorGap2", - "undulatorGap3", - "beamSizeAtSampleX", - "beamSizeAtSampleY", - "centeringMethod", - "averageTemperature", - "ACTUALSAMPLEBARCODE", - "ACTUALSAMPLESLOTINCONTAINER", - "ACTUALCONTAINERBARCODE", - "ACTUALCONTAINERSLOTINSC", - "actualCenteringPosition", - "beamShape", - "dataCollectionGroupId", - "POSITIONID", - "detectorId", - "FOCALSPOTSIZEATSAMPLEX", - "POLARISATION", - "FOCALSPOTSIZEATSAMPLEY", - "APERTUREID", - "screeningOrigId", - "flux", - "strategySubWedgeOrigId", - "blSubSampleId", - "processedDataFile", - "datFullPath", - "magnification", - "totalAbsorbedDose", - "binning", - "particleDiameter", - "boxSize", - "minResolution", - "minDefocus", - "maxDefocus", - "defocusStepSize", - "amountAstigmatism", - "extractSize", - "bgRadius", - "voltage", - "objAperture", - "c1aperture", - "c2aperture", - "c3aperture", - "c1lens", - "c2lens", - "c3lens", - "startPositionId", - "endPositionId", - "flux", - "bestWilsonPlotPath", - "totalExposedDose", - "nominalMagnification", - "nominalDefocus", - "imageSizeX", - "imageSizeY", - "pixelSizeOnImage", - "phasePlate", - "dataCollectionPlanId", - ] -} - GRID_INFO_COLUMN_MAP = { s.lower(): s for s in [ @@ -238,133 +123,6 @@ def grid_detect_then_xray_centre_parameters(): return GridScanWithEdgeDetect(**json_dict) -# noinspection PyUnreachableCode -@pytest.fixture -def grid_detect_then_xray_centre_composite( - fast_grid_scan, - backlight, - smargon, - undulator, - synchrotron, - s4_slit_gaps, - attenuator, - xbpm_feedback, - detector_motion, - zocalo, - aperture_scatterguard, - zebra, - eiger, - robot, - oav: OAV, - dcm, - flux, - ophyd_pin_tip_detection, - sample_shutter, - done_status, -): - composite = GridDetectThenXRayCentreComposite( - zebra_fast_grid_scan=fast_grid_scan, - pin_tip_detection=ophyd_pin_tip_detection, - backlight=backlight, - panda_fast_grid_scan=None, # type: ignore - smargon=smargon, - undulator=undulator, - synchrotron=synchrotron, - s4_slit_gaps=s4_slit_gaps, - attenuator=attenuator, - xbpm_feedback=xbpm_feedback, - detector_motion=detector_motion, - zocalo=zocalo, - aperture_scatterguard=aperture_scatterguard, - zebra=zebra, - eiger=eiger, - panda=None, # type: ignore - robot=robot, - oav=oav, - dcm=dcm, - flux=flux, - sample_shutter=sample_shutter, - ) - oav.zoom_controller.zrst.set("1.0x") - oav.cam.array_size.array_size_x.sim_put(1024) # type: ignore - oav.cam.array_size.array_size_y.sim_put(768) # type: ignore - oav.grid_snapshot.x_size.sim_put(1024) # type: ignore - oav.grid_snapshot.y_size.sim_put(768) # type: ignore - oav.grid_snapshot.top_left_x.set(50) - oav.grid_snapshot.top_left_y.set(100) - oav.grid_snapshot.box_width.set(0.1 * 1000 / 1.25) # size in pixels - set_mock_value(undulator.current_gap, 1.11) - - unpatched_method = oav.parameters.load_microns_per_pixel - - unpatched_snapshot_trigger = oav.grid_snapshot.trigger - - def mock_snapshot_trigger(): - oav.grid_snapshot.last_path_full_overlay.set("test_1_y") - oav.grid_snapshot.last_path_outer.set("test_2_y") - oav.grid_snapshot.last_saved_path.set("test_3_y") - return unpatched_snapshot_trigger() - - def patch_lmpp(zoom, xsize, ysize): - unpatched_method(zoom, 1024, 768) - - def mock_pin_tip_detect(_): - tip_x_px = 100 - tip_y_px = 200 - microns_per_pixel = 2.87 # from zoom levels .xml - grid_width_px = int(400 / microns_per_pixel) - target_grid_height_px = 70 - top_edge_data = ([0] * tip_x_px) + ( - [(tip_y_px - target_grid_height_px // 2)] * grid_width_px - ) - bottom_edge_data = [0] * tip_x_px + [ - (tip_y_px + target_grid_height_px // 2) - ] * grid_width_px - set_mock_value( - ophyd_pin_tip_detection.triggered_top_edge, - numpy.array(top_edge_data, dtype=numpy.uint32), - ) - - set_mock_value( - ophyd_pin_tip_detection.triggered_bottom_edge, - numpy.array(bottom_edge_data, dtype=numpy.uint32), - ) - set_mock_value( - zocalo.bbox_sizes, numpy.array([[10, 10, 10]], dtype=numpy.uint64) - ) - - yield from [] - return tip_x_px, tip_y_px - - @AsyncStatus.wrap - async def mock_complete_status(): - pass - - with ( - patch.object(eiger, "wait_on_arming_if_started"), - # xsize, ysize will always be wrong since computed as 0 before we get here - # patch up load_microns_per_pixel connect to receive non-zero values - patch.object( - oav.parameters, - "load_microns_per_pixel", - new=MagicMock(side_effect=patch_lmpp), - ), - patch.object( - oav_grid_detection_plan, - "wait_for_tip_to_be_found", - side_effect=mock_pin_tip_detect, - ), - patch("dodal.devices.areadetector.plugins.MJPG.requests.get"), - patch("dodal.devices.areadetector.plugins.MJPG.Image.open"), - patch.object(oav.grid_snapshot, "post_processing"), - patch.object(oav.grid_snapshot, "trigger", side_effect=mock_snapshot_trigger), - patch.object(fast_grid_scan, "kickoff", return_value=NullStatus()), - patch.object(fast_grid_scan, "complete", return_value=NullStatus()), - patch.object(zocalo, "trigger", return_value=NullStatus()), - ): - yield composite - - def scan_xy_data_info_for_update( data_collection_group_id, dummy_params: ThreeDGridScan, scan_data_info_for_begin ): @@ -428,57 +186,6 @@ def scan_data_infos_for_update_3d( return [scan_xy_data_info_for_update, scan_xz_data_info_for_update] -@pytest.fixture -def composite_for_rotation_scan(fake_create_rotation_devices: RotationScanComposite): - energy_ev = convert_angstrom_to_eV(0.71) - set_mock_value( - fake_create_rotation_devices.dcm.energy_in_kev.user_readback, - energy_ev / 1000, # pyright: ignore - ) - set_mock_value(fake_create_rotation_devices.undulator.current_gap, 1.12) # pyright: ignore - set_mock_value( - fake_create_rotation_devices.synchrotron.synchrotron_mode, - SynchrotronMode.USER, - ) - set_mock_value( - fake_create_rotation_devices.synchrotron.top_up_start_countdown, # pyright: ignore - -1, - ) - fake_create_rotation_devices.s4_slit_gaps.xgap.user_readback.sim_put( # pyright: ignore - 0.123 - ) - fake_create_rotation_devices.s4_slit_gaps.ygap.user_readback.sim_put( # pyright: ignore - 0.234 - ) - it_snapshot_filenames = iter( - [ - "/tmp/snapshot1.png", - "/tmp/snapshot2.png", - "/tmp/snapshot3.png", - "/tmp/snapshot4.png", - ] - ) - - with ( - patch("bluesky.preprocessors.__read_and_stash_a_motor", fake_read), - patch.object( - fake_create_rotation_devices.oav.snapshot.last_saved_path, "get" - ) as mock_last_saved_path, - patch("bluesky.plan_stubs.wait"), - ): - - @AsyncStatus.wrap - async def apply_snapshot_filename(): - mock_last_saved_path.return_value = next(it_snapshot_filenames) - - with patch.object( - fake_create_rotation_devices.oav.snapshot, - "trigger", - side_effect=apply_snapshot_filename, - ): - yield fake_create_rotation_devices - - @pytest.fixture def params_for_rotation_scan(test_rotation_params: RotationScan): test_rotation_params.rotation_increment_deg = 0.27 @@ -755,7 +462,6 @@ def test_ispyb_deposition_in_rotation_plan( RE: RunEngine, fetch_comment: Callable[..., Any], fetch_datacollection_attribute: Callable[..., Any], - fetch_datacollectiongroup_attribute: Callable[..., Any], fetch_datacollection_position_attribute: Callable[..., Any], ): os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG @@ -812,33 +518,3 @@ def generate_scan_data_infos( else: scan_data_infos = [xy_scan_data_info] return scan_data_infos - - -def compare_actual_and_expected( - id, expected_values, fetch_datacollection_attribute, column_map: dict | None = None -): - results = "\n" - for k, v in expected_values.items(): - actual = fetch_datacollection_attribute( - id, column_map[k.lower()] if column_map else k - ) - if isinstance(actual, Decimal): - actual = float(actual) - if isinstance(v, float): - actual_v = actual == pytest.approx(v) - else: - actual_v = actual == v - if not actual_v: - results += f"expected {k} {v} == {actual}\n" - assert results == "\n", results - - -def compare_comment( - fetch_datacollection_attribute, data_collection_id, expected_comment -): - actual_comment = fetch_datacollection_attribute( - data_collection_id, DATA_COLLECTION_COLUMN_MAP["comments"] - ) - match = re.search(" Zocalo processing took", actual_comment) - truncated_comment = actual_comment[: match.start()] if match else actual_comment - assert truncated_comment == expected_comment diff --git a/tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py b/tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py new file mode 100644 index 000000000..f5c6c2abf --- /dev/null +++ b/tests/system_tests/hyperion/external_interaction/test_load_centre_collect_full_plan.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import os +from collections.abc import Callable +from typing import Any +from unittest.mock import MagicMock + +import pytest +from bluesky.run_engine import RunEngine +from dodal.devices.oav.oav_parameters import OAVParameters +from dodal.devices.synchrotron import SynchrotronMode +from ophyd_async.core import set_mock_value + +from mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan import ( + LoadCentreCollectComposite, + load_centre_collect_full_plan, +) +from mx_bluesky.hyperion.external_interaction.callbacks.robot_load.ispyb_callback import ( + RobotLoadISPyBCallback, +) +from mx_bluesky.hyperion.external_interaction.callbacks.rotation.ispyb_callback import ( + RotationISPyBCallback, +) +from mx_bluesky.hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( + GridscanISPyBCallback, +) +from mx_bluesky.hyperion.parameters.constants import CONST +from mx_bluesky.hyperion.parameters.load_centre_collect import LoadCentreCollect + +from ...conftest import ( + DATA_COLLECTION_COLUMN_MAP, + compare_actual_and_expected, + compare_comment, +) +from .conftest import raw_params_from_file + + +@pytest.fixture +def load_centre_collect_params(): + json_dict = raw_params_from_file( + "tests/test_data/parameter_json_files/example_load_centre_collect_params.json" + ) + return LoadCentreCollect(**json_dict) + + +@pytest.fixture +def load_centre_collect_composite( + grid_detect_then_xray_centre_composite, + composite_for_rotation_scan, + thawer, + vfm, + vfm_mirror_voltages, + undulator_dcm, + webcam, + lower_gonio, +): + composite = LoadCentreCollectComposite( + aperture_scatterguard=composite_for_rotation_scan.aperture_scatterguard, + attenuator=composite_for_rotation_scan.attenuator, + backlight=composite_for_rotation_scan.backlight, + dcm=composite_for_rotation_scan.dcm, + detector_motion=composite_for_rotation_scan.detector_motion, + eiger=grid_detect_then_xray_centre_composite.eiger, + flux=composite_for_rotation_scan.flux, + robot=composite_for_rotation_scan.robot, + smargon=composite_for_rotation_scan.smargon, + undulator=composite_for_rotation_scan.undulator, + synchrotron=composite_for_rotation_scan.synchrotron, + s4_slit_gaps=composite_for_rotation_scan.s4_slit_gaps, + sample_shutter=composite_for_rotation_scan.sample_shutter, + zebra=grid_detect_then_xray_centre_composite.zebra, + oav=grid_detect_then_xray_centre_composite.oav, + xbpm_feedback=composite_for_rotation_scan.xbpm_feedback, + zebra_fast_grid_scan=grid_detect_then_xray_centre_composite.zebra_fast_grid_scan, + pin_tip_detection=grid_detect_then_xray_centre_composite.pin_tip_detection, + zocalo=grid_detect_then_xray_centre_composite.zocalo, + panda=grid_detect_then_xray_centre_composite.panda, + panda_fast_grid_scan=grid_detect_then_xray_centre_composite.panda_fast_grid_scan, + thawer=thawer, + vfm=vfm, + vfm_mirror_voltages=vfm_mirror_voltages, + undulator_dcm=undulator_dcm, + webcam=webcam, + lower_gonio=lower_gonio, + ) + + set_mock_value(composite.dcm.bragg_in_degrees.user_readback, 5) + + yield composite + + +GRID_DC_1_EXPECTED_VALUES = { + "BLSAMPLEID": 5461074, + "detectorid": 78, + "axisstart": 0.0, + "axisrange": 0, + "axisend": 0, + "focalspotsizeatsamplex": 0.02, + "focalspotsizeatsampley": 0.02, + "slitgapvertical": 0.234, + "slitgaphorizontal": 0.123, + "beamsizeatsamplex": 0.02, + "beamsizeatsampley": 0.02, + "transmission": 100, + "datacollectionnumber": 1, + "detectordistance": 255.0, + "exposuretime": 0.002, + "imagedirectory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/xraycentring/", + "imageprefix": "robot_load_centring_file", + "imagesuffix": "h5", + "numberofpasses": 1, + "overlap": 0, + "omegastart": 0, + "startimagenumber": 1, + "wavelength": 0.71, + "xbeam": 75.6027, + "ybeam": 79.4935, + "xtalsnapshotfullpath1": "test_1_y", + "xtalsnapshotfullpath2": "test_2_y", + "xtalsnapshotfullpath3": "test_3_y", + "synchrotronmode": "User", + "undulatorgap1": 1.11, + "filetemplate": "robot_load_centring_file_1_master.h5", + "numberofimages": 120, +} + +GRID_DC_2_EXPECTED_VALUES = GRID_DC_1_EXPECTED_VALUES | { + "axisstart": 90, + "axisend": 90, + "omegastart": 90, + "datacollectionnumber": 2, + "filetemplate": "robot_load_centring_file_2_master.h5", + "numberofimages": 90, +} + +ROTATION_DC_EXPECTED_VALUES = { + "axisStart": 10, + "axisEnd": 370, + # "chiStart": 0, mx-bluesky 325 + "wavelength": 0.71, + "beamSizeAtSampleX": 0.02, + "beamSizeAtSampleY": 0.02, + "exposureTime": 0.004, + "undulatorGap1": 1.11, + "synchrotronMode": SynchrotronMode.USER.value, + "slitGapHorizontal": 0.123, + "slitGapVertical": 0.234, + "xtalSnapshotFullPath1": "/tmp/snapshot2.png", + "xtalSnapshotFullPath2": "/tmp/snapshot3.png", + "xtalSnapshotFullPath3": "/tmp/snapshot4.png", + "xtalSnapshotFullPath4": "/tmp/snapshot5.png", +} + +ROTATION_DC_2_EXPECTED_VALUES = ROTATION_DC_EXPECTED_VALUES | { + "xtalSnapshotFullPath1": "/tmp/snapshot6.png", + "xtalSnapshotFullPath2": "/tmp/snapshot7.png", + "xtalSnapshotFullPath3": "/tmp/snapshot8.png", + "xtalSnapshotFullPath4": "/tmp/snapshot9.png", +} + + +@pytest.mark.s03 +def test_execute_load_centre_collect_full_plan( + load_centre_collect_composite: LoadCentreCollectComposite, + load_centre_collect_params: LoadCentreCollect, + oav_parameters_for_rotation: OAVParameters, + RE: RunEngine, + fetch_datacollection_attribute: Callable[..., Any], + fetch_datacollectiongroup_attribute: Callable[..., Any], + fetch_datacollection_ids_for_group_id: Callable[..., Any], +): + os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG + ispyb_gridscan_cb = GridscanISPyBCallback() + ispyb_rotation_cb = RotationISPyBCallback() + robot_load_cb = RobotLoadISPyBCallback() + robot_load_cb.expeye = MagicMock() + robot_load_cb.expeye.start_load.return_value = 1234 + RE.subscribe(ispyb_gridscan_cb) + RE.subscribe(ispyb_rotation_cb) + RE.subscribe(robot_load_cb) + RE( + load_centre_collect_full_plan( + load_centre_collect_composite, + load_centre_collect_params, + oav_parameters_for_rotation, + ) + ) + + assert robot_load_cb.expeye.start_load.called_once_with("cm37235", 4, 5461074, 2, 6) + assert robot_load_cb.expeye.update_barcode_and_snapshots( + 1234, + "BARCODE", + "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/xraycentring/snapshots/160705_webcam_after_load.png", + "/tmp/snapshot1.png", + ) + assert robot_load_cb.expeye.end_load(1234, "success", "OK") + + # Compare gridscan collection + compare_actual_and_expected( + ispyb_gridscan_cb.ispyb_ids.data_collection_group_id, + {"experimentType": "Mesh3D", "blSampleId": 5461074}, + fetch_datacollectiongroup_attribute, + ) + compare_actual_and_expected( + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[0], + GRID_DC_1_EXPECTED_VALUES, + fetch_datacollection_attribute, + DATA_COLLECTION_COLUMN_MAP, + ) + compare_actual_and_expected( + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[1], + GRID_DC_2_EXPECTED_VALUES, + fetch_datacollection_attribute, + DATA_COLLECTION_COLUMN_MAP, + ) + + compare_comment( + fetch_datacollection_attribute, + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[0], + "Hyperion: Xray centring - Diffraction grid scan of 30 by 4 " + "images in 20.0 um by 20.0 um steps. Top left (px): [100,152], " + "bottom right (px): [844,251]. Aperture: ApertureValue.SMALL. ", + ) + compare_comment( + fetch_datacollection_attribute, + ispyb_gridscan_cb.ispyb_ids.data_collection_ids[1], + "Hyperion: Xray centring - Diffraction grid scan of 30 by 3 " + "images in 20.0 um by 20.0 um steps. Top left (px): [100,165], " + "bottom right (px): [844,239]. Aperture: ApertureValue.SMALL. ", + ) + + rotation_dcg_id = ispyb_rotation_cb.ispyb_ids.data_collection_group_id + rotation_dc_ids = fetch_datacollection_ids_for_group_id(rotation_dcg_id) + compare_actual_and_expected( + rotation_dcg_id, + {"experimentType": "SAD", "blSampleId": 5461074}, + fetch_datacollectiongroup_attribute, + ) + compare_actual_and_expected( + rotation_dc_ids[0], + ROTATION_DC_EXPECTED_VALUES, + fetch_datacollection_attribute, + ) + compare_actual_and_expected( + rotation_dc_ids[1], + ROTATION_DC_2_EXPECTED_VALUES, + fetch_datacollection_attribute, + ) + + compare_comment( + fetch_datacollection_attribute, + ispyb_rotation_cb.ispyb_ids.data_collection_ids[0], + "Sample position (µm): (675, 737, -381) Hyperion Rotation Scan - Aperture: ApertureValue.SMALL. ", + ) diff --git a/tests/test_data/parameter_json_files/example_load_centre_collect_params.json b/tests/test_data/parameter_json_files/example_load_centre_collect_params.json new file mode 100644 index 000000000..538b47549 --- /dev/null +++ b/tests/test_data/parameter_json_files/example_load_centre_collect_params.json @@ -0,0 +1,53 @@ +{ + "parameter_model_version": "5.0.0", + "visit": "cm37235-4", + "detector_distance_mm": 255, + "sample_id": 5461074, + "sample_puck": 2, + "sample_pin": 6, + "robot_load_then_centre": { + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/xraycentring", + "file_name": "robot_load_centring_file", + "exposure_time_s": 0.002, + "use_roi_mode": true, + "demand_energy_ev": 11100, + "transmission_frac": 1.0, + "omega_start_deg": 0, + "chi_start_deg": 30, + "use_panda": false + }, + "multi_rotation_scan": { + "comment": "Hyperion Rotation Scan - ", + "file_name": "protk", + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123457/", + "demand_energy_ev": 11200, + "exposure_time_s": 0.004, + "rotation_increment_deg": 0.1, + "snapshot_omegas_deg": [ + 0, + 90, + 180, + 270 + ], + "rotation_scans": [ + { + "omega_start_deg": 10, + "chi_start_deg": 0, + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 360, + "transmission_frac": 0.05, + "ispyb_experiment_type": "SAD" + }, + { + "omega_start_deg": 10, + "chi_start_deg": 30, + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 360, + "transmission_frac": 0.05, + "ispyb_experiment_type": "SAD" + } + ] + } +} diff --git a/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json b/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json new file mode 100644 index 000000000..a0785f033 --- /dev/null +++ b/tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json @@ -0,0 +1,45 @@ +{ + "parameter_model_version": "5.0.0", + "zocalo_environment": "dev_artemis", + "beamline": "BL03S", + "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt", + "insertion_prefix": "SR03S", + "visit": "cm31105-4", + "detector_distance_mm": 255, + "sample_id": 12345, + "sample_puck": 40, + "sample_pin": 3, + "robot_load_then_centre": { + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123458/xraycentring", + "file_name": "robot_load_centring_file", + "comment": "Robot load and centre", + "exposure_time_s": 0.004, + "use_roi_mode": false, + "demand_energy_ev": 11100, + "run_number": 0 + }, + "multi_rotation_scan": { + "comment": "Rotation", + "storage_directory": "/tmp/dls/i03/data/2024/cm31105-4/auto/123458/", + "file_name": "file_name", + "exposure_time_s": 0.004, + "selected_aperture": "SMALL_APERTURE", + "transmission_frac": 1.0, + "demand_energy_ev": 11100, + "rotation_increment_deg": 0.1, + "shutter_opening_time_s": 0.6, + "snapshot_omegas_deg": [0, 90, 180, 270], + "run_number": 1, + "rotation_scans": [{ + "rotation_axis": "omega", + "rotation_direction": "Negative", + "scan_width_deg": 180.0, + "omega_start_deg": 0, + "phi_start_deg": 0.47, + "chi_start_deg": 23.85, + "x_start_um": 1.0, + "y_start_um": 2.0, + "z_start_um": 3.0 + }] + } +} diff --git a/tests/test_data/parameter_json_files/good_test_robot_load_params.json b/tests/test_data/parameter_json_files/good_test_robot_load_params.json index d57b2890a..3d625064e 100644 --- a/tests/test_data/parameter_json_files/good_test_robot_load_params.json +++ b/tests/test_data/parameter_json_files/good_test_robot_load_params.json @@ -4,6 +4,7 @@ "beamline": "BL03S", "insertion_prefix": "SR03S", "snapshot_directory": "/tmp/", + "storage_directory":"/tmp/", "visit": "cm31105-4", "demand_energy_ev": 11100, "sample_id": 12345, diff --git a/tests/unit_tests/hyperion/experiment_plans/conftest.py b/tests/unit_tests/hyperion/experiment_plans/conftest.py index 25d4b0750..7c2dd7065 100644 --- a/tests/unit_tests/hyperion/experiment_plans/conftest.py +++ b/tests/unit_tests/hyperion/experiment_plans/conftest.py @@ -10,8 +10,15 @@ from dodal.devices.synchrotron import SynchrotronMode from dodal.devices.zocalo import ZocaloResults, ZocaloTrigger from event_model import Event -from ophyd_async.core import AsyncStatus, DeviceCollector +from ophyd.sim import NullStatus +from ophyd_async.core import AsyncStatus, DeviceCollector, set_mock_value +from mx_bluesky.hyperion.experiment_plans.robot_load_and_change_energy import ( + RobotLoadAndEnergyChangeComposite, +) +from mx_bluesky.hyperion.experiment_plans.robot_load_then_centre_plan import ( + RobotLoadThenCentreComposite, +) from mx_bluesky.hyperion.external_interaction.callbacks.common.callback_util import ( create_gridscan_callbacks, ) @@ -200,6 +207,90 @@ def simple_beamline(detector_motion, oav, smargon, synchrotron, test_config_file return magic_mock +@pytest.fixture +def robot_load_composite( + smargon, + dcm, + robot, + aperture_scatterguard, + oav, + webcam, + thawer, + lower_gonio, + eiger, + xbpm_feedback, + attenuator, + fast_grid_scan, + undulator, + undulator_dcm, + s4_slit_gaps, + vfm, + vfm_mirror_voltages, + backlight, + detector_motion, + flux, + ophyd_pin_tip_detection, + zocalo, + synchrotron, + sample_shutter, + zebra, + panda, + panda_fast_grid_scan, +) -> RobotLoadThenCentreComposite: + set_mock_value(dcm.energy_in_kev.user_readback, 11.105) + smargon.stub_offsets.set = MagicMock(return_value=NullStatus()) + aperture_scatterguard.set = MagicMock(return_value=NullStatus()) + return RobotLoadThenCentreComposite( + xbpm_feedback=xbpm_feedback, + attenuator=attenuator, + aperture_scatterguard=aperture_scatterguard, + backlight=backlight, + detector_motion=detector_motion, + eiger=eiger, + zebra_fast_grid_scan=fast_grid_scan, + flux=flux, + oav=oav, + pin_tip_detection=ophyd_pin_tip_detection, + smargon=smargon, + synchrotron=synchrotron, + s4_slit_gaps=s4_slit_gaps, + undulator=undulator, + zebra=zebra, + zocalo=zocalo, + panda=panda, + panda_fast_grid_scan=panda_fast_grid_scan, + thawer=thawer, + sample_shutter=sample_shutter, + vfm=vfm, + vfm_mirror_voltages=vfm_mirror_voltages, + dcm=dcm, + undulator_dcm=undulator_dcm, + robot=robot, + webcam=webcam, + lower_gonio=lower_gonio, + ) + + +@pytest.fixture +def robot_load_and_energy_change_composite( + smargon, dcm, robot, aperture_scatterguard, oav, webcam, thawer, lower_gonio, eiger +) -> RobotLoadAndEnergyChangeComposite: + composite: RobotLoadAndEnergyChangeComposite = MagicMock() + composite.smargon = smargon + composite.dcm = dcm + set_mock_value(composite.dcm.energy_in_kev.user_readback, 11.105) + composite.robot = robot + composite.aperture_scatterguard = aperture_scatterguard + composite.smargon.stub_offsets.set = MagicMock(return_value=NullStatus()) + composite.aperture_scatterguard.set = MagicMock(return_value=NullStatus()) + composite.oav = oav + composite.webcam = webcam + composite.lower_gonio = lower_gonio + composite.thawer = thawer + composite.eiger = eiger + return composite + + def assert_event(mock_call, expected): actual = mock_call.args[0] if "data" in actual: diff --git a/tests/unit_tests/hyperion/experiment_plans/test_flyscan_xray_centre_plan.py b/tests/unit_tests/hyperion/experiment_plans/test_flyscan_xray_centre_plan.py index 223add5b1..77071c9a6 100644 --- a/tests/unit_tests/hyperion/experiment_plans/test_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/hyperion/experiment_plans/test_flyscan_xray_centre_plan.py @@ -1,4 +1,3 @@ -import random import types from pathlib import Path from unittest.mock import DEFAULT, MagicMock, call, patch @@ -30,6 +29,7 @@ ) from mx_bluesky.hyperion.exceptions import WarningException from mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan import ( + CrystalNotFoundException, FlyScanXRayCentreComposite, SmargonSpeedException, _get_feature_controlled, @@ -630,7 +630,7 @@ def test_waits_for_motion_program( "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.move_x_y_z", autospec=True, ) - def test_when_gridscan_fails_ispyb_comment_appended_to( + def test_when_gridscan_finds_no_xtal_ispyb_comment_appended_to( self, move_xyz: MagicMock, run_gridscan: MagicMock, @@ -653,11 +653,13 @@ def wrapped_gridscan_and_move(): ) mock_zocalo_trigger(fgs_composite_with_panda_pcap.zocalo, []) - RE( - ispyb_activation_wrapper( - wrapped_gridscan_and_move(), test_fgs_params_panda_zebra + with pytest.raises(CrystalNotFoundException): + RE( + ispyb_activation_wrapper( + wrapped_gridscan_and_move(), test_fgs_params_panda_zebra + ) ) - ) + app_to_comment: MagicMock = ispyb_cb.ispyb.append_to_comment # type:ignore app_to_comment.assert_called() append_aperture_call = app_to_comment.call_args_list[0].args[1] @@ -666,61 +668,26 @@ def wrapped_gridscan_and_move(): assert "Zocalo found no crystals in this gridscan" in append_zocalo_call @patch( - "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.bps.complete", - autospec=True, - ) - @patch( - "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.bps.kickoff", - autospec=True, - ) - @patch( - "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.bps.mv", + "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.run_gridscan", autospec=True, ) @patch( "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.move_x_y_z", autospec=True, ) - @patch( - "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.check_topup_and_wait_if_necessary", - autospec=True, - ) - def test_GIVEN_no_results_from_zocalo_WHEN_communicator_wait_for_results_called_THEN_fallback_centre_used( + def test_when_gridscan_finds_no_xtal_exception_is_raised( self, - mock_topup, move_xyz: MagicMock, - mock_mv: MagicMock, - mock_kickoff: MagicMock, - mock_complete: MagicMock, + run_gridscan: MagicMock, RE_with_subs: ReWithSubs, test_fgs_params_panda_zebra: ThreeDGridScan, fgs_composite_with_panda_pcap: FlyScanXRayCentreComposite, - done_status: Status, ): RE, (nexus_cb, ispyb_cb) = RE_with_subs feature_controlled = _get_feature_controlled( fgs_composite_with_panda_pcap, test_fgs_params_panda_zebra, ) - fgs_composite_with_panda_pcap.eiger.unstage = MagicMock( - return_value=done_status - ) - initial_x_y_z = np.array( - [ - random.uniform(-0.5, 0.5), - random.uniform(-0.5, 0.5), - random.uniform(-0.5, 0.5), - ] - ) - set_mock_value( - fgs_composite_with_panda_pcap.smargon.x.user_readback, initial_x_y_z[0] - ) - set_mock_value( - fgs_composite_with_panda_pcap.smargon.y.user_readback, initial_x_y_z[1] - ) - set_mock_value( - fgs_composite_with_panda_pcap.smargon.z.user_readback, initial_x_y_z[2] - ) def wrapped_gridscan_and_move(): run_generic_ispyb_handler_setup(ispyb_cb, test_fgs_params_panda_zebra) @@ -731,12 +698,12 @@ def wrapped_gridscan_and_move(): ) mock_zocalo_trigger(fgs_composite_with_panda_pcap.zocalo, []) - RE( - ispyb_activation_wrapper( - wrapped_gridscan_and_move(), test_fgs_params_panda_zebra + with pytest.raises(CrystalNotFoundException): + RE( + ispyb_activation_wrapper( + wrapped_gridscan_and_move(), test_fgs_params_panda_zebra + ) ) - ) - assert np.all(move_xyz.call_args[0][1:] == initial_x_y_z) @patch( "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.run_gridscan", @@ -754,17 +721,13 @@ async def test_given_gridscan_fails_to_centre_then_stub_offsets_not_set( fgs_composite_with_panda_pcap: FlyScanXRayCentreComposite, test_fgs_params_panda_zebra: ThreeDGridScan, ): - class MoveException(Exception): - pass - feature_controlled = _get_feature_controlled( fgs_composite_with_panda_pcap, test_fgs_params_panda_zebra, ) mock_zocalo_trigger(fgs_composite_with_panda_pcap.zocalo, []) - move_xyz.side_effect = MoveException() - with pytest.raises(MoveException): + with pytest.raises(CrystalNotFoundException): RE( run_gridscan_and_move( fgs_composite_with_panda_pcap, @@ -963,6 +926,9 @@ def test_flyscan_xray_centre_sets_directory_stages_arms_disarms_unstages_the_pan sim_run_engine.add_read_handler_for( fgs_composite_with_panda_pcap.smargon.x.max_velocity, 10 ) + sim_run_engine.add_read_handler_for( + fgs_composite_with_panda_pcap.zocalo.centres_of_mass, [(10, 10, 10)] + ) msgs = sim_run_engine.simulate_plan( flyscan_xray_centre(fgs_composite_with_panda_pcap, fgs_params_use_panda) @@ -1046,7 +1012,7 @@ def test_fgs_arms_eiger_without_grid_detect( "mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan.check_topup_and_wait_if_necessary", autospec=True, ) - def test_when_grid_scan_fails_then_detector_disarmed_and_correct_exception_returned( + def test_when_grid_scan_fails_with_exception_then_detector_disarmed_and_correct_exception_returned( self, mock_topup, mock_complete, diff --git a/tests/unit_tests/hyperion/experiment_plans/test_load_centre_collect_full_plan.py b/tests/unit_tests/hyperion/experiment_plans/test_load_centre_collect_full_plan.py new file mode 100644 index 000000000..c29567ffb --- /dev/null +++ b/tests/unit_tests/hyperion/experiment_plans/test_load_centre_collect_full_plan.py @@ -0,0 +1,224 @@ +import dataclasses +from unittest.mock import MagicMock, patch + +import pytest +from bluesky.protocols import Location +from dodal.devices.oav.oav_parameters import OAVParameters +from dodal.devices.oav.pin_image_recognition import PinTipDetection +from dodal.devices.synchrotron import SynchrotronMode +from ophyd.sim import NullStatus +from ophyd_async.core import set_mock_value + +from mx_bluesky.hyperion.exceptions import WarningException +from mx_bluesky.hyperion.experiment_plans.flyscan_xray_centre_plan import ( + CrystalNotFoundException, +) +from mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan import ( + LoadCentreCollectComposite, + load_centre_collect_full_plan, +) +from mx_bluesky.hyperion.experiment_plans.robot_load_then_centre_plan import ( + RobotLoadThenCentreComposite, +) +from mx_bluesky.hyperion.experiment_plans.rotation_scan_plan import ( + RotationScanComposite, +) +from mx_bluesky.hyperion.parameters.load_centre_collect import LoadCentreCollect +from mx_bluesky.hyperion.parameters.robot_load import RobotLoadAndEnergyChange +from mx_bluesky.hyperion.parameters.rotation import MultiRotationScan + +from ....conftest import pin_tip_edge_data, raw_params_from_file + + +def find_a_pin(pin_tip_detection): + def set_good_position(): + set_mock_value(pin_tip_detection.triggered_tip, (100, 110)) + return NullStatus() + + return set_good_position + + +@pytest.fixture +def composite( + robot_load_composite, fake_create_rotation_devices, sim_run_engine +) -> LoadCentreCollectComposite: + rlaec_args = { + field.name: getattr(robot_load_composite, field.name) + for field in dataclasses.fields(robot_load_composite) + } + rotation_args = { + field.name: getattr(fake_create_rotation_devices, field.name) + for field in dataclasses.fields(fake_create_rotation_devices) + } + + composite = LoadCentreCollectComposite(**(rlaec_args | rotation_args)) + minaxis = Location(setpoint=-2, readback=-2) + maxaxis = Location(setpoint=2, readback=2) + tip_x_px, tip_y_px, top_edge_array, bottom_edge_array = pin_tip_edge_data() + sim_run_engine.add_handler( + "locate", lambda _: minaxis, "smargon-x-low_limit_travel" + ) + sim_run_engine.add_handler( + "locate", lambda _: minaxis, "smargon-y-low_limit_travel" + ) + sim_run_engine.add_handler( + "locate", lambda _: minaxis, "smargon-z-low_limit_travel" + ) + sim_run_engine.add_handler( + "locate", lambda _: maxaxis, "smargon-x-high_limit_travel" + ) + sim_run_engine.add_handler( + "locate", lambda _: maxaxis, "smargon-y-high_limit_travel" + ) + sim_run_engine.add_handler( + "locate", lambda _: maxaxis, "smargon-z-high_limit_travel" + ) + sim_run_engine.add_read_handler_for( + composite.synchrotron.synchrotron_mode, SynchrotronMode.USER + ) + sim_run_engine.add_read_handler_for( + composite.synchrotron.top_up_start_countdown, -1 + ) + sim_run_engine.add_read_handler_for( + composite.pin_tip_detection.triggered_top_edge, top_edge_array + ) + sim_run_engine.add_read_handler_for( + composite.pin_tip_detection.triggered_bottom_edge, bottom_edge_array + ) + composite.oav.parameters.update_on_zoom(7.5, 1024, 768) + composite.oav.zoom_controller.frst.set("7.5x") + + sim_run_engine.add_read_handler_for( + composite.pin_tip_detection.triggered_tip, (tip_x_px, tip_y_px) + ) + composite.pin_tip_detection.trigger = MagicMock( + side_effect=find_a_pin(composite.pin_tip_detection) + ) + return composite + + +@pytest.fixture +def load_centre_collect_params(): + params = raw_params_from_file( + "tests/test_data/parameter_json_files/good_test_load_centre_collect_params.json" + ) + return LoadCentreCollect(**params) + + +@pytest.fixture +def grid_detection_callback_with_detected_grid(): + with patch( + "mx_bluesky.hyperion.experiment_plans.grid_detect_then_xray_centre_plan.GridDetectionCallback", + autospec=True, + ) as callback: + callback.return_value.get_grid_parameters.return_value = { + "transmission_frac": 1.0, + "exposure_time_s": 0, + "x_start_um": 0, + "y_start_um": 0, + "y2_start_um": 0, + "z_start_um": 0, + "z2_start_um": 0, + "x_steps": 10, + "y_steps": 10, + "z_steps": 10, + "x_step_size_um": 0.1, + "y_step_size_um": 0.1, + "z_step_size_um": 0.1, + } + yield callback + + +@patch( + "mx_bluesky.hyperion.experiment_plans.robot_load_then_centre_plan.pin_centre_then_xray_centre_plan", + return_value=iter([]), +) +@patch( + "mx_bluesky.hyperion.experiment_plans.robot_load_then_centre_plan.robot_load_and_change_energy_plan", + return_value=iter([]), +) +@patch( + "mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan.multi_rotation_scan", + return_value=iter([]), +) +def test_load_centre_collect_full_plan_happy_path_invokes_all_steps( + mock_rotation_scan: MagicMock, + mock_full_robot_load_plan: MagicMock, + mock_pin_centre_then_xray_centre_plan: MagicMock, + composite: LoadCentreCollectComposite, + load_centre_collect_params: LoadCentreCollect, + oav_parameters_for_rotation: OAVParameters, + sim_run_engine, +): + sim_run_engine.simulate_plan( + load_centre_collect_full_plan( + composite, load_centre_collect_params, oav_parameters_for_rotation + ) + ) + + mock_full_robot_load_plan.assert_called_once() + robot_load_energy_change_composite = mock_full_robot_load_plan.mock_calls[0].args[0] + robot_load_energy_change_params = mock_full_robot_load_plan.mock_calls[0].args[1] + assert isinstance(robot_load_energy_change_composite, RobotLoadThenCentreComposite) + assert isinstance(robot_load_energy_change_params, RobotLoadAndEnergyChange) + mock_pin_centre_then_xray_centre_plan.assert_called_once() + mock_rotation_scan.assert_called_once() + rotation_scan_composite = mock_rotation_scan.mock_calls[0].args[0] + rotation_scan_params = mock_rotation_scan.mock_calls[0].args[1] + assert isinstance(rotation_scan_composite, RotationScanComposite) + assert isinstance(rotation_scan_params, MultiRotationScan) + + +@patch( + "mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan.multi_rotation_scan", + return_value=iter([]), +) +@patch( + "mx_bluesky.hyperion.experiment_plans.robot_load_and_change_energy.set_energy_plan", + new=MagicMock(), +) +def test_load_centre_collect_full_plan_skips_collect_if_pin_tip_not_found( + mock_rotation_scan: MagicMock, + composite: LoadCentreCollectComposite, + load_centre_collect_params: LoadCentreCollect, + oav_parameters_for_rotation: OAVParameters, + sim_run_engine, +): + sim_run_engine.add_read_handler_for( + composite.pin_tip_detection.triggered_tip, PinTipDetection.INVALID_POSITION + ) + + with pytest.raises(WarningException, match="Pin tip centring failed"): + sim_run_engine.simulate_plan( + load_centre_collect_full_plan( + composite, load_centre_collect_params, oav_parameters_for_rotation + ) + ) + + mock_rotation_scan.assert_not_called() + + +@patch( + "mx_bluesky.hyperion.experiment_plans.load_centre_collect_full_plan.multi_rotation_scan", + return_value=iter([]), +) +@patch( + "mx_bluesky.hyperion.experiment_plans.robot_load_and_change_energy.set_energy_plan", + new=MagicMock(), +) +def test_load_centre_collect_full_plan_skips_collect_if_no_diffraction( + mock_rotation_scan: MagicMock, + composite: LoadCentreCollectComposite, + load_centre_collect_params: LoadCentreCollect, + oav_parameters_for_rotation: OAVParameters, + sim_run_engine, + grid_detection_callback_with_detected_grid, +): + with pytest.raises(CrystalNotFoundException): + sim_run_engine.simulate_plan( + load_centre_collect_full_plan( + composite, load_centre_collect_params, oav_parameters_for_rotation + ) + ) + + mock_rotation_scan.assert_not_called() diff --git a/tests/unit_tests/hyperion/experiment_plans/test_pin_centre_then_xray_centre_plan.py b/tests/unit_tests/hyperion/experiment_plans/test_pin_centre_then_xray_centre_plan.py index 1169e2976..6e4820ef1 100644 --- a/tests/unit_tests/hyperion/experiment_plans/test_pin_centre_then_xray_centre_plan.py +++ b/tests/unit_tests/hyperion/experiment_plans/test_pin_centre_then_xray_centre_plan.py @@ -124,6 +124,10 @@ def add_handlers_to_simulate_detector_motion(msg: Msg): add_handlers_to_simulate_detector_motion, CONST.WAIT.GRID_READY_FOR_DC ) + sim_run_engine.add_read_handler_for( + simple_beamline.zocalo.centres_of_mass, [10, 10, 10] + ) + messages = sim_run_engine.simulate_plan( pin_tip_centre_then_xray_centre( simple_beamline, diff --git a/tests/unit_tests/hyperion/experiment_plans/test_robot_load_and_change_energy.py b/tests/unit_tests/hyperion/experiment_plans/test_robot_load_and_change_energy.py index a43722b1b..dad4695f7 100644 --- a/tests/unit_tests/hyperion/experiment_plans/test_robot_load_and_change_energy.py +++ b/tests/unit_tests/hyperion/experiment_plans/test_robot_load_and_change_energy.py @@ -27,26 +27,6 @@ from ....conftest import raw_params_from_file -@pytest.fixture -def robot_load_composite( - smargon, dcm, robot, aperture_scatterguard, oav, webcam, thawer, lower_gonio, eiger -) -> RobotLoadAndEnergyChangeComposite: - composite: RobotLoadAndEnergyChangeComposite = MagicMock() - composite.smargon = smargon - composite.dcm = dcm - set_mock_value(composite.dcm.energy_in_kev.user_readback, 11.105) - composite.robot = robot - composite.aperture_scatterguard = aperture_scatterguard - composite.smargon.stub_offsets.set = MagicMock(return_value=NullStatus()) - composite.aperture_scatterguard.set = MagicMock(return_value=NullStatus()) - composite.oav = oav - composite.webcam = webcam - composite.lower_gonio = lower_gonio - composite.thawer = thawer - composite.eiger = eiger - return composite - - @pytest.fixture def robot_load_and_energy_change_params(): params = raw_params_from_file( @@ -70,7 +50,7 @@ def dummy_set_energy_plan(energy, composite): MagicMock(side_effect=dummy_set_energy_plan), ) def test_when_plan_run_with_requested_energy_specified_energy_change_executes( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params: RobotLoadAndEnergyChange, sim_run_engine: RunEngineSimulator, ): @@ -81,7 +61,7 @@ def test_when_plan_run_with_requested_energy_specified_energy_change_executes( ) messages = sim_run_engine.simulate_plan( robot_load_and_change_energy_plan( - robot_load_composite, robot_load_and_energy_change_params + robot_load_and_energy_change_composite, robot_load_and_energy_change_params ) ) assert_message_and_return_remaining( @@ -94,7 +74,7 @@ def test_when_plan_run_with_requested_energy_specified_energy_change_executes( MagicMock(return_value=iter([Msg("set_energy_plan")])), ) def test_robot_load_and_energy_change_doesnt_set_energy_if_not_specified( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params_no_energy: RobotLoadAndEnergyChange, sim_run_engine: RunEngineSimulator, ): @@ -105,7 +85,7 @@ def test_robot_load_and_energy_change_doesnt_set_energy_if_not_specified( ) messages = sim_run_engine.simulate_plan( robot_load_and_change_energy_plan( - robot_load_composite, + robot_load_and_energy_change_composite, robot_load_and_energy_change_params_no_energy, ) ) @@ -147,14 +127,14 @@ def return_not_disabled_after_reads(_): MagicMock(return_value=iter([])), ) def test_given_smargon_disabled_when_plan_run_then_waits_on_smargon( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params: RobotLoadAndEnergyChange, total_disabled_reads: int, sim_run_engine, ): messages = run_simulating_smargon_wait( robot_load_and_energy_change_params, - robot_load_composite, + robot_load_and_energy_change_composite, total_disabled_reads, sim_run_engine, ) @@ -174,14 +154,14 @@ def test_given_smargon_disabled_when_plan_run_then_waits_on_smargon( MagicMock(return_value=iter([])), ) def test_given_smargon_disabled_for_longer_than_timeout_when_plan_run_then_throws_exception( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params: RobotLoadAndEnergyChange, sim_run_engine, ): with pytest.raises(TimeoutError): run_simulating_smargon_wait( robot_load_and_energy_change_params, - robot_load_composite, + robot_load_and_energy_change_composite, 1000, sim_run_engine, ) @@ -225,12 +205,19 @@ def test_given_ispyb_callback_attached_when_robot_load_then_centre_plan_called_t start_load: MagicMock, update_barcode_and_snapshots: MagicMock, end_load: MagicMock, - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params: RobotLoadAndEnergyChange, ): - robot_load_composite.oav.snapshot.last_saved_path.put("test_oav_snapshot") # type: ignore - set_mock_value(robot_load_composite.webcam.last_saved_path, "test_webcam_snapshot") - robot_load_composite.webcam.trigger = MagicMock(return_value=NullStatus()) + robot_load_and_energy_change_composite.oav.snapshot.last_saved_path.put( + "test_oav_snapshot" + ) # type: ignore + set_mock_value( + robot_load_and_energy_change_composite.webcam.last_saved_path, + "test_webcam_snapshot", + ) + robot_load_and_energy_change_composite.webcam.trigger = MagicMock( + return_value=NullStatus() + ) RE = RunEngine() RE.subscribe(RobotLoadISPyBCallback()) @@ -240,7 +227,7 @@ def test_given_ispyb_callback_attached_when_robot_load_then_centre_plan_called_t RE( robot_load_and_change_energy_plan( - robot_load_composite, robot_load_and_energy_change_params + robot_load_and_energy_change_composite, robot_load_and_energy_change_params ) ) @@ -275,7 +262,7 @@ async def test_when_take_snapshots_called_then_filename_and_directory_set_and_de def test_given_lower_gonio_moved_when_robot_load_then_lower_gonio_moved_to_home_and_back( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params_no_energy: RobotLoadAndEnergyChange, sim_run_engine: RunEngineSimulator, ): @@ -291,7 +278,7 @@ def get_read(axis, msg): messages = sim_run_engine.simulate_plan( robot_load_and_change_energy_plan( - robot_load_composite, + robot_load_and_energy_change_composite, robot_load_and_energy_change_params_no_energy, ) ) @@ -318,7 +305,7 @@ def get_read(axis, msg): MagicMock(return_value=iter([])), ) def test_when_plan_run_then_lower_gonio_moved_before_robot_loads_and_back_after_smargon_enabled( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params_no_energy: RobotLoadAndEnergyChange, sim_run_engine: RunEngineSimulator, ): @@ -334,7 +321,7 @@ def get_read(axis, msg): messages = sim_run_engine.simulate_plan( robot_load_and_change_energy_plan( - robot_load_composite, + robot_load_and_energy_change_composite, robot_load_and_energy_change_params_no_energy, ) ) @@ -370,7 +357,7 @@ def get_read(axis, msg): MagicMock(return_value=iter([])), ) def test_when_plan_run_then_thawing_turned_on_for_expected_time( - robot_load_composite: RobotLoadAndEnergyChangeComposite, + robot_load_and_energy_change_composite: RobotLoadAndEnergyChangeComposite, robot_load_and_energy_change_params_no_energy: RobotLoadAndEnergyChange, sim_run_engine: RunEngineSimulator, ): @@ -384,7 +371,7 @@ def test_when_plan_run_then_thawing_turned_on_for_expected_time( messages = sim_run_engine.simulate_plan( robot_load_and_change_energy_plan( - robot_load_composite, + robot_load_and_energy_change_composite, robot_load_and_energy_change_params_no_energy, ) ) diff --git a/tests/unit_tests/hyperion/experiment_plans/test_robot_load_then_centre.py b/tests/unit_tests/hyperion/experiment_plans/test_robot_load_then_centre.py index 8d37e8a0c..12b556735 100644 --- a/tests/unit_tests/hyperion/experiment_plans/test_robot_load_then_centre.py +++ b/tests/unit_tests/hyperion/experiment_plans/test_robot_load_then_centre.py @@ -5,8 +5,6 @@ from bluesky.simulators import RunEngineSimulator, assert_message_and_return_remaining from bluesky.utils import Msg from dodal.devices.robot import SampleLocation -from ophyd.sim import NullStatus -from ophyd_async.core import set_mock_value from mx_bluesky.hyperion.experiment_plans.grid_detect_then_xray_centre_plan import ( GridDetectThenXRayCentreComposite, @@ -23,72 +21,6 @@ from ....conftest import assert_none_matching, raw_params_from_file -@pytest.fixture -def robot_load_composite( - smargon, - dcm, - robot, - aperture_scatterguard, - oav, - webcam, - thawer, - lower_gonio, - eiger, - xbpm_feedback, - flux, - zocalo, - panda, - backlight, - attenuator, - pin_tip, - fast_grid_scan, - detector_motion, - synchrotron, - s4_slit_gaps, - undulator, - zebra, - panda_fast_grid_scan, - vfm, - vfm_mirror_voltages, - undulator_dcm, - sample_shutter, -) -> RobotLoadThenCentreComposite: - composite: RobotLoadThenCentreComposite = RobotLoadThenCentreComposite( - smargon=smargon, - dcm=dcm, - robot=robot, - aperture_scatterguard=aperture_scatterguard, - oav=oav, - webcam=webcam, - lower_gonio=lower_gonio, - thawer=thawer, - eiger=eiger, - xbpm_feedback=xbpm_feedback, - flux=flux, - zocalo=zocalo, - panda=panda, - backlight=backlight, - attenuator=attenuator, - pin_tip_detection=pin_tip, - zebra_fast_grid_scan=fast_grid_scan, - detector_motion=detector_motion, - synchrotron=synchrotron, - s4_slit_gaps=s4_slit_gaps, - undulator=undulator, - zebra=zebra, - panda_fast_grid_scan=panda_fast_grid_scan, - vfm=vfm, - vfm_mirror_voltages=vfm_mirror_voltages, - undulator_dcm=undulator_dcm, - sample_shutter=sample_shutter, - ) - set_mock_value(composite.dcm.energy_in_kev.user_readback, 11.105) - composite.aperture_scatterguard = aperture_scatterguard - composite.smargon.stub_offsets.set = MagicMock(return_value=NullStatus()) - composite.aperture_scatterguard.set = MagicMock(return_value=NullStatus()) - return composite - - @pytest.fixture def robot_load_then_centre_params(): params = raw_params_from_file(