From 6582f72edbb81ea2e7407159f5612a120e0b8ca6 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Sep 2024 08:36:14 +0200 Subject: [PATCH 01/12] offer XDI-conversion for pco-camera --- plugins/id27.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index 47a6d19..e40720b 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -313,10 +313,11 @@ def fabio_conversion(file_path, folder="xdi", fabioimage="tifimage", extension="tif", - export_icat=False): + export_icat=False, + detector="eiger"): "Convert a set of eiger files to cbf or tiff" results = [] - filename = os.path.join(file_path, scan_number, 'eiger_????.h5') + filename = os.path.join(file_path, scan_number, f'{detector}_????.h5') files = {f:fabio.open(f).nframes for f in sorted(glob.glob(filename))} #since python 3.7 dict are ordered ! all_single = max(files.values())==1 file_path = file_path.rstrip("/") @@ -452,8 +453,9 @@ class XdiConversion(Plugin): This is the plugin to convert an HDF5 to a stack of TIFF files Typical JSON file: - {"file_path": "/data/id27/inhouse/some/file.h5", - "scan_number": "0001" + {"file_path": "/data/id27/inhouse/some/", #skip scan & detector + "scan_number": "0001", + "detector": "eiger" #optionnal } """ @@ -465,12 +467,14 @@ def process(self): file_path = self.input["file_path"] scan_number = self.input["scan_number"] + detector = self.input.get("detector", "eiger") results = fabio_conversion(file_path, scan_number, folder="xdi", fabioimage="tifimage", extension="tif", - export_icat=False) + export_icat=False, + detector=detector) self.output["output"] = results From 260c64b1331f352c50d3bb92de0f790b791df92c Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Mon, 16 Dec 2024 11:17:51 +0100 Subject: [PATCH 02/12] Start working on the automatic XDS processing. Start with the converion from LIMA-files to Neggia format. --- plugins/id27.py | 73 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index e40720b..545dbc7 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -315,7 +315,17 @@ def fabio_conversion(file_path, extension="tif", export_icat=False, detector="eiger"): - "Convert a set of eiger files to cbf or tiff" + """Convert a set of eiger files to cbf or tiff + + :param file_path: First par of the input filename, typically / + :param scan_number: last part of the path, typically "scan0001". + The actual filename is actually guessed since there can be several of them, all will be processed + :param folder: name of the output folder for writing (in PROCESSED_DATA, not in RAW_DATA folder) + :param fabioimage: type of file to write, name of the Fabio class name. + :param export_icat: set to True to export to icat the result. + :param detector: name of the detector, used to determine the input filename + :return: the logs of the conversion. + """ results = [] filename = os.path.join(file_path, scan_number, f'{detector}_????.h5') files = {f:fabio.open(f).nframes for f in sorted(glob.glob(filename))} #since python 3.7 dict are ordered ! @@ -453,8 +463,8 @@ class XdiConversion(Plugin): This is the plugin to convert an HDF5 to a stack of TIFF files Typical JSON file: - {"file_path": "/data/id27/inhouse/some/", #skip scan & detector - "scan_number": "0001", + {"file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename + "scan_number": "scan0001" "detector": "eiger" #optionnal } @@ -557,8 +567,8 @@ class XdsConversion(Plugin): This is the plugin to convert an HDF5 to a stack of CBF files for XDS Typical JSON file: - {"file_path": "/data/id27/inhouse/some/file.h5", - "scan_number": "0001" + {"file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename + "scan_number": "scan0001" } """ @@ -704,6 +714,59 @@ def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", icat_client.store_processed_data(**kwargs) return kwargs +@register +class XdsProcessing(Plugin): + """ + This is the plugin to convert an HDF5-lima file into a Neggia-compatible and processes it using XDS. + + Typical JSON file: + {"file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename + "scan_number": "scan0001", + "ponifile": "/tmp/geometry.poni", + "detector": "eiger" #optionnal + } + """ + + def process(self): + Plugin.process(self) + if not self.input: + logger.error("input is empty") + + script_name = "hdf2neggia" + file_path = self.input["file_path"] + scan_number = self.input["scan_number"] + ponifile = self.input["ponifile"] + detector = self.input.get("detector", "eiger") + + filename = os.path.join(file_path, scan_number, f'{detector}_????.h5') + files = {f:fabio.open(f).nframes for f in sorted(glob.glob(filename))} #since python 3.7 dict are ordered ! + file_path = file_path.rstrip("/") + + splitted = file_path.split("/") + if RAW in splitted: + raw_pos = splitted.index(RAW) + splitted[raw_pos] = PROCESSED + splitted.append(scan_number) + splitted.insert(0, "/") + dest_dir = os.path.join(*splitted) + sample_name = splitted[raw_pos + 1] + else: + dest_dir = os.path.join(file_path.replace(RAW,PROCESSED), scan_number) + sample_name = "unknown sample" + dest_dir = os.path.join(dest_dir, "xsd") + if len(files) == 0: + raise RuntimeError(f"No such file {filename}") + + if not os.path.exists(dest_dir): + os.makedirs(dest_dir, exist_ok=True) + + script_name = os.path.join(PREFIX, script_name) + parameters = [script_name, + "--geometry", ponifile, + "--output", os.path.join(dest_dir, "master.h5")] + files + logger.info('starts with parameters: %s', parameters) + res = subprocess.run(parameters, capture_output=True, check=False) + From efbe65b083adef50dc948451f6a894ce203e3dbc Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 15:02:07 +0100 Subject: [PATCH 03/12] Fix traceback printing --- plugins/id27.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/id27.py b/plugins/id27.py index 545dbc7..dfed6f1 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -688,7 +688,7 @@ def process(self): except Exception as err: import traceback print(f"Error {type(err)}: {err}") - traceback.print_exc(err, file=sys.stdout) + traceback.print_exc(file=sys.stdout) def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", metadata=None): From e1e3cde2ddf9852a38f76bad8ecd3140c6b216d6 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 15:05:49 +0100 Subject: [PATCH 04/12] log error --- plugins/id27.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/id27.py b/plugins/id27.py index dfed6f1..8ff68bc 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -686,8 +686,8 @@ def process(self): try: send_icat(file_path, dest_dir, metadata=metadata) except Exception as err: + self.log_error(f"Error in send_icat {type(err)}: {err}", do_raise=False) import traceback - print(f"Error {type(err)}: {err}") traceback.print_exc(file=sys.stdout) From 4c392c2c2282ef547756e517e46c31ab82226614 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 15:54:15 +0100 Subject: [PATCH 05/12] Implement the processing of MX data with Neggia --- plugins/id27.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index 8ff68bc..2cd2dfd 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -24,6 +24,8 @@ PROCESSED = "PROCESSED_DATA" WORKDIR = "/scratch/shared" PREFIX = os.path.dirname(sys.executable) #where there scripts are +NEGGIA_PLUGIN = "/mnt/data/ID11/Sucrose/Eiger/sucrose_sx_0/XDS/dectris-neggia.so" #TODO fix +XDS_EXE = "/opt/XDS/xds_par" #TODO fix try: from cryio import crysalis @@ -720,7 +722,8 @@ class XdsProcessing(Plugin): This is the plugin to convert an HDF5-lima file into a Neggia-compatible and processes it using XDS. Typical JSON file: - {"file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename + {"plugin_name": "id27.xdsprocessing", + "file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename "scan_number": "scan0001", "ponifile": "/tmp/geometry.poni", "detector": "eiger" #optionnal @@ -740,7 +743,7 @@ def process(self): detector = self.input.get("detector", "eiger") filename = os.path.join(file_path, scan_number, f'{detector}_????.h5') - files = {f:fabio.open(f).nframes for f in sorted(glob.glob(filename))} #since python 3.7 dict are ordered ! + files = sorted(glob.glob(filename)) file_path = file_path.rstrip("/") splitted = file_path.split("/") @@ -750,10 +753,10 @@ def process(self): splitted.append(scan_number) splitted.insert(0, "/") dest_dir = os.path.join(*splitted) - sample_name = splitted[raw_pos + 1] + # sample_name = splitted[raw_pos + 1] else: dest_dir = os.path.join(file_path.replace(RAW,PROCESSED), scan_number) - sample_name = "unknown sample" + # sample_name = "unknown sample" dest_dir = os.path.join(dest_dir, "xsd") if len(files) == 0: raise RuntimeError(f"No such file {filename}") @@ -764,9 +767,16 @@ def process(self): script_name = os.path.join(PREFIX, script_name) parameters = [script_name, "--geometry", ponifile, - "--output", os.path.join(dest_dir, "master.h5")] + files - logger.info('starts with parameters: %s', parameters) + "--output", dest_dir, + "--neggia", NEGGIA_PLUGIN, + "--CdTe"] + files + self.log_warning(f'start script with parameters: `{" ".join(parameters)}`') res = subprocess.run(parameters, capture_output=True, check=False) + self.output["convert"] = unpack_processed(res) + if res.returncode == 0: + # Implement the tuning of the XDS.INP file here... + res = subprocess.run(XDS_EXE, cwd=dest_dir, capture_output=True, check=False) + self.output["xds"] = unpack_processed(res) From 57b18a1fdb928cc6c99228de022879354279e0b3 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 16:01:27 +0100 Subject: [PATCH 06/12] Addapt to ID27 --- plugins/id27.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index 2cd2dfd..7008956 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -24,8 +24,8 @@ PROCESSED = "PROCESSED_DATA" WORKDIR = "/scratch/shared" PREFIX = os.path.dirname(sys.executable) #where there scripts are -NEGGIA_PLUGIN = "/mnt/data/ID11/Sucrose/Eiger/sucrose_sx_0/XDS/dectris-neggia.so" #TODO fix -XDS_EXE = "/opt/XDS/xds_par" #TODO fix +NEGGIA_PLUGIN = os.path.join(os.environ["HOME"], "lib", "dectris-neggia.so") +XDS_EXE = "xds_par" try: from cryio import crysalis @@ -775,8 +775,8 @@ def process(self): self.output["convert"] = unpack_processed(res) if res.returncode == 0: # Implement the tuning of the XDS.INP file here... + res = subprocess.run(["module","load","xds"],capture_output=True, check=True) + self.output["module"] = unpack_processed(res) res = subprocess.run(XDS_EXE, cwd=dest_dir, capture_output=True, check=False) self.output["xds"] = unpack_processed(res) - - - + \ No newline at end of file From 706d4daa5240338b9773f2aa0ccea90404540c60 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 16:44:05 +0100 Subject: [PATCH 07/12] Use modules from python --- plugins/id27.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index 7008956..f77e6a6 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -17,6 +17,12 @@ import pyFAI from dahu.plugin import Plugin from dahu.factory import register + +# This is a security hole ... but I was told to use modules +exec(open(os.environ["MODULESHOME"] + "/init/python.py").read()) +# Shame should go to the people who enforce the usage of modules +module("load", "xds") + lock = Semaphore() logger = logging.getLogger("id27") @@ -774,9 +780,7 @@ def process(self): res = subprocess.run(parameters, capture_output=True, check=False) self.output["convert"] = unpack_processed(res) if res.returncode == 0: - # Implement the tuning of the XDS.INP file here... - res = subprocess.run(["module","load","xds"],capture_output=True, check=True) - self.output["module"] = unpack_processed(res) + # Implement the tuning of the XDS.INP file here... res = subprocess.run(XDS_EXE, cwd=dest_dir, capture_output=True, check=False) self.output["xds"] = unpack_processed(res) \ No newline at end of file From 38a30e3cc916761dae369623f44ae2430725db09 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 16:46:39 +0100 Subject: [PATCH 08/12] XDS without modules --- plugins/id27.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index f77e6a6..b465171 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -19,9 +19,9 @@ from dahu.factory import register # This is a security hole ... but I was told to use modules -exec(open(os.environ["MODULESHOME"] + "/init/python.py").read()) +# exec(open(os.environ["MODULESHOME"] + "/init/python.py").read()) # Shame should go to the people who enforce the usage of modules -module("load", "xds") +# module("load", "xds") lock = Semaphore() logger = logging.getLogger("id27") From c64b803d24311a62f2bb25bcb600d96cd0d7711a Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 18 Dec 2024 16:49:43 +0100 Subject: [PATCH 09/12] Remove "module" from python --- plugins/id27.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index b465171..0209728 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -18,11 +18,6 @@ from dahu.plugin import Plugin from dahu.factory import register -# This is a security hole ... but I was told to use modules -# exec(open(os.environ["MODULESHOME"] + "/init/python.py").read()) -# Shame should go to the people who enforce the usage of modules -# module("load", "xds") - lock = Semaphore() logger = logging.getLogger("id27") From b6125dad250e4417d7d8b452f269ccba59f91397 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Thu, 19 Dec 2024 09:42:13 +0100 Subject: [PATCH 10/12] Implement extra options for XDS processing --- plugins/id27.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/plugins/id27.py b/plugins/id27.py index 0209728..aa12b7e 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -728,6 +728,20 @@ class XdsProcessing(Plugin): "scan_number": "scan0001", "ponifile": "/tmp/geometry.poni", "detector": "eiger" #optionnal + "xds_extra": ["", + "!UNIT_CELL_CONSTANTS= 10.317 10.317 7.3378 90 90 120 ! put correct values if known", + ...] +MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT=4 ! default of 6 is sometimes too high +MAXIMUM_NUMBER_OF_STRONG_PIXELS=18000 ! total number of strong pixels +used for indexation +BACKGROUND_PIXEL=2.0 ! used by COLSPOT and INTEGRATE +SIGNAL_PIXEL=3.0 ! needs to be lager than BACKGROUND_PIXEL, specifies +standard deviation, used in COLSPOT and INTEGRATE + +!EXCLUDE_RESOLUTION_RANGE= 3.93 3.87 !ice-ring at 3.897 Angstrom +!INCLUDE_RESOLUTION_RANGE=40 1.75  ! after CORRECT, insert high resol +limit; re-run CORRECT + } """ @@ -742,6 +756,17 @@ def process(self): scan_number = self.input["scan_number"] ponifile = self.input["ponifile"] detector = self.input.get("detector", "eiger") + xds_extra = self.input.get("xds_extra") + if xds_extra is None: + xds_extra = ["", + "!UNIT_CELL_CONSTANTS= 10.317 10.317 7.3378 90 90 120 ! put correct values if known", + "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT=4 ! default of 6 is sometimes too high", + "MAXIMUM_NUMBER_OF_STRONG_PIXELS=18000 ! total number of strong pixels used for indexation", + "BACKGROUND_PIXEL=2.0 ! used by COLSPOT and INTEGRATE", + "SIGNAL_PIXEL=3.0 ! needs to be lager than BACKGROUND_PIXEL, specifies standard deviation, used in COLSPOT and INTEGRATE", + "!EXCLUDE_RESOLUTION_RANGE= 3.93 3.87 !ice-ring at 3.897 Angstrom", + "!INCLUDE_RESOLUTION_RANGE=40 1.75  ! after CORRECT, insert high resol limit; re-run CORRECT", + ""] filename = os.path.join(file_path, scan_number, f'{detector}_????.h5') files = sorted(glob.glob(filename)) @@ -775,7 +800,13 @@ def process(self): res = subprocess.run(parameters, capture_output=True, check=False) self.output["convert"] = unpack_processed(res) if res.returncode == 0: - # Implement the tuning of the XDS.INP file here... + # Implement the tuning of the XDS.INP file here... + xdsinp = os.path.join(dest_dir, "XDS.INP") + with open(xdsinp, "a") as xdsfile: + xdsfile.write(os.linesep) + for line in xds_extra: + xdsfile.write(line+os.linesep) + res = subprocess.run(XDS_EXE, cwd=dest_dir, capture_output=True, check=False) self.output["xds"] = unpack_processed(res) \ No newline at end of file From 0a492f593535e7adb45fa063c3590c2a0bfc0303 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Thu, 19 Dec 2024 10:36:28 +0100 Subject: [PATCH 11/12] Use full pixel splitting by default --- plugins/id27.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/id27.py b/plugins/id27.py index aa12b7e..6959db0 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -655,7 +655,7 @@ def process(self): ai["error_model"] = "poisson" ai["application"] = "pyfai-integrate" ai["version"] = 3 - ai["method"] = ["bbox", "csr", "opencl"] + ai["method"] = ["full", "csr", "opencl"] ai["opencl_device"] = "gpu" ai["nbpt_rad"] = self.input.get("npt", 1) ai["nbpt_azim"] = 1 @@ -696,6 +696,9 @@ def process(self): def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", metadata=None): "Function that sends to icat the processed data" + if IcatClient is None: + logger.warning("pyicat_plus is not installed, skipping") + return icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"]) metadata = metadata or {"definition": "dummy processing", "Sample_name": "unknown sample"} l = raw_dir.split("/") From 68e955960f5d329ea441783b6a7241637ba47fc4 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Thu, 19 Dec 2024 10:39:51 +0100 Subject: [PATCH 12/12] Correct deprecation warning --- plugins/id27.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/id27.py b/plugins/id27.py index 6959db0..e9108b3 100644 --- a/plugins/id27.py +++ b/plugins/id27.py @@ -665,8 +665,8 @@ def process(self): param["experiment_title"] = os.path.join(os.path.basename(file_path), scan_number) param["fast_motor_name"] = "fast" param["slow_motor_name"] = "slow" - param["fast_motor_points"] = fast_scan - param["slow_motor_points"] = slow_scan + param["nbpt_fast"] = fast_scan + param["nbpt_slow"] = slow_scan param["offset"] = 0 param["output_file"] = dest param["input_data"] = [(i, None, None) for i in files]