Merge pull request #81 from kif/ID27_Neggia

Id27 neggia

plugins/id27.py

@@ -17,13 +17,16 @@
 import pyFAI
 from dahu.plugin import Plugin
 from dahu.factory import register
+
 lock = Semaphore()
 logger = logging.getLogger("id27")
 
 RAW = "RAW_DATA"
 PROCESSED = "PROCESSED_DATA"
 WORKDIR = "/scratch/shared"
 PREFIX = os.path.dirname(sys.executable) #where there scripts are
+NEGGIA_PLUGIN = os.path.join(os.environ["HOME"], "lib", "dectris-neggia.so")
+XDS_EXE = "xds_par"
 
 try:
     from cryio import crysalis
@@ -313,10 +316,21 @@ def fabio_conversion(file_path,
                      folder="xdi",
                      fabioimage="tifimage",
                      extension="tif",
-                     export_icat=False):
-    "Convert a set of eiger files to cbf or tiff"
+                     export_icat=False,
+                     detector="eiger"):
+    """Convert a set of eiger files to cbf or tiff
+    
+    :param file_path: First par of the input filename, typically /
+    :param scan_number: last part of the path, typically "scan0001". 
+        The actual filename is actually guessed since there can be several of them, all will be processed
+    :param folder: name of the output folder for writing (in PROCESSED_DATA, not in RAW_DATA folder)
+    :param fabioimage: type of file to write, name of the Fabio class name.
+    :param export_icat: set to True to export to icat the result.
+    :param detector: name of the detector, used to determine the input filename
+    :return: the logs of the conversion.
+    """
     results = []
-    filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
+    filename = os.path.join(file_path, scan_number, f'{detector}_????.h5')
     files = {f:fabio.open(f).nframes for f in sorted(glob.glob(filename))} #since python 3.7 dict are ordered !
     all_single = max(files.values())==1
     file_path = file_path.rstrip("/")
@@ -452,8 +466,9 @@ class XdiConversion(Plugin):
     This is the plugin to convert an HDF5 to a stack of TIFF files
        
     Typical JSON file:
-    {"file_path": "/data/id27/inhouse/some/file.h5",
-     "scan_number": "0001"
+    {"file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename
+     "scan_number": "scan0001"
+     "detector": "eiger" #optionnal
     }
     
     """
@@ -465,12 +480,14 @@ def process(self):
 
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
+        detector = self.input.get("detector", "eiger")
         results = fabio_conversion(file_path,
                                    scan_number,
                                    folder="xdi",
                                    fabioimage="tifimage",
                                    extension="tif",
-                                   export_icat=False)
+                                   export_icat=False,
+                                   detector=detector)
         self.output["output"] = results
 
 
@@ -553,8 +570,8 @@ class XdsConversion(Plugin):
     This is the plugin to convert an HDF5 to a stack of CBF files for XDS
        
     Typical JSON file:
-    {"file_path": "/data/id27/inhouse/some/file.h5",
-     "scan_number": "0001"
+    {"file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename
+     "scan_number": "scan0001"
     }
     
     """
@@ -638,7 +655,7 @@ def process(self):
         ai["error_model"] = "poisson"
         ai["application"] = "pyfai-integrate"
         ai["version"] = 3
-        ai["method"] = ["bbox", "csr", "opencl"]
+        ai["method"] = ["full", "csr", "opencl"]
         ai["opencl_device"] = "gpu"
         ai["nbpt_rad"] = self.input.get("npt", 1)
         ai["nbpt_azim"] = 1
@@ -648,8 +665,8 @@ def process(self):
         param["experiment_title"] = os.path.join(os.path.basename(file_path), scan_number)
         param["fast_motor_name"] = "fast"
         param["slow_motor_name"] = "slow"
-        param["fast_motor_points"] = fast_scan
-        param["slow_motor_points"] = slow_scan
+        param["nbpt_fast"] = fast_scan
+        param["nbpt_slow"] = slow_scan
         param["offset"] = 0
         param["output_file"] = dest
         param["input_data"] = [(i, None, None) for i in files]
@@ -672,13 +689,16 @@ def process(self):
         try:
             send_icat(file_path, dest_dir, metadata=metadata)
         except Exception as err:
+            self.log_error(f"Error in send_icat {type(err)}: {err}", do_raise=False)
             import traceback
-            print(f"Error {type(err)}: {err}")
-            traceback.print_exc(err, file=sys.stdout)
+            traceback.print_exc(file=sys.stdout)
 
 
 def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", metadata=None):
     "Function that sends to icat the processed data"
+    if IcatClient is None:
+        logger.warning("pyicat_plus is not installed, skipping")
+        return
     icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
     metadata = metadata or {"definition": "dummy processing", "Sample_name": "unknown sample"}
     l = raw_dir.split("/")
@@ -700,6 +720,96 @@ def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="",
     icat_client.store_processed_data(**kwargs)
     return kwargs
 
+@register
+class XdsProcessing(Plugin):
+    """
+    This is the plugin to convert an HDF5-lima file into a Neggia-compatible and processes it using XDS.
+       
+    Typical JSON file:
+    {"plugin_name": "id27.xdsprocessing",
+     "file_path": "/data/id27/inhouse/some/path", #excluding the scan-number and the filename
+     "scan_number": "scan0001",
+     "ponifile": "/tmp/geometry.poni",
+     "detector": "eiger" #optionnal
+     "xds_extra": ["",
+                   "!UNIT_CELL_CONSTANTS= 10.317 10.317 7.3378 90 90 120 ! put correct values if known",
+                   ...]
+MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT=4 ! default of 6 is sometimes too high
+MAXIMUM_NUMBER_OF_STRONG_PIXELS=18000 ! total number of strong pixels 
+used for indexation
+BACKGROUND_PIXEL=2.0 ! used by COLSPOT and INTEGRATE
+SIGNAL_PIXEL=3.0 ! needs to be lager than BACKGROUND_PIXEL, specifies 
+standard deviation, used in COLSPOT and INTEGRATE
+
+!EXCLUDE_RESOLUTION_RANGE= 3.93 3.87 !ice-ring at 3.897 Angstrom
+!INCLUDE_RESOLUTION_RANGE=40 1.75  ! after CORRECT, insert high resol 
+limit; re-run CORRECT
+
+    }
     
-
-
+    """
+
+    def process(self):
+        Plugin.process(self)
+        if not self.input:
+            logger.error("input is empty")
+
+        script_name = "hdf2neggia"
+        file_path = self.input["file_path"]
+        scan_number = self.input["scan_number"]
+        ponifile = self.input["ponifile"]
+        detector = self.input.get("detector", "eiger")
+        xds_extra = self.input.get("xds_extra")
+        if xds_extra is None:
+                xds_extra = ["",
+                             "!UNIT_CELL_CONSTANTS= 10.317 10.317 7.3378 90 90 120 ! put correct values if known",
+                             "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT=4 ! default of 6 is sometimes too high",
+                             "MAXIMUM_NUMBER_OF_STRONG_PIXELS=18000 ! total number of strong pixels used for indexation",
+                             "BACKGROUND_PIXEL=2.0 ! used by COLSPOT and INTEGRATE",
+                             "SIGNAL_PIXEL=3.0 ! needs to be lager than BACKGROUND_PIXEL, specifies standard deviation, used in COLSPOT and INTEGRATE",
+                             "!EXCLUDE_RESOLUTION_RANGE= 3.93 3.87 !ice-ring at 3.897 Angstrom",
+                             "!INCLUDE_RESOLUTION_RANGE=40 1.75  ! after CORRECT, insert high resol limit; re-run CORRECT",
+                             ""]  
+
+        filename = os.path.join(file_path, scan_number, f'{detector}_????.h5')
+        files = sorted(glob.glob(filename))
+        file_path = file_path.rstrip("/")
+
+        splitted = file_path.split("/")
+        if RAW in splitted:
+            raw_pos = splitted.index(RAW)
+            splitted[raw_pos] = PROCESSED
+            splitted.append(scan_number)
+            splitted.insert(0, "/")
+            dest_dir = os.path.join(*splitted)    
+            # sample_name = splitted[raw_pos + 1]
+        else:
+            dest_dir = os.path.join(file_path.replace(RAW,PROCESSED), scan_number)
+            # sample_name = "unknown sample"
+        dest_dir = os.path.join(dest_dir, "xsd")
+        if len(files) == 0:
+            raise RuntimeError(f"No such file {filename}")
+
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir, exist_ok=True)
+
+        script_name = os.path.join(PREFIX, script_name)
+        parameters = [script_name,
+                      "--geometry", ponifile,
+                      "--output",  dest_dir,
+                      "--neggia", NEGGIA_PLUGIN,
+                      "--CdTe"] + files
+        self.log_warning(f'start script with parameters: `{" ".join(parameters)}`')
+        res = subprocess.run(parameters, capture_output=True, check=False)
+        self.output["convert"] = unpack_processed(res)
+        if res.returncode == 0:
+            # Implement the tuning of the XDS.INP file here...
+            xdsinp = os.path.join(dest_dir, "XDS.INP")
+            with open(xdsinp, "a") as xdsfile:
+                xdsfile.write(os.linesep)
+                for line in xds_extra:
+                    xdsfile.write(line+os.linesep)
+
+            res = subprocess.run(XDS_EXE, cwd=dest_dir, capture_output=True, check=False)
+            self.output["xds"] = unpack_processed(res)
+