From f19e6ea9550034507400506e4dcf3b2643c5b02b Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Feb 2022 11:43:04 +0100
Subject: [PATCH 01/98] initial plugin for ID27

---
 plugins/id27.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)
 create mode 100644 plugins/id27.py

diff --git a/plugins/id27.py b/plugins/id27.py
new file mode 100644
index 0000000..d1dfe10
--- /dev/null
+++ b/plugins/id27.py
@@ -0,0 +1,144 @@
+import logging
+from dahu.plugin import Plugin
+from dahu.factory import register
+
+logger = logging.getLogger("id15")
+
+import os, re, glob, sys, shutil, collections, io
+from cryio import crysalis
+import subprocess
+import threading
+import multiprocessing
+
+def crysalis_config(calibration_path,calibration_name, number_of_frames,  omega_start, omega_step, center, distance, wave_length, exposure_time):
+    crysalis_files = {
+        'par_file': os.path.join(calibration_path,calibration_name+'.par'),
+        'set_file': '/users/opid27/file_conversion/scan0001.set',
+        'ccd_file': '/users/opid27/file_conversion/scan0001.ccd'}
+
+
+    scans = collections.OrderedDict()
+    scans[0] = [{
+            'count': number_of_frames,
+            'omega': 0,
+            'omega_start': omega_start,
+            'omega_end': omega_start+number_of_frames*omega_step,
+            'pixel_size': 0.075,
+            'omega_runs': None,
+            'theta': 0,
+            'kappa': 0,
+            'phi': 0,
+            'domega': omega_step,
+            'dtheta': 0,
+            'dkappa': 0,
+            'dphi': 0,
+            'center_x': center[0],
+            'center_y': center[1],
+            'alpha': 50,
+            'dist': distance,
+            'l1': wave_length,
+            'l2': wave_length,
+            'l12': wave_length,
+            'b': wave_length,
+            'mono': 0.99,
+            'monotype': 'SYNCHROTRON',
+            'chip': [1024,1024],
+            'Exposure_time': exposure_time,
+            }]
+
+    return crysalis_files, scans
+
+def copy_set_ccd(crysalis_files, crysalis_dir, basename):
+
+    shutil.copy(crysalis_files['set_file'], os.path.join(crysalis_dir,basename+'.set'))
+    shutil.copy(crysalis_files['ccd_file'], os.path.join(crysalis_dir,basename+'.ccd'))
+
+
+def createCrysalis(scans, crysalis_dir, basename):
+    runHeader = crysalis.RunHeader(basename.encode(), crysalis_dir.encode(), 1)
+    runname = os.path.join(crysalis_dir, basename)
+    runFile = []
+
+    for omega_run in scans[0]:
+        dscr = crysalis.RunDscr(0)
+        dscr.axis = crysalis.SCAN_AXIS['OMEGA']
+        dscr.kappa = omega_run['kappa']
+        dscr.omegaphi = 0
+        dscr.start = omega_run['omega_start']
+        dscr.end = omega_run['omega_end']
+        dscr.width = omega_run['domega']
+        dscr.todo = dscr.done = omega_run['count']
+        dscr.exposure = 1
+        runFile.append(dscr)
+
+    crysalis.saveRun(runname, runHeader, runFile)
+    crysalis.saveCrysalisExpSettings(crysalis_dir)
+
+
+def create_par_file(crysalis_files,crysalis_dir, basename):
+
+    new_par = os.path.join(crysalis_dir,basename+'.par')
+
+    with io.open(new_par, 'w',encoding='iso-8859-1') as new_file:
+        with io.open(crysalis_files['par_file'], 'r',encoding='iso-8859-1') as old_file:
+            for line in old_file:
+                if line.startswith("FILE CHIP"):
+                    new_file.write('FILE CHIP "' + basename + '.ccd" \n')
+                else:
+                    new_file.write(line)
+
+
+
+########################################################## Start the server part
+
+def crysalis_conversion(wave_length=None,distance=None,
+                            center=(None,None),
+                            omega_start=None,omega_step=None,
+                            exposure_time=None,
+                            number_of_points=None,
+                            file_source_path=None,
+                            scan_name=None,
+                            calibration_path="",calibration_name="", **kwargs):
+        print('start data convertion with parameters:\n',
+              f'wave_length={wave_length}\n'
+              f'distance={distance}\n',
+              f'center={center}\n',
+              f'omega_start={omega_start}\n',
+              f'omega_step={omega_step}\n',
+              f'file_source_path={file_source_path}\n',
+              f'scan_name={scan_name}')
+
+        script_name = 'eiger2crysalis'
+
+        dirname = os.path.split(file_source_path)
+        parameters = [script_name,
+                      "-w", f'{wave_length}',
+                      "-d", f'{distance}',
+                      "-b", f'{center[0]}', f'{center[1]}',
+                      f"--omega={omega_start}+{omega_step}*index",
+                      file_source_path,
+                      "-o", os.path.join(dirname[0],'esp',f'{scan_name}_1_''{index}.esperanto')]
+        print('starts with parameters:',parameters)
+
+        crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points,  omega_start, omega_step, center, distance, wave_length, exposure_time)
+        crysalis_dir = os.path.join(dirname[0], 'esp')
+        isExist = os.path.exists( crysalis_dir)
+        if not isExist:
+            os.makedirs(crysalis_dir)
+        copy_set_ccd(crysalis_files,crysalis_dir, scan_name )
+        createCrysalis(scans, crysalis_dir, scan_name)
+        create_par_file(crysalis_files,crysalis_dir, scan_name)
+
+        return subprocess.run(parameters)
+
+
+@register
+class CrysalisConversion(Plugin):
+    def process(self):
+        Plugin.process(self)
+        if self.input is None:
+            logger.warning("input is None")
+        result = crysalis_conversion(**self.input)
+        self.log_warning(str(result))
+        
+                
\ No newline at end of file

From 521af63facea88e203b3733c8f12a8007e165655 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Feb 2022 13:06:33 +0100
Subject: [PATCH 02/98] minor rework

---
 plugins/id27.py | 115 +++++++++++++++++++++++++++++++++---------------
 1 file changed, 80 insertions(+), 35 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index d1dfe10..a49fc02 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -1,14 +1,20 @@
+import os
+import shutil
+import collections
+import io
 import logging
 from dahu.plugin import Plugin
 from dahu.factory import register
+import fabio
+logger = logging.getLogger("id27")
 
-logger = logging.getLogger("id15")
-
-import os, re, glob, sys, shutil, collections, io
 from cryio import crysalis
 import subprocess
-import threading
-import multiprocessing
+# import threading
+# import multiprocessing
+
+WORKDIR = "/scratch/shared"
+
 
 def crysalis_config(calibration_path,calibration_name, number_of_frames,  omega_start, omega_step, center, distance, wave_length, exposure_time):
     crysalis_files = {
@@ -75,7 +81,7 @@ def createCrysalis(scans, crysalis_dir, basename):
     crysalis.saveCrysalisExpSettings(crysalis_dir)
 
 
-def create_par_file(crysalis_files,crysalis_dir, basename):
+def create_par_file(crysalis_files, crysalis_dir, basename):
 
     new_par = os.path.join(crysalis_dir,basename+'.par')
 
@@ -88,57 +94,96 @@ def create_par_file(crysalis_files,crysalis_dir, basename):
                     new_file.write(line)
 
 
-
-########################################################## Start the server part
-
-def crysalis_conversion(wave_length=None,distance=None,
-                            center=(None,None),
-                            omega_start=None,omega_step=None,
-                            exposure_time=None,
-                            number_of_points=None,
-                            file_source_path=None,
-                            scan_name=None,
-                            calibration_path="",calibration_name="", **kwargs):
-        print('start data convertion with parameters:\n',
-              f'wave_length={wave_length}\n'
-              f'distance={distance}\n',
-              f'center={center}\n',
-              f'omega_start={omega_start}\n',
-              f'omega_step={omega_step}\n',
-              f'file_source_path={file_source_path}\n',
-              f'scan_name={scan_name}')
+def create_rsync_file(filename):
+    """create 2 files: 
+        * .source with the LIMA-HDF5 file 
+        * a bash script to synchronise back the reduced crysalis dataset
+    
+    :param filename: name of the LIMA-HDF5 file with the images
+    :return: the path of the crysalis directory  
+    """
+    dest_dir = "/".join(filename.split("/")[3:-1]) #strip /data/visitor ... filename.h5
+    dest_dir = os.path.join(WORKDIR, dest_dir)
+    crysalis_dir = os.path.join(dest_dir, 'esp')
+    if not os.path.exists(crysalis_dir):
+            os.makedirs(crysalis_dir)
+    with open(os.path.join(dest_dir, ".source"), "w") as source:
+        source.write(filename)
+    with open(os.path.join(dest_dir, "sync"), "w") as source:
+        source.write(f'#!/bin/sh\nrsync -avx {os.path.join(dest_dir, "esp")} {os.path.dirname(filename)}\n')
+    return crysalis_dir
+
+
+def crysalis_conversion(wave_length=None, distance=None,
+                        center=(None, None),
+                        omega_start=None,omega_step=None,
+                        exposure_time=None,
+                        number_of_points=None,
+                        file_source_path=None,
+                        scan_name=None,
+                        calibration_path="", calibration_name="",
+                        **kwargs):
+        """Run the `eiger2crysalis` script synchronously
+         
+        """
+        logger.info('start data convertion with parameters:\n'+
+                    f'wave_length={wave_length}\n'+
+                    f'distance={distance}\n'+
+                    f'center={center}\n'+
+                    f'omega_start={omega_start}\n'+
+                    f'omega_step={omega_step}\n'+
+                    f'file_source_path={file_source_path}\n'+
+                    f'scan_name={scan_name}')
 
         script_name = 'eiger2crysalis'
 
-        dirname = os.path.split(file_source_path)
+        crysalis_dir = create_rsync_file(file_source_path)
+        
         parameters = [script_name,
                       "-w", f'{wave_length}',
                       "-d", f'{distance}',
                       "-b", f'{center[0]}', f'{center[1]}',
                       f"--omega={omega_start}+{omega_step}*index",
                       file_source_path,
-                      "-o", os.path.join(dirname[0],'esp',f'{scan_name}_1_''{index}.esperanto')]
-        print('starts with parameters:',parameters)
+                      "-o", os.path.join(crysalis_dir, f'{scan_name}_1_''{index}.esperanto')]
+        logger.info('starts with parameters: %s',parameters)
 
         crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points,  omega_start, omega_step, center, distance, wave_length, exposure_time)
-        crysalis_dir = os.path.join(dirname[0], 'esp')
-        isExist = os.path.exists( crysalis_dir)
-        if not isExist:
-            os.makedirs(crysalis_dir)
+
         copy_set_ccd(crysalis_files,crysalis_dir, scan_name )
         createCrysalis(scans, crysalis_dir, scan_name)
         create_par_file(crysalis_files,crysalis_dir, scan_name)
 
         return subprocess.run(parameters)
 
+##########################
+# Start the server part
+##########################
 
 @register
 class CrysalisConversion(Plugin):
+    """
+    This is the basic plugin of cysalis conversion
+       
+    Typical JSON file:
+    {"wave_length": 1.54,
+     "distance": 100,
+     "center": [400,500],
+     "omega_start":-90,
+     "omega_step": 1,
+     "exposure_time":0.1,
+     "number_of_points":180,
+     "file_source_path": "/data/id27/inhouse/sample/example.h5",
+     "scan_name": "scan",
+     "calibration_path":"/data/id27/inhouse/sample/vanadinite",
+     "calibration_name":"vanadinite"
+    }
+    
+    """
+    
     def process(self):
         Plugin.process(self)
         if self.input is None:
             logger.warning("input is None")
         result = crysalis_conversion(**self.input)
-        self.log_warning(str(result))
-        
-                
\ No newline at end of file
+        self.output["results"] = result        

From 549c43d1ec5dae0d72844b50d929f8feb0a351ed Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Feb 2022 15:00:38 +0100
Subject: [PATCH 03/98] Implement all converters. Untested code

---
 plugins/id27.py | 268 +++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 233 insertions(+), 35 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index a49fc02..3c16f32 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -3,12 +3,18 @@
 import collections
 import io
 import logging
+import re
+import glob
 from dahu.plugin import Plugin
 from dahu.factory import register
 import fabio
 logger = logging.getLogger("id27")
 
-from cryio import crysalis
+try:
+    from cryio import crysalis
+except ImportError:
+    crysalis = None
+
 import subprocess
 # import threading
 # import multiprocessing
@@ -123,38 +129,114 @@ def crysalis_conversion(wave_length=None, distance=None,
                         scan_name=None,
                         calibration_path="", calibration_name="",
                         **kwargs):
-        """Run the `eiger2crysalis` script synchronously
-         
-        """
-        logger.info('start data convertion with parameters:\n'+
-                    f'wave_length={wave_length}\n'+
-                    f'distance={distance}\n'+
-                    f'center={center}\n'+
-                    f'omega_start={omega_start}\n'+
-                    f'omega_step={omega_step}\n'+
-                    f'file_source_path={file_source_path}\n'+
-                    f'scan_name={scan_name}')
-
-        script_name = 'eiger2crysalis'
-
-        crysalis_dir = create_rsync_file(file_source_path)
-        
-        parameters = [script_name,
-                      "-w", f'{wave_length}',
-                      "-d", f'{distance}',
-                      "-b", f'{center[0]}', f'{center[1]}',
-                      f"--omega={omega_start}+{omega_step}*index",
-                      file_source_path,
-                      "-o", os.path.join(crysalis_dir, f'{scan_name}_1_''{index}.esperanto')]
-        logger.info('starts with parameters: %s',parameters)
+    """Run the `eiger2crysalis` script synchronously
+     
+    """
+    
+    assert crysalis, "cryio is not installed"
+    
+    logger.info('start data convertion with parameters:\n'+
+                f'wave_length={wave_length}\n'+
+                f'distance={distance}\n'+
+                f'center={center}\n'+
+                f'omega_start={omega_start}\n'+
+                f'omega_step={omega_step}\n'+
+                f'file_source_path={file_source_path}\n'+
+                f'scan_name={scan_name}')
+
+    script_name = 'eiger2crysalis'
+
+    crysalis_dir = create_rsync_file(file_source_path)
+    
+    parameters = [script_name,
+                  "-w", f'{wave_length}',
+                  "-d", f'{distance}',
+                  "-b", f'{center[0]}', f'{center[1]}',
+                  f"--omega={omega_start}+{omega_step}*index",
+                  file_source_path,
+                  "-o", os.path.join(crysalis_dir, f'{scan_name}_1_''{index}.esperanto')]
+    logger.info('starts with parameters: %s',parameters)
+
+    crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points,  omega_start, omega_step, center, distance, wave_length, exposure_time)
 
-        crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points,  omega_start, omega_step, center, distance, wave_length, exposure_time)
+    copy_set_ccd(crysalis_files,crysalis_dir, scan_name )
+    createCrysalis(scans, crysalis_dir, scan_name)
+    create_par_file(crysalis_files,crysalis_dir, scan_name)
 
-        copy_set_ccd(crysalis_files,crysalis_dir, scan_name )
-        createCrysalis(scans, crysalis_dir, scan_name)
-        create_par_file(crysalis_files,crysalis_dir, scan_name)
+    return subprocess.run(parameters)
 
-        return subprocess.run(parameters)
+def crysalis_conversion_fscannd(wave_length=None,
+                                distance=None,
+                                center=(None,None),
+                                omega_start=None,
+                                omega_step=None,
+                                exposure_time=None,
+                                npoints=None,
+                                motor_mode=None,
+                                dirname=None,
+                                scan_name=None,
+                                calibration_path="",
+                                calibration_name=""):
+    assert crysalis, "cryio is not installed"
+    script_name = 'eiger2crysalis'
+    pattern = re.compile('eiger_([0-9]+).h5')
+    filenames_to_convert = glob.glob(f'{dirname}/eiger*.h5')
+    results = {}
+    for filepath in sorted(filenames_to_convert):
+        dname, filename = os.path.split(filepath)
+        g = pattern.match(filename)
+        if g:
+            number = int(g.group(1))
+            if motor_mode == "ZIGZAG" and (number % 2):
+                revert_omega_start=omega_start+(omega_step*npoints)
+                omega_par = f"--omega={revert_omega_start}-{omega_step}*index"
+            else:
+                omega_par = f"--omega={omega_start}+{omega_step}*index"
+
+            parameters = [script_name,
+                          "-w", f'{wave_length}',
+                          "-d", f'{distance}',
+                          "-b", f'{center[0]}', f'{center[1]}',
+                          omega_par,
+                          filepath,
+                          "-o", os.path.join(dirname,f'esp_{number}',
+                                             f'{scan_name}_{number}_1_''{index}.esperanto')]
+            print('starts with parameters:',parameters)
+            results[filename] = str(subprocess.run(parameters))
+            crysalis_files, scans = crysalis_config(calibration_path, calibration_name, npoints,  omega_start, omega_step, center, distance, wave_length, exposure_time)
+            crysalis_folder_name = 'esp_'+str(number)
+            crysalis_dir = os.path.join(dirname, crysalis_folder_name)
+            isExist = os.path.exists( crysalis_dir)
+            if not isExist:
+                os.makedirs(crysalis_dir)
+            crysalis_scan_name = scan_name + '_' + str(number)
+            copy_set_ccd(crysalis_files,crysalis_dir, crysalis_scan_name )
+            createCrysalis(scans, crysalis_dir,  crysalis_scan_name)
+            create_par_file(crysalis_files,crysalis_dir, crysalis_scan_name)
+    return results
+
+def fabio_conversion(file_path,
+                     scan_number,
+                     folder="xdi",
+                     fabioimage="tifimage",
+                     extension="tif"):
+
+        filename = os.path.join(file_path,scan_number,'eiger_0000.h5')
+        img_data_fabio = fabio.open(filename)
+        dest_dir = os.path.join(file_path,scan_number, folder)
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+        results = []
+        for i, frame in enumerate(img_data_fabio):
+            conv = frame.convert(fabioimage)
+            data = conv.data.astype('int32')
+            data[data <0 ] = 0
+            conv.data = data
+            #print('xdi converison', i)
+            output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
+            conv.write(output)
+            results.append(output)
+        return results
 
 ##########################
 # Start the server part
@@ -165,6 +247,35 @@ class CrysalisConversion(Plugin):
     """
     This is the basic plugin of cysalis conversion
        
+    Typical JSON file:
+    {"wave_length": 0.3435,
+     "distance": 200,
+     "center": [1606, 1715],
+     "omega_start":-90,
+     "omega_step": 1,
+     "exposure_time":0.1,
+     "number_of_points": 107,
+     "file_source_path": "/data/visitor/hc4672/id27/CDMX22/CDMX22_0001/scan0021/eiger_0000.h5",
+     "scan_name": "scan",
+     "calibration_path":"/data/id27/inhouse/sample/vanadinite",
+     "calibration_name":"vanadinite"
+    }
+    
+    """
+    
+    def process(self):
+        Plugin.process(self)
+        if not self.input:
+            logger.error("input is empty")
+        result = crysalis_conversion(**self.input)
+        self.output["results"] = str(result)        
+
+
+@register
+class CrysalisConversionFscannd(Plugin):
+    """
+    This is the plugin of cysalis conversion for fscannd type scans
+       
     Typical JSON file:
     {"wave_length": 1.54,
      "distance": 100,
@@ -172,8 +283,9 @@ class CrysalisConversion(Plugin):
      "omega_start":-90,
      "omega_step": 1,
      "exposure_time":0.1,
-     "number_of_points":180,
-     "file_source_path": "/data/id27/inhouse/sample/example.h5",
+     "npoints":180,
+     "motor_mode":"ZIGZAG",
+     "dirname": "/data/id27/inhouse/sample",
      "scan_name": "scan",
      "calibration_path":"/data/id27/inhouse/sample/vanadinite",
      "calibration_name":"vanadinite"
@@ -183,7 +295,93 @@ class CrysalisConversion(Plugin):
     
     def process(self):
         Plugin.process(self)
-        if self.input is None:
-            logger.warning("input is None")
-        result = crysalis_conversion(**self.input)
+        if not self.input:
+            logger.error("input is empty")
+        result = crysalis_conversion_fscannd(**self.input)
         self.output["results"] = result        
+
+@register
+class XdiConversion(Plugin):
+    """
+    This is the plugin to convert an HDF5 to a stack of TIFF files
+       
+    Typical JSON file:
+    {"file_path": "/data/id27/inhouse/some/file.h5",
+     "scan_number": "0001"
+    }
+    
+    """
+    
+    def process(self):
+        Plugin.process(self)
+        if not self.input:
+            logger.error("input is empty")
+        
+        file_path = self.input["file_path"]
+        scan_number = self.input["file_path"]
+        results = fabio_conversion(file_path,
+                         scan_number,
+                         folder="xdi",
+                         fabioimage="tifimage",
+                         extension="tif")
+        self.output["output"] = results        
+
+@register
+class Average(Plugin):
+    """
+    This is the plugin to average out an HDF5 stack of images
+       
+    Typical JSON file:
+    {"file_path": "/data/id27/inhouse/some/file.h5",
+     "scan_number": "0001"
+    }
+    
+    """
+    
+    def process(self):
+        Plugin.process(self)
+        if not self.input:
+            logger.error("input is empty")
+            
+            
+
+        file_path = self.input["file_path"]
+        scan_number = self.input["file_path"]
+        filename = os.path.join(file_path, scan_number,'eiger_0000.h5')
+
+        dest_dir = os.path.join(file_path, scan_number, 'sum')
+
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+
+        output = os.path.join(file_path,scan_number,'sum','sum.edf')
+        command = ['pyFAI-average', '-m', 'sum', '-o', output, filename ]
+        result = subprocess.run(command)
+        
+        self.output["results"] = str(result)        
+
+@register
+class XdsConversion(Plugin):
+    """
+    This is the plugin to convert an HDF5 to a stack of CBF files for XDS
+       
+    Typical JSON file:
+    {"file_path": "/data/id27/inhouse/some/file.h5",
+     "scan_number": "0001"
+    }
+    
+    """
+    
+    def process(self):
+        Plugin.process(self)
+        if not self.input:
+            logger.error("input is empty")
+        
+        file_path = self.input["file_path"]
+        scan_number = self.input["file_path"]
+        results = fabio_conversion(file_path,
+                         scan_number,
+                         folder="cbf",
+                         fabioimage="cbfimage",
+                         extension="cbf")
+        self.output["output"] = results        

From 8873d700d414e1fff6132adc38283f28dc9bdb88 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Feb 2022 16:56:16 +0100
Subject: [PATCH 04/98] Update default json string

---
 plugins/id27.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 3c16f32..4a54d5a 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -248,19 +248,20 @@ class CrysalisConversion(Plugin):
     This is the basic plugin of cysalis conversion
        
     Typical JSON file:
-    {"wave_length": 0.3435,
-     "distance": 200,
-     "center": [1606, 1715],
-     "omega_start":-90,
-     "omega_step": 1,
-     "exposure_time":0.1,
-     "number_of_points": 107,
-     "file_source_path": "/data/visitor/hc4672/id27/CDMX22/CDMX22_0001/scan0021/eiger_0000.h5",
-     "scan_name": "scan",
-     "calibration_path":"/data/id27/inhouse/sample/vanadinite",
-     "calibration_name":"vanadinite"
-    }
-    
+
+{"wave_length": 0.3738,
+ "distance": 196.44,
+ "center": [1560.9227, 1682.7944],
+ "omega_start":-32,
+ "omega_step": 0.5,
+ "exposure_time":0.1,
+ "number_of_points":128,
+ "file_source_path": "/data/id27/inhouse/blc13357/id27/Vanadinite_Weck/Vanadinite_Weck_0001/scan0001/eiger_0000.h5",
+ "scan_name": "scan0001",
+ "calibration_path": "/data/id27/inhouse/blc13357/id27/Vanadinite_Weck/Vanadinite_Weck_0001/scan0001/esp",
+ "calibration_name":"scan0001",
+ "plugin_name": "id27.CrysalisConversion"
+}    
     """
     
     def process(self):

From 96887f8bc1fd36edb5cdc5e20aa4186bb5bfbd00 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 9 Mar 2022 09:47:13 +0100
Subject: [PATCH 05/98] polish

---
 plugins/id27.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 4a54d5a..81d2025 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -319,7 +319,7 @@ def process(self):
             logger.error("input is empty")
         
         file_path = self.input["file_path"]
-        scan_number = self.input["file_path"]
+        scan_number = self.input["scan_number"]
         results = fabio_conversion(file_path,
                          scan_number,
                          folder="xdi",
@@ -333,8 +333,8 @@ class Average(Plugin):
     This is the plugin to average out an HDF5 stack of images
        
     Typical JSON file:
-    {"file_path": "/data/id27/inhouse/some/file.h5",
-     "scan_number": "0001"
+    {"file_path": "/data/id27/inhouse/some/directory",
+     "scan_number": "scan0001"
     }
     
     """
@@ -347,7 +347,7 @@ def process(self):
             
 
         file_path = self.input["file_path"]
-        scan_number = self.input["file_path"]
+        scan_number = self.input["scan_number"]
         filename = os.path.join(file_path, scan_number,'eiger_0000.h5')
 
         dest_dir = os.path.join(file_path, scan_number, 'sum')
@@ -355,7 +355,7 @@ def process(self):
         if not os.path.exists(dest_dir):
             os.makedirs(dest_dir)
 
-        output = os.path.join(file_path,scan_number,'sum','sum.edf')
+        output = os.path.join(dest_dir,'sum.edf')
         command = ['pyFAI-average', '-m', 'sum', '-o', output, filename ]
         result = subprocess.run(command)
         
@@ -379,7 +379,7 @@ def process(self):
             logger.error("input is empty")
         
         file_path = self.input["file_path"]
-        scan_number = self.input["file_path"]
+        scan_number = self.input["scan_number"]
         results = fabio_conversion(file_path,
                          scan_number,
                          folder="cbf",

From cee3fd3d59df88682b3385ad47d7c7ecf81e4823 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 18 Mar 2022 14:51:10 +0100
Subject: [PATCH 06/98] Provide output filename

---
 plugins/id27.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 81d2025..c17774e 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -358,7 +358,7 @@ def process(self):
         output = os.path.join(dest_dir,'sum.edf')
         command = ['pyFAI-average', '-m', 'sum', '-o', output, filename ]
         result = subprocess.run(command)
-        
+        self.output["output_filename"] = output
         self.output["results"] = str(result)        
 
 @register

From a9931df71fa5654e867cd81330da5c84ba0cac01 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 22 Mar 2022 11:33:50 +0100
Subject: [PATCH 07/98] everything but fscannd works.

fscannd requires synchronization
---
 plugins/id27.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index c17774e..b0106a6 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -115,8 +115,10 @@ def create_rsync_file(filename):
             os.makedirs(crysalis_dir)
     with open(os.path.join(dest_dir, ".source"), "w") as source:
         source.write(filename)
-    with open(os.path.join(dest_dir, "sync"), "w") as source:
+    script = os.path.join(dest_dir, "sync")
+    with open(script, "w") as source:
         source.write(f'#!/bin/sh\nrsync -avx {os.path.join(dest_dir, "esp")} {os.path.dirname(filename)}\n')
+    os.chmod(script, 0o755)
     return crysalis_dir
 
 
@@ -165,6 +167,7 @@ def crysalis_conversion(wave_length=None, distance=None,
 
     return subprocess.run(parameters)
 
+
 def crysalis_conversion_fscannd(wave_length=None,
                                 distance=None,
                                 center=(None,None),
@@ -176,7 +179,8 @@ def crysalis_conversion_fscannd(wave_length=None,
                                 dirname=None,
                                 scan_name=None,
                                 calibration_path="",
-                                calibration_name=""):
+                                calibration_name="",
+                                **kwargs):
     assert crysalis, "cryio is not installed"
     script_name = 'eiger2crysalis'
     pattern = re.compile('eiger_([0-9]+).h5')
@@ -215,6 +219,7 @@ def crysalis_conversion_fscannd(wave_length=None,
             create_par_file(crysalis_files,crysalis_dir, crysalis_scan_name)
     return results
 
+
 def fabio_conversion(file_path,
                      scan_number,
                      folder="xdi",
@@ -268,8 +273,13 @@ def process(self):
         Plugin.process(self)
         if not self.input:
             logger.error("input is empty")
+
+        crysalis_dir = create_rsync_file(self.input["file_source_path"])
+        script = os.path.join(os.path.dirname(crysalis_dir), "sync")
         result = crysalis_conversion(**self.input)
-        self.output["results"] = str(result)        
+        self.output["results"] = str(result)   
+        sync_results = subprocess.run([script])
+        self.output["sync"] = str(sync_results)
 
 
 @register

From bb63a0939b42350d75e2cb4cbb9af1c0273a1399 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Mar 2022 14:20:09 +0100
Subject: [PATCH 08/98] sync also fscannd

---
 plugins/id27.py | 75 ++++++++++++++++++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 25 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index b0106a6..beeebb2 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -65,6 +65,14 @@ def copy_set_ccd(crysalis_files, crysalis_dir, basename):
     shutil.copy(crysalis_files['set_file'], os.path.join(crysalis_dir,basename+'.set'))
     shutil.copy(crysalis_files['ccd_file'], os.path.join(crysalis_dir,basename+'.ccd'))
 
+def unpack_CompletedProcess(cp):
+    """Convert a CompletedProcess object in to something which is serialisable
+    
+    :param cp: Return of subprocess.run
+    :return: dict with the same content
+    """
+    return {k: cp.__getattribute__(k) for k in dir(cp) if not (k.startswith("_") or callable(cp.__getattribute__(k)))}
+    
 
 def createCrysalis(scans, crysalis_dir, basename):
     runHeader = crysalis.RunHeader(basename.encode(), crysalis_dir.encode(), 1)
@@ -100,25 +108,35 @@ def create_par_file(crysalis_files, crysalis_dir, basename):
                     new_file.write(line)
 
 
-def create_rsync_file(filename):
+def create_rsync_file(filename, folder="esp"):
     """create 2 files: 
         * .source with the LIMA-HDF5 file 
         * a bash script to synchronise back the reduced crysalis dataset
     
     :param filename: name of the LIMA-HDF5 file with the images
+    :param folder: name of the folder to create. If None, just return the path of the script
     :return: the path of the crysalis directory  
     """
     dest_dir = "/".join(filename.split("/")[3:-1]) #strip /data/visitor ... filename.h5
     dest_dir = os.path.join(WORKDIR, dest_dir)
-    crysalis_dir = os.path.join(dest_dir, 'esp')
+    script = os.path.join(dest_dir, "sync")
+    if folder is None:
+        return script
+
+    crysalis_dir = os.path.join(dest_dir, folder)
     if not os.path.exists(crysalis_dir):
             os.makedirs(crysalis_dir)
-    with open(os.path.join(dest_dir, ".source"), "w") as source:
-        source.write(filename)
-    script = os.path.join(dest_dir, "sync")
-    with open(script, "w") as source:
-        source.write(f'#!/bin/sh\nrsync -avx {os.path.join(dest_dir, "esp")} {os.path.dirname(filename)}\n')
-    os.chmod(script, 0o755)
+    with open(os.path.join(dest_dir, ".source"), "a") as source:
+        source.write(filename+os.linesep)
+    
+    if os.path.exists(script):
+        with open(script, "a") as source:
+            source.write(os.linesep.join([f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(filename)}', ""]))
+    else:
+        with open(script, "w") as source:
+            source.write(os.linesep.join(['#!/bin/sh', f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(filename)}', '']))
+        os.chmod(script, 0o755)
+
     return crysalis_dir
 
 
@@ -165,7 +183,7 @@ def crysalis_conversion(wave_length=None, distance=None,
     createCrysalis(scans, crysalis_dir, scan_name)
     create_par_file(crysalis_files,crysalis_dir, scan_name)
 
-    return subprocess.run(parameters)
+    return subprocess.run(parameters, capture_output=True)
 
 
 def crysalis_conversion_fscannd(wave_length=None,
@@ -186,11 +204,17 @@ def crysalis_conversion_fscannd(wave_length=None,
     pattern = re.compile('eiger_([0-9]+).h5')
     filenames_to_convert = glob.glob(f'{dirname}/eiger*.h5')
     results = {}
+    if filenames_to_convert:
+        crysalis
     for filepath in sorted(filenames_to_convert):
-        dname, filename = os.path.split(filepath)
+        
+        filename = os.path.basename(filepath)
         g = pattern.match(filename)
         if g:
             number = int(g.group(1))
+            crysalis_folder_name = f'esp_{number}'
+            crysalis_dir = create_rsync_file(filepath, crysalis_folder_name)
+            
             if motor_mode == "ZIGZAG" and (number % 2):
                 revert_omega_start=omega_start+(omega_step*npoints)
                 omega_par = f"--omega={revert_omega_start}-{omega_step}*index"
@@ -203,16 +227,12 @@ def crysalis_conversion_fscannd(wave_length=None,
                           "-b", f'{center[0]}', f'{center[1]}',
                           omega_par,
                           filepath,
-                          "-o", os.path.join(dirname,f'esp_{number}',
+                          "-o", os.path.join(crysalis_dir,
                                              f'{scan_name}_{number}_1_''{index}.esperanto')]
-            print('starts with parameters:',parameters)
-            results[filename] = str(subprocess.run(parameters))
+            #print('starts with parameters:',parameters)
+            results[filename] = str(subprocess.run(parameters, capture_output=True))
             crysalis_files, scans = crysalis_config(calibration_path, calibration_name, npoints,  omega_start, omega_step, center, distance, wave_length, exposure_time)
-            crysalis_folder_name = 'esp_'+str(number)
-            crysalis_dir = os.path.join(dirname, crysalis_folder_name)
-            isExist = os.path.exists( crysalis_dir)
-            if not isExist:
-                os.makedirs(crysalis_dir)
+            
             crysalis_scan_name = scan_name + '_' + str(number)
             copy_set_ccd(crysalis_files,crysalis_dir, crysalis_scan_name )
             createCrysalis(scans, crysalis_dir,  crysalis_scan_name)
@@ -274,12 +294,12 @@ def process(self):
         if not self.input:
             logger.error("input is empty")
 
-        crysalis_dir = create_rsync_file(self.input["file_source_path"])
-        script = os.path.join(os.path.dirname(crysalis_dir), "sync")
         result = crysalis_conversion(**self.input)
-        self.output["results"] = str(result)   
-        sync_results = subprocess.run([script])
-        self.output["sync"] = str(sync_results)
+        self.output["results"] = unpack_CompletedProcess(result)   
+
+        script = create_rsync_file(self.input["file_source_path"], None)
+        sync_results = subprocess.run([script], capture_output=True)
+        self.output["sync"] = unpack_CompletedProcess(sync_results)
 
 
 @register
@@ -309,7 +329,12 @@ def process(self):
         if not self.input:
             logger.error("input is empty")
         result = crysalis_conversion_fscannd(**self.input)
-        self.output["results"] = result        
+        self.output["results"] = {k: unpack_CompletedProcess(v) for k,v in result.items()}
+        random_filename = glob.glob(f'{self.input["dirname"]}/eiger*.h5')[0]        
+        script = create_rsync_file(random_filename, None)
+        sync_results = subprocess.run([script], capture_output=True)
+        self.output["sync"] = unpack_CompletedProcess(sync_results)
+
 
 @register
 class XdiConversion(Plugin):
@@ -367,7 +392,7 @@ def process(self):
 
         output = os.path.join(dest_dir,'sum.edf')
         command = ['pyFAI-average', '-m', 'sum', '-o', output, filename ]
-        result = subprocess.run(command)
+        result = subprocess.run(command, capture_output=True)
         self.output["output_filename"] = output
         self.output["results"] = str(result)        
 

From f0a3f3fe3c5f9c52444da915036868c4b10463f1 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Mar 2022 14:30:05 +0100
Subject: [PATCH 09/98] fix JSON serialization

---
 plugins/id27.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index beeebb2..979e835 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -60,18 +60,31 @@ def crysalis_config(calibration_path,calibration_name, number_of_frames,  omega_
 
     return crysalis_files, scans
 
+
 def copy_set_ccd(crysalis_files, crysalis_dir, basename):
 
     shutil.copy(crysalis_files['set_file'], os.path.join(crysalis_dir,basename+'.set'))
     shutil.copy(crysalis_files['ccd_file'], os.path.join(crysalis_dir,basename+'.ccd'))
 
+
 def unpack_CompletedProcess(cp):
     """Convert a CompletedProcess object in to something which is serialisable
     
     :param cp: Return of subprocess.run
     :return: dict with the same content
     """
-    return {k: cp.__getattribute__(k) for k in dir(cp) if not (k.startswith("_") or callable(cp.__getattribute__(k)))}
+    res = {}
+    for k in dir(cp):
+        
+        if k.startswith("_"):
+            continue
+        v = cp.__getattribute__(k)
+        if callable(v):
+            continue
+        if "decode" in dir(v):
+            v = v.decode()
+        res[k] = v
+    return res
     
 
 def createCrysalis(scans, crysalis_dir, basename):

From c6a7e8573ddd18abdee1f940d6be9f77c1bbb793 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Mar 2022 14:33:20 +0100
Subject: [PATCH 10/98] export averagaed data as JSON-serializable

---
 plugins/id27.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 979e835..589f2ed 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -385,14 +385,11 @@ class Average(Plugin):
      "scan_number": "scan0001"
     }
     
-    """
-    
+    """    
     def process(self):
         Plugin.process(self)
         if not self.input:
             logger.error("input is empty")
-            
-            
 
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
@@ -407,7 +404,8 @@ def process(self):
         command = ['pyFAI-average', '-m', 'sum', '-o', output, filename ]
         result = subprocess.run(command, capture_output=True)
         self.output["output_filename"] = output
-        self.output["results"] = str(result)        
+        self.output["conversion"] = unpack_CompletedProcess(result)        
+
 
 @register
 class XdsConversion(Plugin):

From 5ef9202ea34457b092ee93b04db78e635139b6f1 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Mar 2022 14:38:18 +0100
Subject: [PATCH 11/98] collect parameters

---
 plugins/id27.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 589f2ed..776a40c 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -243,7 +243,7 @@ def crysalis_conversion_fscannd(wave_length=None,
                           "-o", os.path.join(crysalis_dir,
                                              f'{scan_name}_{number}_1_''{index}.esperanto')]
             #print('starts with parameters:',parameters)
-            results[filename] = str(subprocess.run(parameters, capture_output=True))
+            results[filename] = subprocess.run(parameters, capture_output=True)
             crysalis_files, scans = crysalis_config(calibration_path, calibration_name, npoints,  omega_start, omega_step, center, distance, wave_length, exposure_time)
             
             crysalis_scan_name = scan_name + '_' + str(number)

From d9f111170b3f7554415774e042a87979b2d246a7 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 22 Mar 2022 14:42:17 +0100
Subject: [PATCH 12/98] use same name pattern as eiger files

---
 plugins/id27.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 776a40c..ba5073c 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -225,7 +225,7 @@ def crysalis_conversion_fscannd(wave_length=None,
         g = pattern.match(filename)
         if g:
             number = int(g.group(1))
-            crysalis_folder_name = f'esp_{number}'
+            crysalis_folder_name = f'esp_{number:04d}'
             crysalis_dir = create_rsync_file(filepath, crysalis_folder_name)
             
             if motor_mode == "ZIGZAG" and (number % 2):

From 1d3b5156fd62ac62cdf2c25e688e8b1e9ca80022 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 29 Mar 2022 11:40:01 +0200
Subject: [PATCH 13/98] support several input files for each conversions

---
 plugins/id27.py | 81 +++++++++++++++++++++++++++++++++++--------------
 1 file changed, 59 insertions(+), 22 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index ba5073c..70ed8c5 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -22,9 +22,24 @@
 WORKDIR = "/scratch/shared"
 
 
-def crysalis_config(calibration_path,calibration_name, number_of_frames,  omega_start, omega_step, center, distance, wave_length, exposure_time):
+def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega_start, omega_step, center, distance, wave_length, exposure_time):
+    """
+    :param calibration_path: typically "/users/opid27/file_conversion/"
+    :param calibration_name: typically "scan0001"
+    :param number_of_frames: integrer
+    :param omega_start: in deg ?
+    :param omega_step: in deg ?
+    :param center: in pixel ?
+    :param distance: in mm ?
+    :param wave_length: in angstrom
+    :param exposure_time: in seconds
+    :return 2 dicts, one containing the path crysalis_files, the second with scans
+    """
+    if not (calibration_path and calibration_name) :
+        calibration_path = "/users/opid27/file_conversion/"
+        calibration_name = "scan0001"
     crysalis_files = {
-        'par_file': os.path.join(calibration_path,calibration_name+'.par'),
+        'par_file': os.path.join(calibration_path, calibration_name+'.par'),
         'set_file': '/users/opid27/file_conversion/scan0001.set',
         'ccd_file': '/users/opid27/file_conversion/scan0001.ccd'}
 
@@ -110,10 +125,10 @@ def createCrysalis(scans, crysalis_dir, basename):
 
 def create_par_file(crysalis_files, crysalis_dir, basename):
 
-    new_par = os.path.join(crysalis_dir,basename+'.par')
+    new_par = os.path.join(crysalis_dir, basename+'.par')
 
-    with io.open(new_par, 'w',encoding='iso-8859-1') as new_file:
-        with io.open(crysalis_files['par_file'], 'r',encoding='iso-8859-1') as old_file:
+    with io.open(new_par, 'w', encoding='iso-8859-1') as new_file:
+        with io.open(crysalis_files['par_file'], 'r', encoding='iso-8859-1') as old_file:
             for line in old_file:
                 if line.startswith("FILE CHIP"):
                     new_file.write('FILE CHIP "' + basename + '.ccd" \n')
@@ -192,7 +207,7 @@ def crysalis_conversion(wave_length=None, distance=None,
 
     crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points,  omega_start, omega_step, center, distance, wave_length, exposure_time)
 
-    copy_set_ccd(crysalis_files,crysalis_dir, scan_name )
+    copy_set_ccd(crysalis_files, crysalis_dir, scan_name )
     createCrysalis(scans, crysalis_dir, scan_name)
     create_par_file(crysalis_files,crysalis_dir, scan_name)
 
@@ -259,21 +274,42 @@ def fabio_conversion(file_path,
                      fabioimage="tifimage",
                      extension="tif"):
 
-        filename = os.path.join(file_path,scan_number,'eiger_0000.h5')
-        img_data_fabio = fabio.open(filename)
-        dest_dir = os.path.join(file_path,scan_number, folder)
-        if not os.path.exists(dest_dir):
-            os.makedirs(dest_dir)
         results = []
-        for i, frame in enumerate(img_data_fabio):
-            conv = frame.convert(fabioimage)
-            data = conv.data.astype('int32')
-            data[data <0 ] = 0
-            conv.data = data
-            #print('xdi converison', i)
-            output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
-            conv.write(output)
-            results.append(output)
+        filename = os.path.join(file_path, scan_number,'eiger_????.h5')
+        files = sorted(glob.glob(filename))
+        
+        if len(files) == 0:
+            raise RuntimeError(f"No such file {filename}")
+        
+        elif len(files) == 1:
+            filename = files[0]
+            img_data_fabio = fabio.open(filename)
+            dest_dir = os.path.join(file_path, scan_number, folder)
+            if not os.path.exists(dest_dir):
+                os.makedirs(dest_dir)
+            for i, frame in enumerate(img_data_fabio):
+                conv = frame.convert(fabioimage)
+                data = conv.data.astype('int32')
+                data[data <0 ] = 0
+                conv.data = data
+                output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
+                conv.write(output)
+                results.append(output)
+        else:
+            for idx_file, filename in enumerate(files):
+                img_data_fabio = fabio.open(filename)
+                dest_dir = os.path.join(file_path, scan_number, "{folder}_{idx_file+1:04d}")
+                if not os.path.exists(dest_dir):
+                    os.makedirs(dest_dir)                
+                for i, frame in enumerate(img_data_fabio):
+                    conv = frame.convert(fabioimage)
+                    data = conv.data.astype('int32')
+                    data[data <0 ] = 0
+                    conv.data = data
+                    output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
+                    conv.write(output)
+                    results.append(output)
+            
         return results
 
 ##########################
@@ -393,7 +429,8 @@ def process(self):
 
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
-        filename = os.path.join(file_path, scan_number,'eiger_0000.h5')
+        filename = os.path.join(file_path, scan_number,'eiger_????.h5')
+        filenames = sorted(glob.glob(filename))
 
         dest_dir = os.path.join(file_path, scan_number, 'sum')
 
@@ -401,7 +438,7 @@ def process(self):
             os.makedirs(dest_dir)
 
         output = os.path.join(dest_dir,'sum.edf')
-        command = ['pyFAI-average', '-m', 'sum', '-o', output, filename ]
+        command = ['pyFAI-average', '-m', 'sum', '-o', output] + filenames
         result = subprocess.run(command, capture_output=True)
         self.output["output_filename"] = output
         self.output["conversion"] = unpack_CompletedProcess(result)        

From d709e3b917e9d6d01ff91c38f5ea165ba34818dc Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 29 Mar 2022 16:45:32 +0200
Subject: [PATCH 14/98] debug on beamline

---
 plugins/id27.py | 51 ++++++++++++++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 70ed8c5..6f04f5b 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -16,8 +16,6 @@
     crysalis = None
 
 import subprocess
-# import threading
-# import multiprocessing
 
 WORKDIR = "/scratch/shared"
 
@@ -35,15 +33,16 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega
     :param exposure_time: in seconds
     :return 2 dicts, one containing the path crysalis_files, the second with scans
     """
-    if not (calibration_path and calibration_name) :
-        calibration_path = "/users/opid27/file_conversion/"
-        calibration_name = "scan0001"
+    calibration_name = calibration_name or ""
+    calibration_path = calibration_path or ""
+    par_file = os.path.join(calibration_path, calibration_name+'.par')
+    if not os.path.exists(par_file):
+        par_file = "/users/opid27/file_conversion/scan0001.par"
     crysalis_files = {
-        'par_file': os.path.join(calibration_path, calibration_name+'.par'),
+        'par_file': par_file,
         'set_file': '/users/opid27/file_conversion/scan0001.set',
         'ccd_file': '/users/opid27/file_conversion/scan0001.ccd'}
 
-
     scans = collections.OrderedDict()
     scans[0] = [{
             'count': number_of_frames,
@@ -298,7 +297,7 @@ def fabio_conversion(file_path,
         else:
             for idx_file, filename in enumerate(files):
                 img_data_fabio = fabio.open(filename)
-                dest_dir = os.path.join(file_path, scan_number, "{folder}_{idx_file+1:04d}")
+                dest_dir = os.path.join(file_path, scan_number, f"{folder}_{idx_file+1:04d}")
                 if not os.path.exists(dest_dir):
                     os.makedirs(dest_dir)                
                 for i, frame in enumerate(img_data_fabio):
@@ -431,18 +430,30 @@ def process(self):
         scan_number = self.input["scan_number"]
         filename = os.path.join(file_path, scan_number,'eiger_????.h5')
         filenames = sorted(glob.glob(filename))
-
-        dest_dir = os.path.join(file_path, scan_number, 'sum')
-
-        if not os.path.exists(dest_dir):
-            os.makedirs(dest_dir)
-
-        output = os.path.join(dest_dir,'sum.edf')
-        command = ['pyFAI-average', '-m', 'sum', '-o', output] + filenames
-        result = subprocess.run(command, capture_output=True)
-        self.output["output_filename"] = output
-        self.output["conversion"] = unpack_CompletedProcess(result)        
-
+        if len(filenames) == 0:
+            raise RuntimeError(f"File does not exist {filename}")
+        elif len(filenames) == 1:
+            dest_dir = os.path.join(file_path, scan_number, 'sum')
+            if not os.path.exists(dest_dir):
+                os.makedirs(dest_dir)
+            output = os.path.join(dest_dir,'sum.edf')
+            command = ['pyFAI-average', '-m', 'sum', '-o', output] + filenames
+            result = subprocess.run(command, capture_output=True)
+            self.output["output_filename"] = [output]
+            self.output["conversion"] = unpack_CompletedProcess(result)        
+        else:
+            results = {}
+            outputs = []
+            for idx_h5, filename in enumerate(filenames):
+                dest_dir = os.path.join(file_path, scan_number, f'sum_{idx_h5+1:04d}')
+                if not os.path.exists(dest_dir):
+                    os.makedirs(dest_dir)
+                output = os.path.join(dest_dir,'sum.edf')
+                command = ['pyFAI-average', '-m', 'sum', '-o', output, filename]
+                results[filename] = unpack_CompletedProcess(subprocess.run(command, capture_output=True))
+                outputs.append(output)
+            self.output["output_filename"] = outputs
+            self.output["conversion"] = results   
 
 @register
 class XdsConversion(Plugin):

From 925a7a6adf73045be42f5eab47ce6a56f3bfc8af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Kieffer?=
 <jerome.kieffer@terre-adelie.org>
Date: Thu, 31 Mar 2022 15:22:17 +0200
Subject: [PATCH 15/98] further polishing

---
 plugins/id27.py | 263 ++++++++++++++++++++++++------------------------
 1 file changed, 130 insertions(+), 133 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 6f04f5b..e9f6ca6 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -1,3 +1,6 @@
+"""Online data analysis for ID27
+"""
+
 import os
 import shutil
 import collections
@@ -5,9 +8,11 @@
 import logging
 import re
 import glob
+import subprocess
+import fabio
 from dahu.plugin import Plugin
 from dahu.factory import register
-import fabio
+
 logger = logging.getLogger("id27")
 
 try:
@@ -15,12 +20,11 @@
 except ImportError:
     crysalis = None
 
-import subprocess
-
 WORKDIR = "/scratch/shared"
 
 
-def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega_start, omega_step, center, distance, wave_length, exposure_time):
+def crysalis_config(calibration_path, calibration_name, number_of_frames,
+                    omega_start, omega_step, center, distance, wave_length, exposure_time):
     """
     :param calibration_path: typically "/users/opid27/file_conversion/"
     :param calibration_name: typically "scan0001"
@@ -35,7 +39,7 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega
     """
     calibration_name = calibration_name or ""
     calibration_path = calibration_path or ""
-    par_file = os.path.join(calibration_path, calibration_name+'.par')
+    par_file = os.path.join(calibration_path, calibration_name + '.par')
     if not os.path.exists(par_file):
         par_file = "/users/opid27/file_conversion/scan0001.par"
     crysalis_files = {
@@ -48,7 +52,7 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega
             'count': number_of_frames,
             'omega': 0,
             'omega_start': omega_start,
-            'omega_end': omega_start+number_of_frames*omega_step,
+            'omega_end': omega_start + number_of_frames * omega_step,
             'pixel_size': 0.075,
             'omega_runs': None,
             'theta': 0,
@@ -68,7 +72,7 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega
             'b': wave_length,
             'mono': 0.99,
             'monotype': 'SYNCHROTRON',
-            'chip': [1024,1024],
+            'chip': [1024, 1024],
             'Exposure_time': exposure_time,
             }]
 
@@ -76,30 +80,30 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,  omega
 
 
 def copy_set_ccd(crysalis_files, crysalis_dir, basename):
+    "copy .set and .ccd file"
+    shutil.copy(crysalis_files['set_file'], os.path.join(crysalis_dir, basename + '.set'))
+    shutil.copy(crysalis_files['ccd_file'], os.path.join(crysalis_dir, basename + '.ccd'))
 
-    shutil.copy(crysalis_files['set_file'], os.path.join(crysalis_dir,basename+'.set'))
-    shutil.copy(crysalis_files['ccd_file'], os.path.join(crysalis_dir,basename+'.ccd'))
 
-
-def unpack_CompletedProcess(cp):
+def unpack_processed(completed_process):
     """Convert a CompletedProcess object in to something which is serialisable
     
-    :param cp: Return of subprocess.run
+    :param completed_process: Return of subprocess.run
     :return: dict with the same content
     """
     res = {}
-    for k in dir(cp):
-        
-        if k.startswith("_"):
+    for key in dir(completed_process):
+
+        if key.startswith("_"):
             continue
-        v = cp.__getattribute__(k)
-        if callable(v):
+        val = completed_process.__getattribute__(key)
+        if callable(val):
             continue
-        if "decode" in dir(v):
-            v = v.decode()
-        res[k] = v
+        if "decode" in dir(val):
+            val = val.decode()
+        res[key] = val
     return res
-    
+
 
 def createCrysalis(scans, crysalis_dir, basename):
     runHeader = crysalis.RunHeader(basename.encode(), crysalis_dir.encode(), 1)
@@ -124,7 +128,7 @@ def createCrysalis(scans, crysalis_dir, basename):
 
 def create_par_file(crysalis_files, crysalis_dir, basename):
 
-    new_par = os.path.join(crysalis_dir, basename+'.par')
+    new_par = os.path.join(crysalis_dir, basename + '.par')
 
     with io.open(new_par, 'w', encoding='iso-8859-1') as new_file:
         with io.open(crysalis_files['par_file'], 'r', encoding='iso-8859-1') as old_file:
@@ -144,7 +148,7 @@ def create_rsync_file(filename, folder="esp"):
     :param folder: name of the folder to create. If None, just return the path of the script
     :return: the path of the crysalis directory  
     """
-    dest_dir = "/".join(filename.split("/")[3:-1]) #strip /data/visitor ... filename.h5
+    dest_dir = "/".join(filename.split("/")[3:-1])  # strip /data/visitor ... filename.h5
     dest_dir = os.path.join(WORKDIR, dest_dir)
     script = os.path.join(dest_dir, "sync")
     if folder is None:
@@ -152,10 +156,10 @@ def create_rsync_file(filename, folder="esp"):
 
     crysalis_dir = os.path.join(dest_dir, folder)
     if not os.path.exists(crysalis_dir):
-            os.makedirs(crysalis_dir)
+        os.makedirs(crysalis_dir)
     with open(os.path.join(dest_dir, ".source"), "a") as source:
-        source.write(filename+os.linesep)
-    
+        source.write(filename + os.linesep)
+
     if os.path.exists(script):
         with open(script, "a") as source:
             source.write(os.linesep.join([f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(filename)}', ""]))
@@ -169,7 +173,7 @@ def create_rsync_file(filename, folder="esp"):
 
 def crysalis_conversion(wave_length=None, distance=None,
                         center=(None, None),
-                        omega_start=None,omega_step=None,
+                        omega_start=None, omega_step=None,
                         exposure_time=None,
                         number_of_points=None,
                         file_source_path=None,
@@ -179,22 +183,22 @@ def crysalis_conversion(wave_length=None, distance=None,
     """Run the `eiger2crysalis` script synchronously
      
     """
-    
+
     assert crysalis, "cryio is not installed"
-    
-    logger.info('start data convertion with parameters:\n'+
-                f'wave_length={wave_length}\n'+
-                f'distance={distance}\n'+
-                f'center={center}\n'+
-                f'omega_start={omega_start}\n'+
-                f'omega_step={omega_step}\n'+
-                f'file_source_path={file_source_path}\n'+
+
+    logger.info('start data convertion with parameters:\n' +
+                f'wave_length={wave_length}\n' +
+                f'distance={distance}\n' +
+                f'center={center}\n' +
+                f'omega_start={omega_start}\n' +
+                f'omega_step={omega_step}\n' +
+                f'file_source_path={file_source_path}\n' +
                 f'scan_name={scan_name}')
 
     script_name = 'eiger2crysalis'
 
     crysalis_dir = create_rsync_file(file_source_path)
-    
+
     parameters = [script_name,
                   "-w", f'{wave_length}',
                   "-d", f'{distance}',
@@ -202,20 +206,20 @@ def crysalis_conversion(wave_length=None, distance=None,
                   f"--omega={omega_start}+{omega_step}*index",
                   file_source_path,
                   "-o", os.path.join(crysalis_dir, f'{scan_name}_1_''{index}.esperanto')]
-    logger.info('starts with parameters: %s',parameters)
+    logger.info('starts with parameters: %s', parameters)
 
-    crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points,  omega_start, omega_step, center, distance, wave_length, exposure_time)
+    crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points, omega_start, omega_step, center, distance, wave_length, exposure_time)
 
-    copy_set_ccd(crysalis_files, crysalis_dir, scan_name )
+    copy_set_ccd(crysalis_files, crysalis_dir, scan_name)
     createCrysalis(scans, crysalis_dir, scan_name)
-    create_par_file(crysalis_files,crysalis_dir, scan_name)
+    create_par_file(crysalis_files, crysalis_dir, scan_name)
 
-    return subprocess.run(parameters, capture_output=True)
+    return subprocess.run(parameters, capture_output=True, check=False)
 
 
 def crysalis_conversion_fscannd(wave_length=None,
                                 distance=None,
-                                center=(None,None),
+                                center=(None, None),
                                 omega_start=None,
                                 omega_step=None,
                                 exposure_time=None,
@@ -229,21 +233,20 @@ def crysalis_conversion_fscannd(wave_length=None,
     assert crysalis, "cryio is not installed"
     script_name = 'eiger2crysalis'
     pattern = re.compile('eiger_([0-9]+).h5')
-    filenames_to_convert = glob.glob(f'{dirname}/eiger*.h5')
+    filenames_to_convert = glob.glob(f'{dirname}/eiger_????.h5')
     results = {}
-    if filenames_to_convert:
-        crysalis
+
     for filepath in sorted(filenames_to_convert):
-        
+
         filename = os.path.basename(filepath)
         g = pattern.match(filename)
         if g:
             number = int(g.group(1))
             crysalis_folder_name = f'esp_{number:04d}'
             crysalis_dir = create_rsync_file(filepath, crysalis_folder_name)
-            
+
             if motor_mode == "ZIGZAG" and (number % 2):
-                revert_omega_start=omega_start+(omega_step*npoints)
+                revert_omega_start = omega_start + (omega_step * npoints)
                 omega_par = f"--omega={revert_omega_start}-{omega_step}*index"
             else:
                 omega_par = f"--omega={omega_start}+{omega_step}*index"
@@ -256,14 +259,14 @@ def crysalis_conversion_fscannd(wave_length=None,
                           filepath,
                           "-o", os.path.join(crysalis_dir,
                                              f'{scan_name}_{number}_1_''{index}.esperanto')]
-            #print('starts with parameters:',parameters)
-            results[filename] = subprocess.run(parameters, capture_output=True)
-            crysalis_files, scans = crysalis_config(calibration_path, calibration_name, npoints,  omega_start, omega_step, center, distance, wave_length, exposure_time)
-            
+            # print('starts with parameters:',parameters)
+            results[filename] = subprocess.run(parameters, capture_output=True, check=False)
+            crysalis_files, scans = crysalis_config(calibration_path, calibration_name, npoints, omega_start, omega_step, center, distance, wave_length, exposure_time)
+
             crysalis_scan_name = scan_name + '_' + str(number)
-            copy_set_ccd(crysalis_files,crysalis_dir, crysalis_scan_name )
-            createCrysalis(scans, crysalis_dir,  crysalis_scan_name)
-            create_par_file(crysalis_files,crysalis_dir, crysalis_scan_name)
+            copy_set_ccd(crysalis_files, crysalis_dir, crysalis_scan_name)
+            createCrysalis(scans, crysalis_dir, crysalis_scan_name)
+            create_par_file(crysalis_files, crysalis_dir, crysalis_scan_name)
     return results
 
 
@@ -272,49 +275,36 @@ def fabio_conversion(file_path,
                      folder="xdi",
                      fabioimage="tifimage",
                      extension="tif"):
+    "Convert a set of eiger files to cbf or tiff"
+    results = []
+    filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
+    files = sorted(glob.glob(filename))
+
+    if len(files) == 0:
+        raise RuntimeError(f"No such file {filename}")
+
+    for idx_file, filename in enumerate(files):
+        img_data_fabio = fabio.open(filename)
+        basename = folder if len(files) == 1 else f"{folder}_{idx_file+1:04d}"
+        dest_dir = os.path.join(file_path, scan_number, basename)
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+        for i, frame in enumerate(img_data_fabio):
+            conv = frame.convert(fabioimage)
+            data = conv.data.astype('int32')
+            data[data < 0 ] = 0
+            conv.data = data
+            output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
+            conv.write(output)
+            results.append(output)
 
-        results = []
-        filename = os.path.join(file_path, scan_number,'eiger_????.h5')
-        files = sorted(glob.glob(filename))
-        
-        if len(files) == 0:
-            raise RuntimeError(f"No such file {filename}")
-        
-        elif len(files) == 1:
-            filename = files[0]
-            img_data_fabio = fabio.open(filename)
-            dest_dir = os.path.join(file_path, scan_number, folder)
-            if not os.path.exists(dest_dir):
-                os.makedirs(dest_dir)
-            for i, frame in enumerate(img_data_fabio):
-                conv = frame.convert(fabioimage)
-                data = conv.data.astype('int32')
-                data[data <0 ] = 0
-                conv.data = data
-                output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
-                conv.write(output)
-                results.append(output)
-        else:
-            for idx_file, filename in enumerate(files):
-                img_data_fabio = fabio.open(filename)
-                dest_dir = os.path.join(file_path, scan_number, f"{folder}_{idx_file+1:04d}")
-                if not os.path.exists(dest_dir):
-                    os.makedirs(dest_dir)                
-                for i, frame in enumerate(img_data_fabio):
-                    conv = frame.convert(fabioimage)
-                    data = conv.data.astype('int32')
-                    data[data <0 ] = 0
-                    conv.data = data
-                    output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
-                    conv.write(output)
-                    results.append(output)
-            
-        return results
+    return results
 
 ##########################
 # Start the server part
 ##########################
 
+
 @register
 class CrysalisConversion(Plugin):
     """
@@ -336,25 +326,25 @@ class CrysalisConversion(Plugin):
  "plugin_name": "id27.CrysalisConversion"
 }    
     """
-    
+
     def process(self):
         Plugin.process(self)
         if not self.input:
             logger.error("input is empty")
 
         result = crysalis_conversion(**self.input)
-        self.output["results"] = unpack_CompletedProcess(result)   
+        self.output["results"] = unpack_processed(result)
 
         script = create_rsync_file(self.input["file_source_path"], None)
-        sync_results = subprocess.run([script], capture_output=True)
-        self.output["sync"] = unpack_CompletedProcess(sync_results)
+        sync_results = subprocess.run([script], capture_output=True, check=False)
+        self.output["sync"] = unpack_processed(sync_results)
 
 
 @register
 class CrysalisConversionFscannd(Plugin):
     """
     This is the plugin of cysalis conversion for fscannd type scans
-       
+
     Typical JSON file:
     {"wave_length": 1.54,
      "distance": 100,
@@ -371,17 +361,17 @@ class CrysalisConversionFscannd(Plugin):
     }
     
     """
-    
+
     def process(self):
         Plugin.process(self)
         if not self.input:
             logger.error("input is empty")
         result = crysalis_conversion_fscannd(**self.input)
-        self.output["results"] = {k: unpack_CompletedProcess(v) for k,v in result.items()}
-        random_filename = glob.glob(f'{self.input["dirname"]}/eiger*.h5')[0]        
+        self.output["results"] = {k: unpack_processed(v) for k, v in result.items()}
+        random_filename = glob.glob(f'{self.input["dirname"]}/eiger*.h5')[0]
         script = create_rsync_file(random_filename, None)
-        sync_results = subprocess.run([script], capture_output=True)
-        self.output["sync"] = unpack_CompletedProcess(sync_results)
+        sync_results = subprocess.run([script], capture_output=True, check=False)
+        self.output["sync"] = unpack_processed(sync_results)
 
 
 @register
@@ -395,12 +385,12 @@ class XdiConversion(Plugin):
     }
     
     """
-    
+
     def process(self):
         Plugin.process(self)
         if not self.input:
             logger.error("input is empty")
-        
+
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
         results = fabio_conversion(file_path,
@@ -408,7 +398,8 @@ def process(self):
                          folder="xdi",
                          fabioimage="tifimage",
                          extension="tif")
-        self.output["output"] = results        
+        self.output["output"] = results
+
 
 @register
 class Average(Plugin):
@@ -419,41 +410,47 @@ class Average(Plugin):
     {"file_path": "/data/id27/inhouse/some/directory",
      "scan_number": "scan0001"
     }
-    
-    """    
+    # optional: "output_directory": "/path/to/some/output"
+    """
+
     def process(self):
         Plugin.process(self)
+
+        results = {}
+        outputs = []
+        basename = 'sum.edf'
+        prefix = basename.split('.')[0]
+
         if not self.input:
             logger.error("input is empty")
-
+        output_directory = self.input.get("output_directory")
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
-        filename = os.path.join(file_path, scan_number,'eiger_????.h5')
+        filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
         filenames = sorted(glob.glob(filename))
+
         if len(filenames) == 0:
             raise RuntimeError(f"File does not exist {filename}")
-        elif len(filenames) == 1:
-            dest_dir = os.path.join(file_path, scan_number, 'sum')
+
+        for idx_h5, filename in enumerate(filenames):
+            if output_directory:
+                dest_dir = output_directory
+                sample, dataset = file_path.strip().strip("/").split("/")[-2:]
+                tmpname = basename if len(filenames) == 1 else f'{prefix}_{idx_h5+1:04d}.edf'
+                basename = "_".join((sample, dataset, scan_number, tmpname))
+            else:
+                tmpname = prefix if len(filenames) == 1 else f'{prefix}_{idx_h5+1:04d}'
+                dest_dir = os.path.join(file_path, scan_number, tmpname)
             if not os.path.exists(dest_dir):
                 os.makedirs(dest_dir)
-            output = os.path.join(dest_dir,'sum.edf')
-            command = ['pyFAI-average', '-m', 'sum', '-o', output] + filenames
-            result = subprocess.run(command, capture_output=True)
-            self.output["output_filename"] = [output]
-            self.output["conversion"] = unpack_CompletedProcess(result)        
-        else:
-            results = {}
-            outputs = []
-            for idx_h5, filename in enumerate(filenames):
-                dest_dir = os.path.join(file_path, scan_number, f'sum_{idx_h5+1:04d}')
-                if not os.path.exists(dest_dir):
-                    os.makedirs(dest_dir)
-                output = os.path.join(dest_dir,'sum.edf')
-                command = ['pyFAI-average', '-m', 'sum', '-o', output, filename]
-                results[filename] = unpack_CompletedProcess(subprocess.run(command, capture_output=True))
-                outputs.append(output)
-            self.output["output_filename"] = outputs
-            self.output["conversion"] = results   
+
+            output = os.path.join(dest_dir, basename)
+            command = ['pyFAI-average', '-m', 'sum', '-o', output, filename]
+            results[filename] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
+            outputs.append(output)
+        self.output["output_filename"] = outputs
+        self.output["conversion"] = results
+
 
 @register
 class XdsConversion(Plugin):
@@ -466,12 +463,12 @@ class XdsConversion(Plugin):
     }
     
     """
-    
+
     def process(self):
         Plugin.process(self)
         if not self.input:
             logger.error("input is empty")
-        
+
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
         results = fabio_conversion(file_path,
@@ -479,4 +476,4 @@ def process(self):
                          folder="cbf",
                          fabioimage="cbfimage",
                          extension="cbf")
-        self.output["output"] = results        
+        self.output["output"] = results

From 2d8ba90dc568d04638a5d0d38f71fc676c0ad732 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 14:52:22 +0200
Subject: [PATCH 16/98] try to cope with files under writing

---
 plugins/bm29/integrate.py | 71 +++++++++++++++++++++------------------
 plugins/bm29/nexus.py     | 53 ++++++++++++++++++++---------
 2 files changed, 77 insertions(+), 47 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 270cf5a..be16e11 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -3,48 +3,53 @@
 
 """Data Analysis plugin for BM29: BioSaxs
 
-* IntegrateMultiframe: perform the integration of many frames contained in a HDF5 file and average them  
- 
+* IntegrateMultiframe: perform the integration of many frames contained in a HDF5 file and average them
+
 """
 
 __authors__ = ["Jérôme Kieffer"]
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "20/09/2021"
+__date__ = "21/04/2022"
 __status__ = "development"
 __version__ = "0.2.3"
 
 import os
 import time
 import json
-from urllib3.util import parse_url
+import logging
 from collections import namedtuple
+from urllib3.util import parse_url
 from dahu.plugin import Plugin
 from dahu.factory import register
 from dahu.utils import fully_qualified_name
-import logging
-logger = logging.getLogger("bm29.integrate")
+
 import numpy
+import h5py
+import pyFAI
+import pyFAI.azimuthalIntegrator
+import freesas
+import freesas.cormap
+
+from .common import Sample, Ispyb, get_equivalent_frames, cmp_int, cmp_float, get_integrator, KeyCache, \
+                    method, polarization_factor, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \
+                    create_nexus_sample
+from .ispyb import IspybConnector, NumpyEncoder
+
+
+logger = logging.getLogger("bm29.integrate")
 try:
     import numexpr
 except ImportError:
     logger.error("Numexpr is not installed, falling back on numpy's implementations")
     numexpr = None
-import h5py
-import fabio
-import pyFAI, pyFAI.azimuthalIntegrator
-import freesas, freesas.cormap
+
 try:
     import memcache
 except (ImportError, ModuleNotFoundError):
     memcache = None
 
-# from pyFAI.azimuthalIntegrator import AzimuthalIntegrator
-from .common import Sample, Ispyb, get_equivalent_frames, cmp_int, cmp_float, get_integrator, KeyCache, \
-                    method, polarization_factor, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \
-                    create_nexus_sample
-from .ispyb import IspybConnector, NumpyEncoder
 
 IntegrationResult = namedtuple("IntegrationResult", "radial intensity sigma")
 CormapResult = namedtuple("CormapResult", "probability count tomerge")
@@ -67,7 +72,7 @@ class IntegrateMultiframe(Plugin):
       "timestamps": [1580985678.47, 1580985678.58],
       "monitor_values": [1, 1.1],
       "storage_ring_current": [199.6, 199.5]
-      "exposure_time": 0.1, 
+      "exposure_time": 0.1,
       "normalisation_factor": 1.0,
       "poni_file": "/tmp/example.poni",
       "mask_file": "/tmp/mask.edf",
@@ -76,6 +81,7 @@ class IntegrateMultiframe(Plugin):
       "fidelity_abs": 1e-5,
       "fidelity_rel": 1e-3,
       "hplc_mode": 0,
+      "timeout": 10,
       "sample": {
         "name": "bsa",
         "description": "protein description like Bovine Serum Albumin",
@@ -88,7 +94,7 @@ class IntegrateMultiframe(Plugin):
         "url": "http://ispyb.esrf.fr:1234",
         "login": "mx1234",
         "passwd": "secret",
-        "pyarch": "/data/pyarch/mx1234/sample", 
+        "pyarch": "/data/pyarch/mx1234/sample",
         "measurement_id": -1,
         "collection_id": -1
        }
@@ -110,6 +116,7 @@ def __init__(self):
         self.nb_frames = None
         self.ai = None
         self.npt = 1000
+        self.timeout = 10
         self.unit = pyFAI.units.to_unit("q_nm^-1")
         # self.polarization_factor = 0.9 --> constant
         self.poni = self.mask = None
@@ -130,9 +137,10 @@ def setup(self, kwargs=None):
         if not self.sample.name:
             self.sample = Sample("Unknown sample", *self.sample[1:])
 
+        self.timeout = self.input.get("timeout", self.timeout)
         self.input_file = self.input.get("input_file")
         if self.input_file is not None:
-            self.wait_file(self.input_file)
+            self.wait_file(self.input_file, timeout=self.timeout)
         else:
             self.log_error(f"No valid input file provided {self.input_file}")
 
@@ -182,7 +190,7 @@ def input_frames(self):
         "For performance reasons, all frames are read in one bloc and cached, this returns a 3D numpy array"
         if self._input_frames is None:
             try:
-                with Nexus(self.input_file, "r") as nxs:
+                with Nexus(self.input_file, "r", timeout=self.timeout) as nxs:
                     entry = nxs.get_entries()[0]
                     if "measurement" in entry:
                         measurement = entry["measurement"]
@@ -207,38 +215,40 @@ def process(self):
         if not self.input.get("hplc_mode"):
             self.send_to_ispyb()
 
-    def wait_file(self, filename, timeout=10):
+    def wait_file(self, filename, timeout=None):
         """Wait for a file to appear on a filesystem
 
- 	:param filename: name of the file
+        :param filename: name of the file
         :param timeout: time-out in seconds
-        
+
         Raises an exception and ends the processing in case of missing file!
 	"""
-        end_time = time.time() + timeout
+        timeout = self.timeout if timeout is None else timeout
+        end_time = time.perf_counter() + timeout
         dirname = os.path.dirname(filename)
         while not os.path.isdir(dirname):
-            if time.time() > end_time:
+            if time.perf_counter() > end_time:
                 self.log_error(f"Filename {filename} did not appear in {timeout} seconds")
             time.sleep(0.1)
             os.stat(os.path.dirname(dirname))
 
         while not os.path.exists(filename):
-            if time.time() > end_time:
+            if time.perf_counter() > end_time:
                 self.log_error(f"Filename {filename} did not appear in {timeout} seconds")
             time.sleep(0.1)
             os.stat(dirname)
 
         while not os.stat(filename).st_size:
-            if time.time() > end_time:
+            if time.perf_counter() > end_time:
                 self.log_error(f"Filename {filename} did not appear in {timeout} seconds")
             time.sleep(0.1)
             os.stat(dirname)
-        appear_time = time.time() - end_time + timeout
+        appear_time = time.perf_counter() - end_time + timeout
         if appear_time > 1.0:
             self.log_warning(f"Filename {filename} took {appear_time:.3f}s to appear on filesystem!")
 
     def create_nexus(self):
+        "create the nexus result file with basic structure"
         if not os.path.isdir(os.path.dirname(self.output_file)):
             os.makedirs(os.path.dirname(self.output_file))
         creation_time = os.stat(self.input_file).st_ctime
@@ -404,10 +414,9 @@ def create_nexus(self):
         if self.input.get("hplc_mode"):
             entry_grp.attrs["default"] = entry_grp.attrs["default"] = integration_grp.attrs["default"] = hplc_data.name
             self.log_warning("HPLC mode detected, stopping after frame per frame integration")
-
             return
-        else:
-            integration_grp.attrs["default"] = integration_data.name
+
+        integration_grp.attrs["default"] = integration_data.name
 
     # Process 2: Freesas cormap
         cormap_grp = nxs.new_class(entry_grp, "2_correlation_mapping", "NXprocess")
@@ -599,7 +608,5 @@ def send_to_memcached(self):
                 keys[k] = key
                 value = json.dumps(self.to_memcached[k], cls=NumpyEncoder)
                 rc[k] = mc.set(key, value)
-                # self.log_warning(f"key {k} dtype {self.to_memcached[k].dtype}, shape {self.to_memcached[k].shape}, nbytes {self.to_memcached[k].nbytes}") 
         self.log_warning(f"Return codes for memcached {rc}")
         return keys
-
diff --git a/plugins/bm29/nexus.py b/plugins/bm29/nexus.py
index b78809c..0f4b383 100644
--- a/plugins/bm29/nexus.py
+++ b/plugins/bm29/nexus.py
@@ -4,7 +4,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "12/10/2020"
+__date__ = "21/04/2022"
 __status__ = "production"
 __docformat__ = 'restructuredtext'
 
@@ -45,7 +45,7 @@ def from_isotime(text, use_tz=False):
         text = str(text)
     if len(text) < 19:
         logger.warning("Not a iso-time string: %s", text)
-        return
+        return None
     base = text[:19]
     if use_tz and len(text) == 25:
         sgn = 1 if text[:19] == "+" else -1
@@ -70,7 +70,7 @@ def is_hdf5(filename):
     return sig == signature
 
 
-class Nexus(object):
+class Nexus:
     """
     Writer class to handle Nexus/HDF5 data
 
@@ -85,13 +85,14 @@ class Nexus(object):
     TODO: make it thread-safe !!!
     """
 
-    def __init__(self, filename, mode=None, creator=None):
+    def __init__(self, filename, mode=None, creator=None, timeout=None):
         """
         Constructor
 
         :param filename: name of the hdf5 file containing the nexus
         :param mode: can be 'r', 'a', 'w', '+' ....
         :param creator: set as attr of the NXroot
+        :param timeout: retry for that amount of time (in seconds)
         """
         self.filename = os.path.abspath(filename)
         self.mode = mode
@@ -106,12 +107,30 @@ def __init__(self, filename, mode=None, creator=None):
             else:
                 self.mode = "a"
 
-        if self.mode == "r":
-            self.file_handle = open(self.filename, mode=self.mode + "b")
-            self.h5 = h5py.File(self.file_handle, mode=self.mode)
+        if timeout:
+            end = time.perf_counter() + timeout
+            while time.perf_counter() < end :
+                try:
+                    if self.mode == "r":
+                        self.file_handle = open(self.filename, mode="rb")
+                        self.h5 = h5py.File(self.file_handle, mode="r")
+                    else:
+                        self.file_handle = None
+                        self.h5 = h5py.File(self.filename, mode=self.mode)
+                except OSError:
+                    os.stat(os.path.dirname(self.filename))
+                    time.sleep(1)
+                else:
+                    break
+            else:
+                raise OSError(f"Unable to open HDF5 file {self.filename}")
         else:
-            self.file_handle = None
-            self.h5 = h5py.File(self.filename, mode=self.mode)
+            if self.mode == "r":
+                self.file_handle = open(self.filename, mode=self.mode + "b")
+                self.h5 = h5py.File(self.file_handle, mode=self.mode)
+            else:
+                self.file_handle = None
+                self.h5 = h5py.File(self.filename, mode=self.mode)
         self.to_close = []
 
         if not pre_existing:
@@ -142,6 +161,7 @@ def __exit__(self, *arg, **kwarg):
         self.close()
 
     def flush(self):
+        "write to disk"
         if self.h5:
             self.h5.flush()
 
@@ -158,7 +178,8 @@ def get_entry(self, name):
                 if isinstance(grp, h5py.Group) and \
                    ("start_time" in grp) and  \
                    self.get_attr(grp, "NX_class") == "NXentry":
-                        return grp
+                    return grp
+        return None
 
     def get_entries(self):
         """
@@ -228,9 +249,11 @@ def new_instrument(self, entry="entry", instrument_name="id00",):
 #        howto external link
         # myfile['ext link'] = h5py.ExternalLink("otherfile.hdf5", "/path/to/resource")
 
-    def new_class(self, grp, name, class_type="NXcollection"):
+    @staticmethod
+    def new_class(grp, name, class_type="NXcollection"):
         """
         create a new sub-group with  type class_type
+        
         :param grp: parent group
         :param name: name of the sub-group
         :param class_type: NeXus class name
@@ -282,7 +305,7 @@ def get_data(self, grp, attr=None, value=None):
 
     def get_default_NXdata(self):
         """Return the default plot configured in the nexus structure.
-        
+
         :return: the group with the default plot or None if not found
         """
         entry_name = self.h5.attrs.get("default")
@@ -292,8 +315,8 @@ def get_default_NXdata(self):
             if nxdata_name:
                 if nxdata_name.startswith("/"):
                     return self.h5.get(nxdata_name)
-                else:
-                    return entry_grp.get(nxdata_name)
+                return entry_grp.get(nxdata_name)
+        return None
 
     def deep_copy(self, name, obj, where="/", toplevel=None, excluded=None, overwrite=False):
         """
@@ -313,7 +336,7 @@ def deep_copy(self, name, obj, where="/", toplevel=None, excluded=None, overwrit
             if name not in toplevel:
                 grp = toplevel.require_group(name)
                 for k, v in obj.attrs.items():
-                        grp.attrs[k] = v
+                    grp.attrs[k] = v
         elif isinstance(obj, h5py.Dataset):
             if name in toplevel:
                 if overwrite:

From 8835754a07d6618284c2fd343434453a4dd35073 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 17:12:11 +0200
Subject: [PATCH 17/98] save the normalization factor associated with the
 averaged frames

---
 plugins/bm29/integrate.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index be16e11..f0e5dfa 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -13,7 +13,7 @@
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
 __date__ = "21/04/2022"
 __status__ = "development"
-__version__ = "0.2.3"
+__version__ = "0.3.0"
 
 import os
 import time
@@ -53,7 +53,7 @@
 
 IntegrationResult = namedtuple("IntegrationResult", "radial intensity sigma")
 CormapResult = namedtuple("CormapResult", "probability count tomerge")
-AverageResult = namedtuple("AverageResult", "average deviation")
+AverageResult = namedtuple("AverageResult", "average deviation normalization")
 
 
 @register
@@ -464,6 +464,7 @@ def create_nexus(self):
 
         Iavg = numpy.ascontiguousarray(res3.average, dtype=numpy.float32)
         sigma_avg = numpy.ascontiguousarray(res3.deviation, dtype=numpy.float32)
+        norm = numpy.ascontiguousarray(res3.normalization, dtype=numpy.float32)
 
         int_avg_ds = average_data.create_dataset("intensity_normed",
                                                   data=Iavg,
@@ -474,8 +475,12 @@ def create_nexus(self):
                                                    data=sigma_avg,
                                                    **cmp_float)
         int_std_ds.attrs["interpretation"] = "image"
-        int_std_ds.attrs["formula"] = "sqrt(sum_i(variance_i))/sum(normalization_i)"
+        int_std_ds.attrs["formula"] = "sqrt(sum_i(variance_i)/sum_i(normalization_i))"
         int_std_ds.attrs["method"] = "Propagated error from weighted mean assuming poissonian behavour of every data-point"
+        
+        int_nrm_ds = average_data.create_dataset("normalization", data=norm)
+        int_nrm_ds.attrs["formula"] = "sum_i(normalization_i))"
+        
         average_grp.attrs["default"] = average_data.name
 
     # Process 4: Azimuthal integration of the time average image
@@ -580,14 +585,14 @@ def process3_average(self, tomerge):
         if numexpr is not None:
             # Numexpr is many-times faster than numpy when it comes to element-wise operations
             intensity_avg = numexpr.evaluate("where(mask==0, sum_data/sum_norm, 0.0)")
-            intensity_std = numexpr.evaluate("where(mask==0, sqrt(sum_data)/sum_norm, 0.0)")
+            intensity_std = numexpr.evaluate("sqrt(intensity_avg)")
         else:
             intensity_avg = sum_data / sum_norm
-            intensity_std = numpy.sqrt(sum_data) / sum_norm
+            intensity_std = numpy.sqrt(intensity_avg)
             wmask = numpy.where(mask)
             intensity_avg[wmask] = 0.0
             intensity_std[wmask] = 0.0
-        return AverageResult(intensity_avg, intensity_std)
+        return AverageResult(intensity_avg, intensity_std, sum_norm)
 
     def send_to_ispyb(self):
         if self.ispyb.url and parse_url(self.ispyb.url).host:

From 70f2d4d88eeddd868fec194bf249b345b73dc7fa Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 17:24:18 +0200
Subject: [PATCH 18/98] propagate uncertainties in 2d merge

+ issue with kratki plot
---
 plugins/bm29/subtracte.py | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index ed20e14..86fe7e3 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "20/04/2021"
+__date__ = "21/04/2022"
 __status__ = "development"
 __version__ = "0.2.0"
 
@@ -43,7 +43,7 @@
                     Sample, create_nexus_sample
 from .ispyb import IspybConnector
 
-NexusJuice = namedtuple("NexusJuice", "filename h5path npt unit q I sigma poni mask energy polarization method signal2d error2d sample")
+NexusJuice = namedtuple("NexusJuice", "filename h5path npt unit q I sigma poni mask energy polarization method signal2d error2d normalization sample")
 
 
 class SubtractBuffer(Plugin):
@@ -252,19 +252,19 @@ def create_nexus(self):
         average_data.attrs["signal"] = "intensity_normed"
     # Stage 2 processing
 
-        # Nota: formula probably wrong ! Take into account the number of input frames in each averaged buffer !
-        # TODO implement those math using numexpr:
+        # Nota: This formula takes into account the number of input frames in each averaged buffer !        
         #  avg = Σdata / Σnorm
-        #  var = Σdata / (Σnorm)²
-        #  Σnorm = avg / var
-        #  Σdata = avg² / var
-        # The propagate the error based on the number of frames in each buffer with quadratic summation
-
-        buffer_average = numpy.mean([self.buffer_juices[i].signal2d for i in range(*tomerge)], axis=0)
-        buffer_variance = numpy.sum([(self.buffer_juices[i].error2d) ** 2 for i in range(*tomerge)], axis=0) / (tomerge[1] - tomerge[0]) ** 2
+        #  var = sigma² = ΣV / Σnorm
+        # TODO implement those math using numexpr:
+        sum_signal = sum([self.buffer_juices[i].normalization * self.buffer_juices[i].signal2d for i in range(*tomerge)])
+        sum_variance = sum([(self.buffer_juices[i].normalization * self.buffer_juices[i].error2d) ** 2 for i in range(*tomerge)])
+        norm = [self.buffer_juices[i].normalization for i in range(*tomerge)]
+        sum_norm = sum(norm)
+        buffer_average = sum_signal / sum_norm
+        buffer_variance = sum_variance / sum_norm
         sample_average = self.sample_juice.signal2d
         sample_variance = self.sample_juice.error2d ** 2
-        sub_average = self.sample_juice.signal2d - buffer_average
+        sub_average = sample_average - buffer_average
         sub_variance = sample_variance + buffer_variance
         sub_std = numpy.sqrt(sub_variance)
 
@@ -272,12 +272,12 @@ def create_nexus(self):
                                                  data=numpy.ascontiguousarray(sub_average, dtype=numpy.float32),
                                                  **cmp_float)
         int_avg_ds.attrs["interpretation"] = "image"
-        int_avg_ds.attrs["formula"] = "sample_signal - mean(buffer_signal_i)"
+        int_avg_ds.attrs["formula"] = "sample_signal - weighted_mean(buffer_signal_i)"
         int_std_ds = average_data.create_dataset("intensity_std",
                                                  data=numpy.ascontiguousarray(sub_std, dtype=numpy.float32),
                                                  **cmp_float)
         int_std_ds.attrs["interpretation"] = "image"
-        int_std_ds.attrs["formula"] = "sqrt( sample_variance + (sum(buffer_variance)/n_buffer**2 ))"
+        int_std_ds.attrs["formula"] = "sqrt( sample_variance + weighted_mean(buffer_variance_i) )"
         int_std_ds.attrs["method"] = "quadratic sum of sample error and buffer errors"
         average_grp.attrs["default"] = average_data.name
 
@@ -518,13 +518,15 @@ def create_nexus(self):
         qRg_ds = kratky_data.create_dataset("qRg", data=xdata.astype(numpy.float32))
         qRg_ds.attrs["interpretation"] = "spectrum"
         qRg_ds.attrs["long_name"] = "q·Rg (unit-less)"
-        k_ds = kratky_data.create_dataset("q2Rg2I/I0", data=ydata.astype(numpy.float32))
+        
+        #Nota the "/" hereafter is chr(8725), the division sign and not the usual slash
+        k_ds = kratky_data.create_dataset("q2Rg2I∕I0", data=ydata.astype(numpy.float32)) 
         k_ds.attrs["interpretation"] = "spectrum"
         k_ds.attrs["long_name"] = "q²Rg²I(q)/I₀"
         ke_ds = kratky_data.create_dataset("errors", data=dy.astype(numpy.float32))
         ke_ds.attrs["interpretation"] = "spectrum"
         kratky_data_attrs = kratky_data.attrs
-        kratky_data_attrs["signal"] = "q2Rg2I/I0"
+        kratky_data_attrs["signal"] = "q2Rg2I∕I0"
         kratky_data_attrs["axes"] = "qRg"
 
     # stage 6: Rambo-Tainer invariant
@@ -676,6 +678,7 @@ def read_nexus(filename):
             img_grp = nxsr.get_class(entry_grp["3_time_average"], class_type="NXdata")[0]
             image2d = img_grp["intensity_normed"][()]
             error2d = img_grp["intensity_std"][()]
+            norm =  img_grp["normalization"][()] if "normalization" in img_grp else None
             # Read the sample description:
             sample_grp = nxsr.get_class(entry_grp, class_type="NXsample")[0]
             sample_name = sample_grp.name
@@ -688,7 +691,7 @@ def read_nexus(filename):
             temperature_env = sample_grp["temperature_env"][()] if "temperature_env" in sample_grp else ""
             sample = Sample(sample_name, description, buffer, concentration, hplc, temperature_env, temperature)
 
-        return NexusJuice(filename, h5path, npt, unit, q, I, sigma, poni, mask, energy, polarization, method, image2d, error2d, sample)
+        return NexusJuice(filename, h5path, npt, unit, q, I, sigma, poni, mask, energy, polarization, method, image2d, error2d, norm, sample)
 
     def send_to_ispyb(self):
         if self.ispyb.url and parse_url(self.ispyb.url).host:

From 5821b71bc80ddd8967f8521023f1d9e25fcddd6b Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 17:49:55 +0200
Subject: [PATCH 19/98] stop providing q/I/sigma as part of the output, use
 memcached

---
 plugins/bm29/integrate.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index f0e5dfa..1b48ef4 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -535,9 +535,9 @@ def create_nexus(self):
         entry_grp.attrs["default"] = ai2_data.name
 
         # Export this to the output JSON
-        self.output["q"] = res2.radial
-        self.output["I"] = res2.intensity
-        self.output["std"] = res2.sigma
+        # self.output["q"] = res2.radial
+        # self.output["I"] = res2.intensity
+        # self.output["std"] = res2.sigma
 
     def process1_integration(self, data):
         "First step of the processing, integrate all frames, return a IntegrationResult namedtuple"

From 674fc6119ef35f457db3629e5a5b345be9da5d1d Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 18:00:48 +0200
Subject: [PATCH 20/98] subtract with memcached

---
 plugins/bm29/subtracte.py | 42 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index 86fe7e3..8aada20 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -43,6 +43,11 @@
                     Sample, create_nexus_sample
 from .ispyb import IspybConnector
 
+try:
+    import memcache
+except (ImportError, ModuleNotFoundError):
+    memcache = None
+
 NexusJuice = namedtuple("NexusJuice", "filename h5path npt unit q I sigma poni mask energy polarization method signal2d error2d normalization sample")
 
 
@@ -54,7 +59,7 @@ class SubtractBuffer(Plugin):
       "buffer_files": ["buffer_001.h5", "buffer_002.h5"],
       "sample_file": "sample.h5",
       "output_file": "subtracted.h5"
-      "fidelity":= 0.001,
+      "fidelity": 0.001,
       "ispyb": {
         "url": "http://ispyb.esrf.fr:1234",
         "login": "mx1234",
@@ -83,6 +88,7 @@ def __init__(self):
         self.Rg = self.I0 = self.Dmax = self.Vc = self.mass = None
         self.ispyb = None
         self.to_pyarch = {}
+        self.to_memcached = {}  # data to be shared via memcached
 
     def setup(self, kwargs=None):
         logger.debug("SubtractBuffer.setup")
@@ -117,12 +123,15 @@ def teardown(self):
         self.output["Dmax"] = self.Dmax
         self.output["Vc"] = self.Vc
         self.output["mass"] = self.mass
+        self.output["memcached"] = self.send_to_memcached()
+        #teardown everything else:
         if self.nxs is not None:
             self.nxs.close()
         self.sample_juice = None
         self.buffer_juices = []
         self.ispyb = None
         self.to_pyarch = None
+        self.to_memcached = None
 
     def process(self):
         Plugin.process(self)
@@ -142,7 +151,9 @@ def process(self):
                                      (type(err2), err2, "\n".join(traceback.format_exc(limit=10))))
                 raise(err)
         else:
-            self.send_to_ispyb()
+            self.send_to_ispyb()        
+        
+
 
     def validate_buffer(self, buffer_file):
         "Validate if a buffer is consitent with the sample, return some buffer_juice or None when unconsistent"
@@ -239,6 +250,7 @@ def create_nexus(self):
         count_ds.attrs["long_name"] = "Longest sequence where curves do not cross each other"
 
         to_merge_ds = cormap_data.create_dataset("to_merge", data=numpy.arange(*tomerge, dtype=numpy.uint16))
+        self.log_warning(f"to_merge: {tomerge}")
         to_merge_ds.attrs["long_name"] = "Index of equivalent frames"
         cormap_grp.attrs["default"] = cormap_data.name
 
@@ -353,9 +365,12 @@ def create_nexus(self):
         if self.ispyb.url and parse_url(self.ispyb.url).host:
             self.to_pyarch["subtracted"] = res3
         # Export this to the output JSON
-        self.output["q"] = res3.radial
-        self.output["I"] = res3.intensity
-        self.output["std"] = res3.sigma
+        #self.output["q"] = res3.radial
+        #self.output["I"] = res3.intensity
+        #self.output["std"] = res3.sigma
+        self.to_memcached["q"] = res3.radial
+        self.to_memcached["I"] = res3.intensity
+        self.to_memcached["std"] = res3.sigma
 
         #  Finally declare the default entry and default dataset ...
         #  overlay the BIFT fitted data on top of the scattering curve
@@ -699,3 +714,20 @@ def send_to_ispyb(self):
             ispyb.send_subtracted(self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
+
+
+    def send_to_memcached(self):
+        "Send the content of self.to_memcached to the storage"
+        keys = {}
+        rc = {}
+        if memcache is not None:
+            mc = memcache.Client([('stanza', 11211)])
+            key_base = self.output_file
+            for k in sorted(self.to_memcached.keys(), key=lambda i:self.to_memcached[i].nbytes):
+                key = key_base + "_" + k
+                keys[k] = key
+                value = json.dumps(self.to_memcached[k], cls=NumpyEncoder)
+                rc[k] = mc.set(key, value)
+        self.log_warning(f"Return codes for memcached {rc}")
+        return keys
+

From d5555ffab5f80559624170d4bb3b3584065a0eac Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 18:03:42 +0200
Subject: [PATCH 21/98] missing import

---
 plugins/bm29/subtracte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index 8aada20..43a4277 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -41,7 +41,7 @@
 from .common import Sample, Ispyb, get_equivalent_frames, cmp_float, get_integrator, KeyCache, \
                     polarization_factor, method, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \
                     Sample, create_nexus_sample
-from .ispyb import IspybConnector
+from .ispyb import IspybConnector, NumpyEncoder
 
 try:
     import memcache

From 59eca02465797b2758e196921e591180fae8bea3 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 21 Apr 2022 18:21:55 +0200
Subject: [PATCH 22/98] minor cleanup

---
 plugins/bm29/subtracte.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index 43a4277..9eebb5e 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -249,8 +249,9 @@ def create_nexus(self):
         count_ds.attrs["interpretation"] = "image"
         count_ds.attrs["long_name"] = "Longest sequence where curves do not cross each other"
 
-        to_merge_ds = cormap_data.create_dataset("to_merge", data=numpy.arange(*tomerge, dtype=numpy.uint16))
-        self.log_warning(f"to_merge: {tomerge}")
+        to_merge_idx = numpy.arange(*tomerge, dtype=numpy.uint16)
+        to_merge_ds = cormap_data.create_dataset("to_merge", data=to_merge_idx)
+        # self.log_warning(f"to_merge: {tomerge}")
         to_merge_ds.attrs["long_name"] = "Index of equivalent frames"
         cormap_grp.attrs["default"] = cormap_data.name
 
@@ -268,10 +269,9 @@ def create_nexus(self):
         #  avg = Σdata / Σnorm
         #  var = sigma² = ΣV / Σnorm
         # TODO implement those math using numexpr:
-        sum_signal = sum([self.buffer_juices[i].normalization * self.buffer_juices[i].signal2d for i in range(*tomerge)])
-        sum_variance = sum([(self.buffer_juices[i].normalization * self.buffer_juices[i].error2d) ** 2 for i in range(*tomerge)])
-        norm = [self.buffer_juices[i].normalization for i in range(*tomerge)]
-        sum_norm = sum(norm)
+        sum_signal = sum(self.buffer_juices[i].normalization * self.buffer_juices[i].signal2d for i in to_merge_idx)
+        sum_variance = sum((self.buffer_juices[i].normalization * self.buffer_juices[i].error2d) ** 2 for i in to_merge_idx)
+        sum_norm = sum(self.buffer_juices[i].normalization for i in to_merge_idx)
         buffer_average = sum_signal / sum_norm
         buffer_variance = sum_variance / sum_norm
         sample_average = self.sample_juice.signal2d

From 3c25242a935ecaa26375e1a0dc57f6a8354f9b2a Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Tue, 14 Jun 2022 15:15:20 +0200
Subject: [PATCH 23/98] error in uncertainty propagation remainintg

---
 plugins/bm29/integrate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 1b48ef4..c3ec605 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "21/04/2022"
+__date__ = "02/05/2022"
 __status__ = "development"
 __version__ = "0.3.0"
 
@@ -582,6 +582,7 @@ def process3_average(self, tomerge):
         mask = self.ai.detector.mask
         sum_data = (self.input_frames[valid_slice]).sum(axis=0)
         sum_norm = (numpy.array(self.monitor_values)[valid_slice]).sum() * self.scale_factor
+        # TODO: There is an error at hte sqrt level ... to be investigated further   
         if numexpr is not None:
             # Numexpr is many-times faster than numpy when it comes to element-wise operations
             intensity_avg = numexpr.evaluate("where(mask==0, sum_data/sum_norm, 0.0)")

From ada3acdf047cb4e76a19315b616af2392a957ad5 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 17 Jun 2022 13:52:11 +0200
Subject: [PATCH 24/98] Update requirements.txt

---
 plugins/bm29/requirements.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/plugins/bm29/requirements.txt b/plugins/bm29/requirements.txt
index e9a494e..985b61c 100644
--- a/plugins/bm29/requirements.txt
+++ b/plugins/bm29/requirements.txt
@@ -1,5 +1,7 @@
 #This file contains the list of requirements for the BM29 plugin to work
 numpy
+scipy
+sklearn
 pyfai
 h5py
 hdf5plugin

From 38ac6bbe885bc6c98a4912f9e9996d7c07e9d5f6 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 17 Jun 2022 13:54:06 +0200
Subject: [PATCH 25/98] Update requirements.txt

---
 plugins/bm29/requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/bm29/requirements.txt b/plugins/bm29/requirements.txt
index 985b61c..f21b28a 100644
--- a/plugins/bm29/requirements.txt
+++ b/plugins/bm29/requirements.txt
@@ -1,8 +1,8 @@
 #This file contains the list of requirements for the BM29 plugin to work
 numpy
 scipy
-sklearn
-pyfai
+scikit-learn
+pyFAI
 h5py
 hdf5plugin
 freesas

From 2164d6239c448d2aad63cbd0dd357ee1bb4a5798 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Mon, 20 Jun 2022 11:02:16 +0200
Subject: [PATCH 26/98] fix formula for standard deviation assuming Poissonian
 noise

---
 plugins/bm29/integrate.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index c3ec605..b6e5320 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "02/05/2022"
+__date__ = "20/06/2022"
 __status__ = "development"
 __version__ = "0.3.0"
 
@@ -581,15 +581,15 @@ def process3_average(self, tomerge):
         valid_slice = slice(*tomerge)
         mask = self.ai.detector.mask
         sum_data = (self.input_frames[valid_slice]).sum(axis=0)
-        sum_norm = (numpy.array(self.monitor_values)[valid_slice]).sum() * self.scale_factor
-        # TODO: There is an error at hte sqrt level ... to be investigated further   
+        sum_norm = self.scale_factor * sum(self.monitor_values[valid_slice])    
         if numexpr is not None:
             # Numexpr is many-times faster than numpy when it comes to element-wise operations
             intensity_avg = numexpr.evaluate("where(mask==0, sum_data/sum_norm, 0.0)")
-            intensity_std = numexpr.evaluate("sqrt(intensity_avg)")
+            intensity_std = numexpr.evaluate("where(mask==0, sqrt(sum_data)/sum_norm, 0.0") # Assuming Poisson, no uncertainties on the diode
         else:
-            intensity_avg = sum_data / sum_norm
-            intensity_std = numpy.sqrt(intensity_avg)
+            with numpy.errstate(divide='ignore'):
+                intensity_avg = sum_data / sum_norm
+                intensity_std = numpy.sqrt(sum_data)/sum_norm
             wmask = numpy.where(mask)
             intensity_avg[wmask] = 0.0
             intensity_std[wmask] = 0.0

From e3501aa237e875e2f61c1cbbe92fdbd6177a15dd Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@stanza.esrf.fr>
Date: Mon, 20 Jun 2022 11:24:20 +0200
Subject: [PATCH 27/98] implement timeout

---
 plugins/bm29/integrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index be16e11..ee811fc 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -333,7 +333,7 @@ def create_nexus(self):
             frames_ds.attrs["interpretation"] = "image"
             measurement_grp["images"] = frames_ds
         else:  # use external links
-            with Nexus(self.input_file, "r") as nxsr:
+            with Nexus(self.input_file, "r", timeout=self.timeout) as nxsr:
                 entry = nxsr.get_entries()[0]
                 if "measurement" in entry:
                     measurement = entry["measurement"]

From 50f858b17ec383f1672e80c8188f7ce343bf499c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Mon, 20 Jun 2022 11:32:16 +0200
Subject: [PATCH 28/98] typo

---
 plugins/bm29/integrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index b6e5320..8c34747 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -585,7 +585,7 @@ def process3_average(self, tomerge):
         if numexpr is not None:
             # Numexpr is many-times faster than numpy when it comes to element-wise operations
             intensity_avg = numexpr.evaluate("where(mask==0, sum_data/sum_norm, 0.0)")
-            intensity_std = numexpr.evaluate("where(mask==0, sqrt(sum_data)/sum_norm, 0.0") # Assuming Poisson, no uncertainties on the diode
+            intensity_std = numexpr.evaluate("where(mask==0, sqrt(sum_data)/sum_norm, 0.0)") # Assuming Poisson, no uncertainties on the diode
         else:
             with numpy.errstate(divide='ignore'):
                 intensity_avg = sum_data / sum_norm

From 94624bbee6ca7b3e18e617e5c55f055c8b2c506f Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 14:31:34 +0200
Subject: [PATCH 29/98] save output data into processed directory

---
 plugins/bm29/hplc.py      | 12 +++++++++++-
 plugins/bm29/integrate.py | 12 +++++++++++-
 plugins/bm29/subtracte.py | 13 +++++++++++--
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index da7134b..8668101 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -175,7 +175,17 @@ def setup(self):
 
         self.output_file = self.input.get("output_file")
         if not self.output_file:
-            self.output_file = os.path.commonprefix(self.input_files) + "_hplc.h5"
+            dirname, basename = os.path.split(os.path.commonprefix(self.input_files) + "_hplc.h5")
+            dirname = os.path.dirname(dirname)
+#            dirname = os.path.join(dirname, "processed")
+            dirname = os.path.join(dirname, "hplc")
+            self.output_file = os.path.join(dirname, basename)
+            if not os.path.isdir(dirname):
+                try:
+                    os.makedirs(dirname)
+                except Exception as err:
+                    self.log_warning(f"Unable to create dir {dirname}. {type(err)}: {err}")
+
             self.log_warning("No output file provided, using " + self.output_file)
         self.nmf_components = int(self.input.get("nmf_components", self.NMF_COMP))
         self.ispyb = Ispyb._fromdict(self.input.get("ispyb", {}))
diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 8f825c4..dbe9e71 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -151,7 +151,17 @@ def setup(self, kwargs=None):
         if self.output_file is None:
             lst = list(os.path.splitext(self.input_file))
             lst.insert(1, "-integrate")
-            self.output_file = "".join(lst)
+            dirname, basename = os.path.split("".join(lst))
+            dirname = os.path.dirname(dirname)
+            dirname = os.path.join(dirname, "processed")
+            dirname = os.path.join(dirname, "integrate")
+            self.output_file = os.path.join(dirname, basename)
+            if not os.path.isdir(dirname):
+                try:
+                    os.makedirs(dirname)
+                except Exception as err:
+                    self.log_warning(f"Unable to create dir {dirname}. {type(err)}: {err}")
+
             self.log_warning(f"No output file provided, using: {self.output_file}")
         self.nb_frames = len(self.input.get("frame_ids", []))
         self.npt = self.input.get("npt", self.npt)
diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index 9eebb5e..9d764db 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -106,8 +106,17 @@ def setup(self, kwargs=None):
         if self.output_file is None:
             lst = list(os.path.splitext(self.sample_file))
             lst.insert(1, "-sub")
-            self.output_file = "".join(lst)
-            self.log_warning(f"No output file provided, using: {self.output_file}")
+            dirname, basename = os.path.split("".join(lst))
+            dirname = os.path.dirname(dirname)
+#            dirname = os.path.join(dirname, "processed")
+            dirname = os.path.join(dirname, "subtract")
+            self.output_file = os.path.join(dirname, basename)
+            if not os.path.isdir(dirname):
+                try:
+                    os.makedirs(dirname)
+                except Exception as err:
+                    self.log_warning(f"Unable to create dir {dirname}. {type(err)}: {err}")
+
         self.buffer_files = [os.path.abspath(fn) for fn in self.input.get("buffer_files", [])
                              if os.path.exists(fn)]
         self.ispyb = Ispyb._fromdict(self.input.get("ispyb", {}))

From 931edcd7f667453d42b85f4c0c240fba3b00c369 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:07:04 +0200
Subject: [PATCH 30/98] implement gallery into integrate (1 curve for now)

---
 plugins/bm29/common.py    |  1 +
 plugins/bm29/integrate.py | 13 ++++++++++++-
 plugins/bm29/ispyb.py     | 16 ++++++++++++----
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/plugins/bm29/common.py b/plugins/bm29/common.py
index 6f38b42..35edda1 100644
--- a/plugins/bm29/common.py
+++ b/plugins/bm29/common.py
@@ -115,6 +115,7 @@ class Ispyb(NamedTuple):
     url: str = None
     login: str = _default_passwd.get("username")
     passwd: str = _default_passwd.get("password")
+    gallery: str = ""
     pyarch: str = ""
 #     collection_id: int = -1 # This is now deprecated
 #     measurement_id: int = -1  # This is now deprecated
diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index dbe9e71..e59cb75 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -132,7 +132,6 @@ def setup(self, kwargs=None):
         logger.debug("IntegrateMultiframe.setup")
         Plugin.setup(self, kwargs)
 
-        self.ispyb = Ispyb._fromdict(self.input.get("ispyb", {}))
         self.sample = Sample._fromdict(self.input.get("sample", {}))
         if not self.sample.name:
             self.sample = Sample("Unknown sample", *self.sample[1:])
@@ -163,6 +162,18 @@ def setup(self, kwargs=None):
                     self.log_warning(f"Unable to create dir {dirname}. {type(err)}: {err}")
 
             self.log_warning(f"No output file provided, using: {self.output_file}")
+        #Manage gallery here
+        dirname = os.path.dirname(self.output_file)
+        gallery = os.path.join(dirname, "gallery")
+        if not isdir(gallery):
+            try:
+                os.makedirs(gallery)
+            except Exception as err:
+                self.log_warning(f"Unable to create dir {gallery}. {type(err)}: {err}")
+        ispydict = self.input.get("ispyb", {})
+        ispydict["gallery"] = gallery
+        self.ispyb = Ispyb._fromdict(ispydict)
+
         self.nb_frames = len(self.input.get("frame_ids", []))
         self.npt = self.input.get("npt", self.npt)
         self.unit = pyFAI.units.to_unit(self.input.get("unit", self.unit))
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 5cfcc09..65d2d13 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -20,6 +20,7 @@
 import os
 import shutil
 import json
+import tempfile
 import numpy
 from suds.client import Client
 from suds.transport.https import HttpAuthenticated
@@ -45,7 +46,7 @@ def str_list(lst):
 class IspybConnector:
     "This class is a conector to the web-service"
 
-    def __init__(self, url, login=None, passwd=None, pyarch=None,
+    def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
                  experiment_id=-1, run_number=-1, **kwargs):
         """Constructor of the ISPyB connections
 
@@ -59,6 +60,11 @@ def __init__(self, url, login=None, passwd=None, pyarch=None,
         self._url = url
         self.authentication = HttpAuthenticated(username=login, password=passwd)
         self.client = Client(url, transport=self.authentication, cache=None)
+        if gallery:
+            self.gallery = os.path.abspath(gallery)
+        else:
+            logger.error("No `gallery` destination provided ... things will go wrong")
+            self.gallery = tempfile.gettempdir()
         if pyarch:
             self.pyarch = os.path.abspath(pyarch)
         else:
@@ -151,13 +157,15 @@ def guinier_plot(self, sasm, guinier, basename="frame"):
         return filename
 
     def scatter_plot(self, sasm, guinier, ift, basename="frame"):
-        filename = self._mk_filename("Scattering", "plot", basename, ext=".png")
+        pyarch_fn = self._mk_filename("Scattering", "plot", basename, ext=".png")
+        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
         fig = scatter_plot(sasm, guinier, ift,
-                           filename=filename, img_format="png", unit="nm",
+                           filename=gallery_fn, img_format="png", unit="nm",
                            title="Scattering curve ",
                            ax=None, labelsize=None, fontsize=None)
         matplotlib.pyplot.close(fig)
-        return filename
+        shutil.copyfile(gallery_fn,pyarch_fn)
+        return pyarch_fn
 
     def density_plot(self, ift, basename="frame"):
         filename = self._mk_filename("Density", "plot", basename, ext=".png")

From 6064a522923fbf4ab5ee7dae5da93fe1e771754e Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:09:23 +0200
Subject: [PATCH 31/98] typo

---
 plugins/bm29/integrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index e59cb75..dcbca8c 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -165,7 +165,7 @@ def setup(self, kwargs=None):
         #Manage gallery here
         dirname = os.path.dirname(self.output_file)
         gallery = os.path.join(dirname, "gallery")
-        if not isdir(gallery):
+        if not os.path.isdir(gallery):
             try:
                 os.makedirs(gallery)
             except Exception as err:

From 557b835e9d96d4621f35588239477e7d460461d1 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:24:28 +0200
Subject: [PATCH 32/98] save thumbnail of scatering curve

---
 plugins/bm29/integrate.py | 6 +++---
 plugins/bm29/ispyb.py     | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index dcbca8c..53da40d 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -233,8 +233,7 @@ def process(self):
         self.ai = get_integrator(KeyCache(self.npt, self.unit, self.poni, self.mask, self.energy))
         self.create_nexus()
         self.output["memcached"] = self.send_to_memcached()
-        if not self.input.get("hplc_mode"):
-            self.send_to_ispyb()
+        self.send_to_ispyb()
 
     def wait_file(self, filename, timeout=None):
         """Wait for a file to appear on a filesystem
@@ -619,7 +618,8 @@ def process3_average(self, tomerge):
     def send_to_ispyb(self):
         if self.ispyb.url and parse_url(self.ispyb.url).host:
             ispyb = IspybConnector(*self.ispyb)
-            ispyb.send_averaged(self.to_pyarch)
+            if not self.input.get("hplc_mode"):
+                ispyb.send_averaged(self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 65d2d13..9d0ed89 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -105,6 +105,8 @@ def send_averaged(self, data):
                                         str_list(frames),
                                         str_list(discarded),
                                         str(averaged))
+        sasm = numpy.vstack((aver_data.radial, aver_data.intensity, aver_data.sigma)).T
+        scatterPlot = self.scatter_plot(sasm, basename=basename)
 
     def _mk_filename(self, index, path, basename="frame", ext=".dat"):
         dest = os.path.join(self.pyarch, path)

From 7085a8f20bb5bb46ed2847704df26dfd8ccb0cf0 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:27:06 +0200
Subject: [PATCH 33/98] typo

---
 plugins/bm29/ispyb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 9d0ed89..5ca24fd 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -158,7 +158,7 @@ def guinier_plot(self, sasm, guinier, basename="frame"):
         matplotlib.pyplot.close(fig)
         return filename
 
-    def scatter_plot(self, sasm, guinier, ift, basename="frame"):
+    def scatter_plot(self, sasm, guinier=None, ift=None, basename="frame"):
         pyarch_fn = self._mk_filename("Scattering", "plot", basename, ext=".png")
         gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
         fig = scatter_plot(sasm, guinier, ift,

From 7c5fd4939631972262e0624f62d8a507050e0910 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:33:17 +0200
Subject: [PATCH 34/98] save plots IFT+guinier+Kratky

---
 plugins/bm29/ispyb.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 5ca24fd..db1044d 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -142,20 +142,24 @@ def save_bift(self, bift, basename="frame"):
         return filename
 
     def kratky_plot(self, sasm, guinier, basename="frame"):
-        filename = self._mk_filename("Kratky", "plot", basename, ext=".png")
+        pyarch_fn = self._mk_filename("Kratky", "plot", basename, ext=".png")
+        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
         fig = kratky_plot(sasm, guinier,
-                           filename=filename, img_format="png", unit="nm",
+                           filename=gallery_fn, img_format="png", unit="nm",
                            title="Dimensionless Kratky plot",
                            ax=None, labelsize=None, fontsize=None)
         matplotlib.pyplot.close(fig)
-        return filename
+        shutil.copyfile(gallery_fn, pyarch_fn)
+        return pyarch_fn
 
     def guinier_plot(self, sasm, guinier, basename="frame"):
-        filename = self._mk_filename("Guinier", "plot", basename, ext=".png")
-        fig = guinier_plot(sasm, guinier, filename=filename,
+        pyarch_fn = self._mk_filename("Guinier", "plot", basename, ext=".png")
+        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
+        fig = guinier_plot(sasm, guinier, filename=gallery_fn,
                             img_format="png", unit="nm",
                             ax=None, labelsize=None, fontsize=None)
         matplotlib.pyplot.close(fig)
+        shutil.copyfile(gallery_fn, pyarch_fn)
         return filename
 
     def scatter_plot(self, sasm, guinier=None, ift=None, basename="frame"):
@@ -166,15 +170,17 @@ def scatter_plot(self, sasm, guinier=None, ift=None, basename="frame"):
                            title="Scattering curve ",
                            ax=None, labelsize=None, fontsize=None)
         matplotlib.pyplot.close(fig)
-        shutil.copyfile(gallery_fn,pyarch_fn)
+        shutil.copyfile(gallery_fn, pyarch_fn)
         return pyarch_fn
 
     def density_plot(self, ift, basename="frame"):
-        filename = self._mk_filename("Density", "plot", basename, ext=".png")
-        fig = density_plot(ift, filename=filename, img_format="png", unit="nm",
+        pyarch_fn = self._mk_filename("Density", "plot", basename, ext=".png")
+        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
+        fig = density_plot(ift, filename=gallery_fn, img_format="png", unit="nm",
                      ax=None, labelsize=None, fontsize=None)
         matplotlib.pyplot.close(fig)
-        return filename
+        shutil.copyfile(gallery_fn, pyarch_fn)
+        return pyarch_fn
 
     def send_subtracted(self, data):
         """send the result of the subtraction to Ispyb

From 46cb022658dbd1dbebff1cb240ecf0604c787150 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:35:40 +0200
Subject: [PATCH 35/98] update subtract to provide gallery

---
 plugins/bm29/subtracte.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index 9d764db..a88c365 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -119,7 +119,17 @@ def setup(self, kwargs=None):
 
         self.buffer_files = [os.path.abspath(fn) for fn in self.input.get("buffer_files", [])
                              if os.path.exists(fn)]
-        self.ispyb = Ispyb._fromdict(self.input.get("ispyb", {}))
+        #Manage gallery here
+        dirname = os.path.dirname(self.output_file)
+        gallery = os.path.join(dirname, "gallery")
+        if not os.path.isdir(gallery):
+            try:
+                os.makedirs(gallery)
+            except Exception as err:
+                self.log_warning(f"Unable to create dir {gallery}. {type(err)}: {err}")
+        ispydict = self.input.get("ispyb", {})
+        ispydict["gallery"] = gallery
+        self.ispyb = Ispyb._fromdict(ispydict)
 
     def teardown(self):
         Plugin.teardown(self)

From 35aa5a70b7e9d016a6fd846a4b48c4b2ab069382 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:40:33 +0200
Subject: [PATCH 36/98] typo

---
 plugins/bm29/ispyb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index db1044d..b6ad36c 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -160,7 +160,7 @@ def guinier_plot(self, sasm, guinier, basename="frame"):
                             ax=None, labelsize=None, fontsize=None)
         matplotlib.pyplot.close(fig)
         shutil.copyfile(gallery_fn, pyarch_fn)
-        return filename
+        return pyarch_fn
 
     def scatter_plot(self, sasm, guinier=None, ift=None, basename="frame"):
         pyarch_fn = self._mk_filename("Scattering", "plot", basename, ext=".png")

From 94fbd9e979cd8213a723244cf375be0016aaf1c6 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 14 Sep 2022 15:56:14 +0200
Subject: [PATCH 37/98] implement saving of curves

---
 plugins/bm29/ispyb.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index b6ad36c..18891a0 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -125,10 +125,12 @@ def save_curve(self, index, integrate_result, basename="frame"):
         :param: integrate_result: an IntegrationResult to be saved
         :return: the full path of the file in pyarch
         """
-        filename = self._mk_filename(index, "1d", basename)
+        pyarch_fn = self._mk_filename(index, "1d", basename)
+        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
         sasl = numpy.vstack((integrate_result.radial, integrate_result.intensity, integrate_result.sigma))
-        numpy.savetxt(filename, sasl.T)
-        return filename
+        numpy.savetxt(gallery_fn, sasl.T)
+        shutil.copyfile(gallery_fn, pyarch_fn)
+        return pyarch_fn
 
     def save_bift(self, bift, basename="frame"):
         """Save a  IFT curve into the pyarch. Not those file do not exist outside pyarch
@@ -137,9 +139,11 @@ def save_bift(self, bift, basename="frame"):
         :param: bift: an StatResults object to be saved (freesas >= 0.8.4)
         :return: the full path of the file in pyarch
         """
-        filename = self._mk_filename("BIFT", "plot", basename, ext=".out")
-        bift.save(filename)
-        return filename
+        pyarch_fn = self._mk_filename("BIFT", "1d", basename, ext=".out")
+        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
+        bift.save(gallery_fn)
+        shutil.copyfile(gallery_fn, pyarch_fn)
+        return pyarch_fn
 
     def kratky_plot(self, sasm, guinier, basename="frame"):
         pyarch_fn = self._mk_filename("Kratky", "plot", basename, ext=".png")

From f2461303a948a8f6b844cb5533769152e4890fe0 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 15 Sep 2022 16:10:42 +0200
Subject: [PATCH 38/98] polish ... should be working now.

---
 plugins/bm29/hplc.py  | 25 ++++++++++++++++++-------
 plugins/bm29/ispyb.py | 29 +++++++++++++++++++++++------
 2 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index 8668101..5d30bfd 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -10,9 +10,9 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "03/09/2021"
+__date__ = "15/09/2022"
 __status__ = "development"
-__version__ = "0.1.2"
+__version__ = "0.2.0"
 
 import time
 import os
@@ -23,12 +23,12 @@
 from collections import namedtuple
 from urllib3.util import parse_url
 from dahu.plugin import Plugin
-from dahu.utils import fully_qualified_name
+# from dahu.utils import fully_qualified_name
 import logging
 logger = logging.getLogger("bm29.hplc")
 import numpy
 import h5py
-import pyFAI, pyFAI.azimuthalIntegrator
+import pyFAI, pyFAI.azimuthalIntegrator, pyFAI.units
 from pyFAI.method_registry import IntegrationMethod
 import freesas, freesas.cormap, freesas.invariants
 from freesas.autorg import auto_gpa, autoRg, auto_guinier
@@ -141,7 +141,7 @@ class HPLC(Plugin):
         "measurement_id": -1,
         "collection_id": -1
        },
-       "nmf_components = 5, 
+       "nmf_components": 5, 
       "wait_for": [jobid_img001, jobid_img002],
       "plugin_name": "bm29.hplc"
     } 
@@ -188,7 +188,18 @@ def setup(self):
 
             self.log_warning("No output file provided, using " + self.output_file)
         self.nmf_components = int(self.input.get("nmf_components", self.NMF_COMP))
-        self.ispyb = Ispyb._fromdict(self.input.get("ispyb", {}))
+
+        #Manage gallery here
+        dirname = os.path.dirname(self.output_file)
+        gallery = os.path.join(dirname, "gallery")
+        if not os.path.isdir(gallery):
+            try:
+                os.makedirs(gallery)
+            except Exception as err:
+                self.log_warning(f"Unable to create dir {gallery}. {type(err)}: {err}")
+        ispydict = self.input.get("ispyb", {})
+        ispydict["gallery"] = gallery
+        self.ispyb = Ispyb._fromdict(ispydict)
 
     def process(self):
         self.create_nexus()
@@ -403,7 +414,7 @@ def create_nexus(self):
         fraction_grp["minimum_size"] = window = 10
 
         fractions, nfractions = search_peaks(Isum, window)
-        self.to_pyarch["merge_frames"] = numpy.zeros((nfractions, 2), dtype=numpy.float32)
+        self.to_pyarch["merge_frames"] = numpy.zeros((nfractions, 2), dtype=numpy.int32)
         self.to_pyarch["merge_I"] = numpy.zeros((nfractions, nbin), dtype=numpy.float32)
         self.to_pyarch["merge_Stdev"] = numpy.zeros((nfractions, nbin), dtype=numpy.float32)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 18891a0..8d108e5 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -11,9 +11,9 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "03/06/2021"
+__date__ = "15/09/2022"
 __status__ = "development"
-version = "0.1.1"
+version = "0.2.0"
 
 import logging
 logger = logging.getLogger("bm29.ispyb")
@@ -27,7 +27,7 @@
 import matplotlib.pyplot
 matplotlib.use("Agg")
 from freesas.collections import RG_RESULT, RT_RESULT, StatsResult
-from freesas.plot import kratky_plot, guinier_plot, scatter_plot, density_plot
+from freesas.plot import kratky_plot, guinier_plot, scatter_plot, density_plot, hplc_plot
 
 
 class NumpyEncoder(json.JSONEncoder):
@@ -58,8 +58,11 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
         :param run_number: identify the run in an experiment (sample/buffer localization)
         """
         self._url = url
-        self.authentication = HttpAuthenticated(username=login, password=passwd)
-        self.client = Client(url, transport=self.authentication, cache=None)
+        try:
+            self.authentication = HttpAuthenticated(username=login, password=passwd)
+            self.client = Client(url, transport=self.authentication, cache=None)
+        except Exception as err:
+            logger.error(f"Connection to the web-service {type(err)}:{err}")
         if gallery:
             self.gallery = os.path.abspath(gallery)
         else:
@@ -249,8 +252,22 @@ def send_hplc(self, data):
 
         :param data: a dict with all information to be saved in Ispyb
         """
+        sample = data.get("sample_name", "sample")
+        #gallery
+        gallery = os.path.join(self.gallery, f'{sample}_hplc.png')
+        chromatogram = data.get("sum_I")
+        
+        if chromatogram is not None:
+            fractions = data.get("merge_frames")
+            if fractions is not None:
+                fractions.sort()
+            hplc_plot(chromatogram, fractions,
+                      title=f"Chromatogram of {sample}", 
+                      filename=gallery, 
+                      img_format="png", )
+
         hdf5_file = data.get("hdf5_filename")
-        filename = self._mk_filename("hplc", ".", data.get("sample_name", "sample"), ext=".h5")
+        filename = self._mk_filename("hplc", ".", sample, ext=".h5")
         filename = os.path.abspath(filename)
         json_file = os.path.splitext(filename)[0] + ".json"
         with open(json_file, mode="w") as w:

From ce6a92d6d2d3bbc61c39530c72f87b6a78d00677 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 15 Sep 2022 16:11:32 +0200
Subject: [PATCH 39/98] should break if ispyb is not referenced

---
 plugins/bm29/ispyb.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 8d108e5..2a0dcf6 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -58,11 +58,11 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
         :param run_number: identify the run in an experiment (sample/buffer localization)
         """
         self._url = url
-        try:
-            self.authentication = HttpAuthenticated(username=login, password=passwd)
-            self.client = Client(url, transport=self.authentication, cache=None)
-        except Exception as err:
-            logger.error(f"Connection to the web-service {type(err)}:{err}")
+        # try:
+        self.authentication = HttpAuthenticated(username=login, password=passwd)
+        self.client = Client(url, transport=self.authentication, cache=None)
+        # except Exception as err:
+        #     logger.error(f"Connection to the web-service {type(err)}:{err}")
         if gallery:
             self.gallery = os.path.abspath(gallery)
         else:

From 8b8d0d27b9b26e06bc3f87fea236d5430c68b4de Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 15 Sep 2022 16:50:14 +0200
Subject: [PATCH 40/98] debug

---
 plugins/bm29/hplc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index 5d30bfd..e1ffe8c 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -200,6 +200,7 @@ def setup(self):
         ispydict = self.input.get("ispyb", {})
         ispydict["gallery"] = gallery
         self.ispyb = Ispyb._fromdict(ispydict)
+        print(self.ispyb)
 
     def process(self):
         self.create_nexus()

From 298b316c54d863e814ea8c30d91688bf65d7d1b3 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 08:56:22 +0200
Subject: [PATCH 41/98] clean-up

---
 plugins/bm29/hplc.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index e1ffe8c..75a1aa8 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -10,7 +10,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "15/09/2022"
+__date__ = "16/09/2022"
 __status__ = "development"
 __version__ = "0.2.0"
 
@@ -200,7 +200,6 @@ def setup(self):
         ispydict = self.input.get("ispyb", {})
         ispydict["gallery"] = gallery
         self.ispyb = Ispyb._fromdict(ispydict)
-        print(self.ispyb)
 
     def process(self):
         self.create_nexus()

From 982f21094a58f725398ab6e04fd721b04e81f9bd Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 15:48:28 +0200
Subject: [PATCH 42/98] start to upload data to icat

---
 plugins/bm29/integrate.py     |  3 +-
 plugins/bm29/ispyb.py         | 62 +++++++++++++++++++++++++++++++++--
 plugins/bm29/requirements.txt |  1 +
 3 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 53da40d..1c993e8 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "20/06/2022"
+__date__ = "16/09/2022"
 __status__ = "development"
 __version__ = "0.3.0"
 
@@ -620,6 +620,7 @@ def send_to_ispyb(self):
             ispyb = IspybConnector(*self.ispyb)
             if not self.input.get("hplc_mode"):
                 ispyb.send_averaged(self.to_pyarch)
+                ispyb.icat_client(data=self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 2a0dcf6..91bba08 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "15/09/2022"
+__date__ = "16/09/2022"
 __status__ = "development"
 version = "0.2.0"
 
@@ -24,6 +24,7 @@
 import numpy
 from suds.client import Client
 from suds.transport.https import HttpAuthenticated
+from pyicat_plus.client.main import IcatClient
 import matplotlib.pyplot
 matplotlib.use("Agg")
 from freesas.collections import RG_RESULT, RT_RESULT, StatsResult
@@ -63,6 +64,7 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
         self.client = Client(url, transport=self.authentication, cache=None)
         # except Exception as err:
         #     logger.error(f"Connection to the web-service {type(err)}:{err}")
+
         if gallery:
             self.gallery = os.path.abspath(gallery)
         else:
@@ -75,10 +77,62 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
 
         self.experiment_id = experiment_id
         self.run_number = run_number
+        
 
     def __repr__(self):
         return f"Ispyb connector to {self._url}"
 
+    def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, path=None, raw=None,  data=None):
+        """
+        :param proposal: mx1324
+        :param beamline: name of the beamline
+        :param sample: sample name as registered in icat
+        :param dataset: name given by BLISS
+        :param path: directory name where processed data are staying
+        :param raw: directory name of the raw data (not the processed ones)
+        :param data: dict with all data sent to ISpyB
+        """
+        tmp = self.gallery.strip("/").split("/")
+        idx_process = [i for i,j in enumerate(tmp) if j.lower().startswith("process")][-1]
+        assert idx_process>5
+        if proposal is None:
+            proposal = tmp[idx_process-5]
+        if beamline is None:
+            beamline = tmp[idx_process-4]
+        if sample is None:
+            sample = tmp[idx_process-2]
+        if dataset is None:
+            dataset = tmp[idx_process-1]
+            if len(dataset) > len(sample):
+                dataset = dataset[len(sample)+1:]
+        if path is None:
+            path = os.path.dirname(self.gallery)
+        if raw is None:            
+            raw = os.path.abspath(self.gallery[:self.gallery.lower().index("process")])            
+        
+        metadata = {"definition": "SAXS",
+                    "Sample_name": sample}
+        guinier = data.get("guinier")
+        if guinier:
+            metadata["SAXS_guinier_rg"] = f"{guinier.Rg:.1f}±{guinier.sigma_Rg:.1f}"
+            metadata["SAXS_guinier_points"] = f"{guinier.start_point}-{guinier.end_point}"
+            metadata["SAXS_guinier_i0"] = f"{guinier.I0:.1f}±{guinier.sigma_I0:.1f}"
+        bift = data.get("bift")
+        if bift:
+            metadata["SAXS_rg"] =  f"{bift.Rg_avg:.1f}±{bift.Rg_std:.1f}"
+            metadata["SAXS_d_max"] = f"{bift.Dmax_avg:.1f}±{bift.Dmax_std:.1f}"
+        tomerge = data.get("to_merge")
+        if tomerge:
+            metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
+        
+        icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
+        icat_client.store_processed_data(beamline=beamline, 
+                                         proposal=proposal, 
+                                         dataset=dataset, 
+                                         path=path, 
+                                         metadata=metadata, 
+                                         raw=[raw])
+
     def send_averaged(self, data):
         """Send this to ISPyB and backup to PyArch
 
@@ -109,7 +163,11 @@ def send_averaged(self, data):
                                         str_list(discarded),
                                         str(averaged))
         sasm = numpy.vstack((aver_data.radial, aver_data.intensity, aver_data.sigma)).T
-        scatterPlot = self.scatter_plot(sasm, basename=basename)
+        self.scatter_plot(sasm, basename=basename)
+        
+        
+        
+        
 
     def _mk_filename(self, index, path, basename="frame", ext=".dat"):
         dest = os.path.join(self.pyarch, path)
diff --git a/plugins/bm29/requirements.txt b/plugins/bm29/requirements.txt
index f21b28a..52e1855 100644
--- a/plugins/bm29/requirements.txt
+++ b/plugins/bm29/requirements.txt
@@ -8,3 +8,4 @@ hdf5plugin
 freesas
 suds-jurko
 memcache
+pyicat-plus

From 6f1ad8b4cf9ef6d7c01f1d747051ce5803e5fa99 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 15:50:31 +0200
Subject: [PATCH 43/98] typo

---
 plugins/bm29/integrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 1c993e8..5458a98 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -620,7 +620,7 @@ def send_to_ispyb(self):
             ispyb = IspybConnector(*self.ispyb)
             if not self.input.get("hplc_mode"):
                 ispyb.send_averaged(self.to_pyarch)
-                ispyb.icat_client(data=self.to_pyarch)
+                ispyb.send_icat(data=self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 

From 1d0564b215acf83b7464c9e146cbe8d24991a3ee Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 15:56:37 +0200
Subject: [PATCH 44/98] more debug

---
 plugins/bm29/ispyb.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 91bba08..9131259 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -126,12 +126,14 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
             metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
         
         icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
-        icat_client.store_processed_data(beamline=beamline, 
-                                         proposal=proposal, 
-                                         dataset=dataset, 
-                                         path=path, 
-                                         metadata=metadata, 
-                                         raw=[raw])
+        kwargs = {"beamline":beamline, 
+                  "proposal":proposal, 
+                  "dataset":dataset, 
+                  "path":path, 
+                  "metadata":metadata, 
+                  "raw":[raw]}
+        print(json.dumps(kwargs))
+        icat_client.store_processed_data(**kwargs)
 
     def send_averaged(self, data):
         """Send this to ISPyB and backup to PyArch

From 63df928d9670b9cbc9256bd615c2c737aa972bf5 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 15:59:43 +0200
Subject: [PATCH 45/98] fix tomerge

---
 plugins/bm29/ispyb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 9131259..c43a335 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -121,7 +121,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         if bift:
             metadata["SAXS_rg"] =  f"{bift.Rg_avg:.1f}±{bift.Rg_std:.1f}"
             metadata["SAXS_d_max"] = f"{bift.Dmax_avg:.1f}±{bift.Dmax_std:.1f}"
-        tomerge = data.get("to_merge")
+        tomerge = data.get("merged")
         if tomerge:
             metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
         

From d75e8218268e05cabe93e6bf8fe2b45a45f06c8c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 16:23:17 +0200
Subject: [PATCH 46/98] some more metadata

---
 plugins/bm29/integrate.py |  8 ++++++--
 plugins/bm29/ispyb.py     | 28 ++++++++++++++++++++++++++--
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 5458a98..d92a53d 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -618,9 +618,13 @@ def process3_average(self, tomerge):
     def send_to_ispyb(self):
         if self.ispyb.url and parse_url(self.ispyb.url).host:
             ispyb = IspybConnector(*self.ispyb)
-            if not self.input.get("hplc_mode"):
+            if self.input.get("hplc_mode"):
+                self.to_pyarch["experiment_type"]="hplc"
+            else:
                 ispyb.send_averaged(self.to_pyarch)
-                ispyb.send_icat(data=self.to_pyarch)
+                self.to_pyarch["experiment_type"]="sample-changer"
+            self.to_pyarch["sample"] = self.sample
+            ispyb.send_icat(data=self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index c43a335..449b305 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -124,7 +124,31 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         tomerge = data.get("merged")
         if tomerge:
             metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
-        
+        #Other metadata one may collect ...
+        metadata["SAXS_experimentType"]= data.get("experiment_type", "") 
+        #SAXS_maskFile
+        #SAXS_numberFrames
+        #SAXS_timePerFrame
+        #SAXS_detector_distance
+        #SAXS_waveLength
+        #SAXS_pixelSizeX
+        #SAXS_pixelSizeY
+        #SAXS_beam_center_x
+        #SAXS_beam_center_y
+        #SAXS_normalisation
+        #SAXS_diode_currents
+        #SAXS_porod_volume
+        #
+        sample = data.get("sample")
+        if sample:
+            metadata["SAXS_concentration"] = sample.concentration
+            metadata["SAXS_code"] = sample.name
+            metadata["SAXS_comments"] = sample.description
+            metadata["SAXS_storage_temperature"] = sample.temperature_env
+            metadata["SAXS_exposure_temperature"] = sample.temperature
+            metadata["SAXS_column_type"] = sample.hplc
+            #"buffer": "description of buffer, pH, ...",
+
         icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
         kwargs = {"beamline":beamline, 
                   "proposal":proposal, 
@@ -132,7 +156,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
                   "path":path, 
                   "metadata":metadata, 
                   "raw":[raw]}
-        print(json.dumps(kwargs))
+        print("Sent to iCAT:",json.dumps(kwargs, indent=2))
         icat_client.store_processed_data(**kwargs)
 
     def send_averaged(self, data):

From 42494d329f8e2e76b964ccff0199bc30a99a6eb8 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 16:40:23 +0200
Subject: [PATCH 47/98] do not duplicate all files

---
 plugins/bm29/ispyb.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 449b305..c64c63b 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -146,7 +146,8 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
             metadata["SAXS_comments"] = sample.description
             metadata["SAXS_storage_temperature"] = sample.temperature_env
             metadata["SAXS_exposure_temperature"] = sample.temperature
-            metadata["SAXS_column_type"] = sample.hplc
+            if sample.hplc:
+                metadata["SAXS_column_type"] = sample.hplc
             #"buffer": "description of buffer, pH, ...",
 
         icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
@@ -174,11 +175,11 @@ def send_averaged(self, data):
         merged = data.pop("merged")
 
         aver_data = data.pop("avg")
-        averaged = self.save_curve("avg", aver_data, basename)
+        averaged = self.save_curve("avg", aver_data, basename, gallery=True)
         list_merged = list(range(*merged))
         for k, v in data.items():
             if isinstance(k, int):
-                fn = self.save_curve(k, v, basename)
+                fn = self.save_curve(k, v, basename, gallery=False)
                 if k in list_merged:
                     frames.append(fn)
                 else:
@@ -205,18 +206,20 @@ def _mk_filename(self, index, path, basename="frame", ext=".dat"):
             filename = os.path.join(dest, "%s_%s%s" % (basename, index, ext))
         return filename
 
-    def save_curve(self, index, integrate_result, basename="frame"):
+    def save_curve(self, index, integrate_result, basename="frame", gallery=False):
         """Save a  1D curve into the pyarch. Not those file do not exist outside pyarch
 
         :param: index: prefix or index value for
         :param: integrate_result: an IntegrationResult to be saved
+        :param gallery: make a copy to the gallery
         :return: the full path of the file in pyarch
         """
         pyarch_fn = self._mk_filename(index, "1d", basename)
-        gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
         sasl = numpy.vstack((integrate_result.radial, integrate_result.intensity, integrate_result.sigma))
-        numpy.savetxt(gallery_fn, sasl.T)
-        shutil.copyfile(gallery_fn, pyarch_fn)
+        numpy.savetxt(pyarch_fn, sasl.T)
+        if gallery:
+            gallery_fn = os.path.join(self.gallery, os.path.basename(pyarch_fn))
+            shutil.copyfile(pyarch_fn, gallery_fn)
         return pyarch_fn
 
     def save_bift(self, bift, basename="frame"):

From d7f8bc25ccd6b1f4491058b369d5c5b9304e5d50 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 16:42:23 +0200
Subject: [PATCH 48/98] do not mangle data

---
 plugins/bm29/ispyb.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index c64c63b..249d23e 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -169,6 +169,7 @@ def send_averaged(self, data):
                 "merged": list of index merged
                 0,1,2,3 the different indexes for individual frames.
         """
+        data = data.copy()
         basename = data.pop("basename")
         discarded = []
         frames = []

From 4779c7f636404845251eae7d697f7c4e5f0012d6 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:18:56 +0200
Subject: [PATCH 49/98] much more metadata

---
 plugins/bm29/integrate.py | 14 ++++++++++++
 plugins/bm29/ispyb.py     | 45 ++++++++++++++++++---------------------
 plugins/bm29/subtracte.py |  5 ++++-
 3 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index d92a53d..45481f4 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -623,7 +623,21 @@ def send_to_ispyb(self):
             else:
                 ispyb.send_averaged(self.to_pyarch)
                 self.to_pyarch["experiment_type"]="sample-changer"
+            #Some more metadata for iCat, as strings:
             self.to_pyarch["sample"] = self.sample
+            self.to_pyarch["SAXS_maskFile"] = self.mask
+            self.to_pyarch["SAXS_waveLength"] = str(self.ai.wavelength)
+            self.to_pyarch["SAXS_normalisation"] = str(self.normalization_factor)
+            self.to_pyarch["SAXS_diode_currents"] = str(self.monitor_values)
+            self.to_pyarch["SAXS_numberFrames"] = str(self.nb_frames)
+            self.to_pyarch["SAXS_timePerFrame"] = self.input.get("exposure_time", "?")
+            self.to_pyarch["SAXS_detector_distance"] = str(self.ai.dist)
+            self.to_pyarch["SAXS_pixelSizeX"] = str(self.ai.detector.pixel2)
+            self.to_pyarch["SAXS_pixelSizeY"] = str(self.ai.detector.pixel1)
+            f2d = self.ai.getFit2D()
+            self.to_pyarch["SAXS_beam_center_x"] = str(f2d["centerX"])
+            self.to_pyarch["SAXS_beam_center_x"] = str(f2d["centerY"])
+
             ispyb.send_icat(data=self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 249d23e..42d15a9 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -112,43 +112,40 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         
         metadata = {"definition": "SAXS",
                     "Sample_name": sample}
+        for k,v in data.items():
+            if k.startswith("SAXS_"):
+                metadata[k] = v
+        sample = data.get("sample")
+        if sample:
+            metadata["SAXS_concentration"] = sample.concentration
+            metadata["SAXS_code"] = sample.name
+            metadata["SAXS_comments"] = sample.description
+            metadata["SAXS_storage_temperature"] = sample.temperature_env
+            metadata["SAXS_exposure_temperature"] = sample.temperature
+            if sample.hplc:
+                metadata["SAXS_column_type"] = sample.hplc
+            #"buffer": "description of buffer, pH, ...",
+
         guinier = data.get("guinier")
         if guinier:
             metadata["SAXS_guinier_rg"] = f"{guinier.Rg:.1f}±{guinier.sigma_Rg:.1f}"
             metadata["SAXS_guinier_points"] = f"{guinier.start_point}-{guinier.end_point}"
             metadata["SAXS_guinier_i0"] = f"{guinier.I0:.1f}±{guinier.sigma_I0:.1f}"
+
         bift = data.get("bift")
         if bift:
             metadata["SAXS_rg"] =  f"{bift.Rg_avg:.1f}±{bift.Rg_std:.1f}"
             metadata["SAXS_d_max"] = f"{bift.Dmax_avg:.1f}±{bift.Dmax_std:.1f}"
+
         tomerge = data.get("merged")
         if tomerge:
             metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
+        
+        volume = data.get("volume")
+        if volume:
+            metadata["SAXS_porod_volume"] = volume 
         #Other metadata one may collect ...
-        metadata["SAXS_experimentType"]= data.get("experiment_type", "") 
-        #SAXS_maskFile
-        #SAXS_numberFrames
-        #SAXS_timePerFrame
-        #SAXS_detector_distance
-        #SAXS_waveLength
-        #SAXS_pixelSizeX
-        #SAXS_pixelSizeY
-        #SAXS_beam_center_x
-        #SAXS_beam_center_y
-        #SAXS_normalisation
-        #SAXS_diode_currents
-        #SAXS_porod_volume
-        #
-        sample = data.get("sample")
-        if sample:
-            metadata["SAXS_concentration"] = sample.concentration
-            metadata["SAXS_code"] = sample.name
-            metadata["SAXS_comments"] = sample.description
-            metadata["SAXS_storage_temperature"] = sample.temperature_env
-            metadata["SAXS_exposure_temperature"] = sample.temperature
-            if sample.hplc:
-                metadata["SAXS_column_type"] = sample.hplc
-            #"buffer": "description of buffer, pH, ...",
+        metadata["SAXS_experimentType"]= data.get("experiment_type", "")         
 
         icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
         kwargs = {"beamline":beamline, 
diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index a88c365..793702a 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "21/04/2022"
+__date__ = "16/09/2022"
 __status__ = "development"
 __version__ = "0.2.0"
 
@@ -731,6 +731,9 @@ def send_to_ispyb(self):
         if self.ispyb.url and parse_url(self.ispyb.url).host:
             ispyb = IspybConnector(*self.ispyb)
             ispyb.send_subtracted(self.to_pyarch)
+            self.to_pyarch["experiment_type"]="sample-changer"
+            self.to_pyarch["sample"] = self.sample_juice.sample
+            ispyb.send_icat(data=self.to_pyarch)
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 

From 8e64003cfe505baf91ca42de7adf23a649a9688b Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:20:05 +0200
Subject: [PATCH 50/98] bug

---
 plugins/bm29/ispyb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 42d15a9..c070706 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -113,7 +113,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         metadata = {"definition": "SAXS",
                     "Sample_name": sample}
         for k,v in data.items():
-            if k.startswith("SAXS_"):
+            if isinstance(k, str) and k.startswith("SAXS_"):
                 metadata[k] = v
         sample = data.get("sample")
         if sample:

From b188b05cc128c4c09bb381eb278db6288884096b Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:21:54 +0200
Subject: [PATCH 51/98] typo

---
 plugins/bm29/integrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index 45481f4..fce55b5 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -636,7 +636,7 @@ def send_to_ispyb(self):
             self.to_pyarch["SAXS_pixelSizeY"] = str(self.ai.detector.pixel1)
             f2d = self.ai.getFit2D()
             self.to_pyarch["SAXS_beam_center_x"] = str(f2d["centerX"])
-            self.to_pyarch["SAXS_beam_center_x"] = str(f2d["centerY"])
+            self.to_pyarch["SAXS_beam_center_y"] = str(f2d["centerY"])
 
             ispyb.send_icat(data=self.to_pyarch)
         else:

From 67769a82577ab044bb27cf8d9c74506a2ec21bd9 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:32:00 +0200
Subject: [PATCH 52/98] minor corrections

---
 plugins/bm29/ispyb.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index c070706..0b8b8ef 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -117,7 +117,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
                 metadata[k] = v
         sample = data.get("sample")
         if sample:
-            metadata["SAXS_concentration"] = sample.concentration
+            metadata["SAXS_concentration"] = str(sample.concentration)
             metadata["SAXS_code"] = sample.name
             metadata["SAXS_comments"] = sample.description
             metadata["SAXS_storage_temperature"] = sample.temperature_env
@@ -141,12 +141,12 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         if tomerge:
             metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
         
-        volume = data.get("volume")
+        volume = str(data.get("volume"))
         if volume:
             metadata["SAXS_porod_volume"] = volume 
         #Other metadata one may collect ...
         metadata["SAXS_experimentType"]= data.get("experiment_type", "")         
-
+        metadata["datasetName"] = dataset
         icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
         kwargs = {"beamline":beamline, 
                   "proposal":proposal, 
@@ -284,7 +284,7 @@ def send_subtracted(self, data):
         gnom = data.get("bift")
         subtracted = data.get("subtracted")
         basename = data.get("basename", "frame")
-        sub = self.save_curve("subtracted", subtracted, basename)
+        sub = self.save_curve("subtracted", subtracted, basename, gallery=True)
         buf = self.save_curve("buffer_avg", data.get("buffer"), basename)
         individual_buffers = []
         for i, bufi in enumerate(data.get("buffers", [])):

From 5e209d4c03db5f0909f9832dd0bfc95f0e5987b6 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:38:50 +0200
Subject: [PATCH 53/98] also hplc

---
 plugins/bm29/hplc.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index 75a1aa8..68d77fd 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -940,5 +940,9 @@ def send_to_ispyb(self):
         if self.ispyb and self.ispyb.url and parse_url(self.ispyb.url).host:
             ispyb = IspybConnector(*self.ispyb)
             ispyb.send_hplc(self.to_pyarch)
+            self.to_pyarch["experiment_type"]="hplc"
+            self.to_pyarch["sample"] = self.juices[0].sample
+            ispyb.send_icat(data=self.to_pyarch)
+
         else:
             self.log_warning(f"Not sending to ISPyB: no valid URL in {self.ispyb}")

From d450b7835ab4d6c99fa24042fe18f976ce02cb60 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:41:18 +0200
Subject: [PATCH 54/98] debug

---
 plugins/bm29/ispyb.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 0b8b8ef..77c641b 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -154,7 +154,8 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
                   "path":path, 
                   "metadata":metadata, 
                   "raw":[raw]}
-        print("Sent to iCAT:",json.dumps(kwargs, indent=2))
+        print("Sent to iCAT:",
+              kwargs)
         icat_client.store_processed_data(**kwargs)
 
     def send_averaged(self, data):

From 51075ef8b76816d862de7722cdd928522c14e85a Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:45:30 +0200
Subject: [PATCH 55/98] fix datatypes

---
 plugins/bm29/hplc.py  | 2 ++
 plugins/bm29/ispyb.py | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index 68d77fd..f696bc5 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -941,6 +941,8 @@ def send_to_ispyb(self):
             ispyb = IspybConnector(*self.ispyb)
             ispyb.send_hplc(self.to_pyarch)
             self.to_pyarch["experiment_type"]="hplc"
+            if "volume" in self.to_pyarch:
+                self.to_pyarch.pop("volume")
             self.to_pyarch["sample"] = self.juices[0].sample
             ispyb.send_icat(data=self.to_pyarch)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 77c641b..9b8c57b 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -120,8 +120,8 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
             metadata["SAXS_concentration"] = str(sample.concentration)
             metadata["SAXS_code"] = sample.name
             metadata["SAXS_comments"] = sample.description
-            metadata["SAXS_storage_temperature"] = sample.temperature_env
-            metadata["SAXS_exposure_temperature"] = sample.temperature
+            metadata["SAXS_storage_temperature"] = str(sample.temperature_env)
+            metadata["SAXS_exposure_temperature"] = str(sample.temperature)
             if sample.hplc:
                 metadata["SAXS_column_type"] = sample.hplc
             #"buffer": "description of buffer, pH, ...",
@@ -156,6 +156,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
                   "raw":[raw]}
         print("Sent to iCAT:",
               kwargs)
+        print(json.dumps(kwargs))
         icat_client.store_processed_data(**kwargs)
 
     def send_averaged(self, data):

From 9c839c3c6b177dd2302d3dc186c3f0da0a2e9db8 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 16 Sep 2022 17:47:35 +0200
Subject: [PATCH 56/98] remove debug

---
 plugins/bm29/ispyb.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 9b8c57b..2edded7 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -141,9 +141,9 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         if tomerge:
             metadata["SAXS_frames_averaged"] = f"{tomerge[0]}-{tomerge[1]}"
         
-        volume = str(data.get("volume"))
+        volume = data.get("volume")
         if volume:
-            metadata["SAXS_porod_volume"] = volume 
+            metadata["SAXS_porod_volume"] = str(volume) 
         #Other metadata one may collect ...
         metadata["SAXS_experimentType"]= data.get("experiment_type", "")         
         metadata["datasetName"] = dataset
@@ -154,9 +154,6 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
                   "path":path, 
                   "metadata":metadata, 
                   "raw":[raw]}
-        print("Sent to iCAT:",
-              kwargs)
-        print(json.dumps(kwargs))
         icat_client.store_processed_data(**kwargs)
 
     def send_averaged(self, data):

From 813566ff0d3858e34e17b3f1339b7461d5f4674c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@stanza.esrf.fr>
Date: Mon, 26 Sep 2022 15:23:00 +0200
Subject: [PATCH 57/98] log what is sent to icat

---
 plugins/bm29/integrate.py | 3 ++-
 plugins/bm29/ispyb.py     | 7 ++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index fce55b5..eb73425 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -638,7 +638,8 @@ def send_to_ispyb(self):
             self.to_pyarch["SAXS_beam_center_x"] = str(f2d["centerX"])
             self.to_pyarch["SAXS_beam_center_y"] = str(f2d["centerY"])
 
-            ispyb.send_icat(data=self.to_pyarch)
+            icat = ispyb.send_icat(data=self.to_pyarch)
+            self.log_warning("Sent to icat: " + str(icat))
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 2edded7..81cb953 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -102,9 +102,9 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         if sample is None:
             sample = tmp[idx_process-2]
         if dataset is None:
-            dataset = tmp[idx_process-1]
-            if len(dataset) > len(sample):
-                dataset = dataset[len(sample)+1:]
+            dataset = tmp[idx_process+1]
+            #f len(dataset) > len(sample):
+            #   dataset = dataset[len(sample)+1:]
         if path is None:
             path = os.path.dirname(self.gallery)
         if raw is None:            
@@ -155,6 +155,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
                   "metadata":metadata, 
                   "raw":[raw]}
         icat_client.store_processed_data(**kwargs)
+        return kwargs
 
     def send_averaged(self, data):
         """Send this to ISPyB and backup to PyArch

From 1e5958ffe3145712187a2df775b793f96996acc7 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@stanza.esrf.fr>
Date: Mon, 26 Sep 2022 15:52:47 +0200
Subject: [PATCH 58/98] cope with NMF failing analysis

---
 plugins/bm29/hplc.py | 57 +++++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 32 deletions(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index f696bc5..d92a3a4 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -36,6 +36,7 @@
 from scipy.optimize import minimize
 import scipy.signal
 import scipy.ndimage
+import sklearn
 from sklearn.decomposition import NMF
 from .common import Sample, Ispyb, get_equivalent_frames, cmp_float, get_integrator, KeyCache, \
                     polarization_factor, method, Nexus, get_isotime, SAXS_STYLE, NORMAL_STYLE, \
@@ -338,40 +339,32 @@ def create_nexus(self):
     # Process 3: NMF matrix decomposition
         nmf_grp = nxs.new_class(entry_grp, "3_NMF", "NXprocess")
         nmf_grp["sequence_index"] = self.sequence_index()
+        nmf_grp["program"] = "sklearn.decomposition.NMF"
+        nmf_grp["version"] = sklearn.__version__
         nmf = NMF(n_components=self.nmf_components, init='nndsvd',
                   max_iter=1000)
-        W = nmf.fit_transform(I.T)
-        eigen_data = nxs.new_class(nmf_grp, "eigenvectors", "NXdata")
-        eigen_ds = eigen_data.create_dataset("W", data=numpy.ascontiguousarray(W.T, dtype=numpy.float32))
-        eigen_ds.attrs["interpretation"] = "spectrum"
-        eigen_data.attrs["signal"] = "W"
-        eigen_data.attrs["SILX_style"] = SAXS_STYLE
-
-        eigen_ds.attrs["units"] = "arbitrary"
-        eigen_ds.attrs["long_name"] = "Intensity (absolute, normalized on water)"
-
-        H = nmf.components_
-        chroma_data = nxs.new_class(nmf_grp, "chromatogram", "NXdata")
-        chroma_ds = chroma_data.create_dataset("H", data=numpy.ascontiguousarray(H, dtype=numpy.float32))
-        chroma_ds.attrs["interpretation"] = "spectrum"
-        chroma_data.attrs["signal"] = "H"
-        chroma_data.attrs["SILX_style"] = NORMAL_STYLE
-        nmf_grp.attrs["default"] = chroma_data.name
-        # Background obtained from NMF is not as good as the one obtained from SVD ...
-        # quantiles = (0.1, 0.6)  # assume weakest diffracting are background keep 10-40%
-        # background = W[:, 0] * (numpy.sort(H[0])[int(nframes * quantiles[0]): int(nframes * quantiles[-1])]).mean()
-        # bg_data = nxs.new_class(nmf_grp, "background", "NXdata")
-        # bg_data.attrs["signal"] = "I"
-        # bg_data.attrs["SILX_style"] = SAXS_STYLE
-        # bg_data.attrs["axes"] = radial_unit
-        # bg_ds = bg_data.create_dataset("I", data=numpy.ascontiguousarray(background, dtype=numpy.float32))
-        # bg_ds.attrs["interpretation"] = "spectrum"
-        # bg_data.attrs["quantiles"] = quantiles
-        # bg_q_ds = bg_data.create_dataset(radial_unit,
-        #                                  data=numpy.ascontiguousarray(q, dtype=numpy.float32))
-        # bg_q_ds.attrs["units"] = unit_name
-        # radius_unit = "nm" if "nm" in unit_name else "Å"
-        # bg_q_ds.attrs["long_name"] = f"Scattering vector q ({radius_unit}⁻¹)"
+        try:
+            W = nmf.fit_transform(I.T)
+        except ValueError as err:
+            self.log_warning(f"NMF data decomposition failed with: {err}")
+            nmf_grp[str(type(err))] = str(err)
+        else:
+            eigen_data = nxs.new_class(nmf_grp, "eigenvectors", "NXdata")
+            eigen_ds = eigen_data.create_dataset("W", data=numpy.ascontiguousarray(W.T, dtype=numpy.float32))
+            eigen_ds.attrs["interpretation"] = "spectrum"
+            eigen_data.attrs["signal"] = "W"
+            eigen_data.attrs["SILX_style"] = SAXS_STYLE
+
+            eigen_ds.attrs["units"] = "arbitrary"
+            eigen_ds.attrs["long_name"] = "Intensity (absolute, normalized on water)"
+
+            H = nmf.components_
+            chroma_data = nxs.new_class(nmf_grp, "chromatogram", "NXdata")
+            chroma_ds = chroma_data.create_dataset("H", data=numpy.ascontiguousarray(H, dtype=numpy.float32))
+            chroma_ds.attrs["interpretation"] = "spectrum"
+            chroma_data.attrs["signal"] = "H"
+            chroma_data.attrs["SILX_style"] = NORMAL_STYLE
+            nmf_grp.attrs["default"] = chroma_data.name
 
     # Process 5: Background estimation
         bg_grp = nxs.new_class(entry_grp, "4_background", "NXprocess")

From 295d6f474fb495de3f4996c845210ca4505e931f Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@stanza.esrf.fr>
Date: Wed, 28 Sep 2022 13:16:18 +0200
Subject: [PATCH 59/98] better logging

---
 plugins/bm29/hplc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index d92a3a4..ac98aed 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -347,7 +347,7 @@ def create_nexus(self):
             W = nmf.fit_transform(I.T)
         except ValueError as err:
             self.log_warning(f"NMF data decomposition failed with: {err}")
-            nmf_grp[str(type(err))] = str(err)
+            nmf_grp[err.__class__.__name__] = str(err)
         else:
             eigen_data = nxs.new_class(nmf_grp, "eigenvectors", "NXdata")
             eigen_ds = eigen_data.create_dataset("W", data=numpy.ascontiguousarray(W.T, dtype=numpy.float32))

From 69b6b7dc603bf28a0e2056f1ad414d7e41380b5e Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@stanza.esrf.fr>
Date: Wed, 28 Sep 2022 14:20:40 +0200
Subject: [PATCH 60/98] add some extra checks for directory creation

---
 plugins/bm29/hplc.py      |  1 -
 plugins/bm29/integrate.py |  2 +-
 plugins/bm29/ispyb.py     | 18 +++++++++++-------
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/plugins/bm29/hplc.py b/plugins/bm29/hplc.py
index ac98aed..6f14219 100644
--- a/plugins/bm29/hplc.py
+++ b/plugins/bm29/hplc.py
@@ -938,6 +938,5 @@ def send_to_ispyb(self):
                 self.to_pyarch.pop("volume")
             self.to_pyarch["sample"] = self.juices[0].sample
             ispyb.send_icat(data=self.to_pyarch)
-
         else:
             self.log_warning(f"Not sending to ISPyB: no valid URL in {self.ispyb}")
diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index eb73425..ddcb94f 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -639,7 +639,7 @@ def send_to_ispyb(self):
             self.to_pyarch["SAXS_beam_center_y"] = str(f2d["centerY"])
 
             icat = ispyb.send_icat(data=self.to_pyarch)
-            self.log_warning("Sent to icat: " + str(icat))
+            #self.log_warning("Sent to icat: " + str(icat))
         else:
             self.log_warning("Not sending to ISPyB: no valid URL %s" % self.ispyb.url)
 
diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 81cb953..3d7502d 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -67,6 +67,11 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
 
         if gallery:
             self.gallery = os.path.abspath(gallery)
+            if not os.path.isdir(self.gallery):
+                try:
+                   os.makedirs(self.gallery)
+                except Exception as err:
+                    logger.warning(f"Unable to create dir {self.gallery}. {type(err)}: {err}")
         else:
             logger.error("No `gallery` destination provided ... things will go wrong")
             self.gallery = tempfile.gettempdir()
@@ -103,8 +108,6 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
             sample = tmp[idx_process-2]
         if dataset is None:
             dataset = tmp[idx_process+1]
-            #f len(dataset) > len(sample):
-            #   dataset = dataset[len(sample)+1:]
         if path is None:
             path = os.path.dirname(self.gallery)
         if raw is None:            
@@ -189,15 +192,15 @@ def send_averaged(self, data):
                                         str(averaged))
         sasm = numpy.vstack((aver_data.radial, aver_data.intensity, aver_data.sigma)).T
         self.scatter_plot(sasm, basename=basename)
-        
-        
-        
-        
 
     def _mk_filename(self, index, path, basename="frame", ext=".dat"):
         dest = os.path.join(self.pyarch, path)
         if not os.path.isdir(dest):
-            os.makedirs(dest)
+            try:
+                os.makedirs(dest)
+            except Exception as err:
+                logger.error("Unable to create directory %s: %s: %s", dest, type(err), err)
+        os.stat(dest) #this is to enforce the mounting of the directory
         if isinstance(index, int):
             filename = os.path.join(dest, "%s_%04d%s" % (basename, index, ext))
         else:
@@ -361,3 +364,4 @@ def send_hplc(self, data):
         self.client.service.storeHPLC(str(self.experiment_id),
                                       filename,
                                       json_file)
+

From 93fb9c15f3da4a52a5be8265514cda78573d1180 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Mon, 3 Oct 2022 08:56:29 +0200
Subject: [PATCH 61/98] shorter name for detector

---
 plugins/bm29/integrate.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/bm29/integrate.py b/plugins/bm29/integrate.py
index ddcb94f..466e4ea 100644
--- a/plugins/bm29/integrate.py
+++ b/plugins/bm29/integrate.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "16/09/2022"
+__date__ = "03/10/2022"
 __status__ = "development"
 __version__ = "0.3.0"
 
@@ -311,7 +311,7 @@ def create_nexus(self):
         nrj_ds.attrs["units"] = "keV"
         nrj_ds.attrs["resolution"] = 0.014
 
-        detector_grp = nxs.new_class(instrument_grp, str(self.ai.detector), "NXdetector")
+        detector_grp = nxs.new_class(instrument_grp, self.ai.detector.name, "NXdetector")
         dist_ds = detector_grp.create_dataset("distance", data=self.ai.dist)
         dist_ds.attrs["units"] = "m"
         xpix_ds = detector_grp.create_dataset("x_pixel_size", data=self.ai.pixel2)

From 19e3db82ba4e182bafeb863589514562a001a2b9 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 5 Oct 2022 16:11:46 +0200
Subject: [PATCH 62/98] change name of the chromatogram png file

---
 plugins/bm29/ispyb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index 3d7502d..f334bdd 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "16/09/2022"
+__date__ = "05/10/2022"
 __status__ = "development"
 version = "0.2.0"
 
@@ -342,7 +342,7 @@ def send_hplc(self, data):
         """
         sample = data.get("sample_name", "sample")
         #gallery
-        gallery = os.path.join(self.gallery, f'{sample}_hplc.png')
+        gallery = os.path.join(self.gallery, 'chromatogram.png')
         chromatogram = data.get("sum_I")
         
         if chromatogram is not None:

From ac5a22ca198bb9f4c4ea80e090e1856ad124c790 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 3 Nov 2022 11:46:55 +0100
Subject: [PATCH 63/98] fix normalization issue with buffers' std

---
 plugins/bm29/ispyb.py     | 19 ++++++++++++++-----
 plugins/bm29/subtracte.py |  9 +++++----
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/plugins/bm29/ispyb.py b/plugins/bm29/ispyb.py
index f334bdd..1e358a4 100644
--- a/plugins/bm29/ispyb.py
+++ b/plugins/bm29/ispyb.py
@@ -11,9 +11,9 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "05/10/2022"
+__date__ = "03/11/2022"
 __status__ = "development"
-version = "0.2.0"
+version = "0.2.1"
 
 import logging
 logger = logging.getLogger("bm29.ispyb")
@@ -22,9 +22,18 @@
 import json
 import tempfile
 import numpy
-from suds.client import Client
-from suds.transport.https import HttpAuthenticated
-from pyicat_plus.client.main import IcatClient
+try:
+    from suds.client import Client
+    from suds.transport.https import HttpAuthenticated
+except ImportError:
+    print("ISPyB connection will no work")
+    Client = HttpAuthenticated = None
+try:
+    from pyicat_plus.client.main import IcatClient
+except ImportError:
+    print("iCat connection will no work")
+    IcatClient = None
+
 import matplotlib.pyplot
 matplotlib.use("Agg")
 from freesas.collections import RG_RESULT, RT_RESULT, StatsResult
diff --git a/plugins/bm29/subtracte.py b/plugins/bm29/subtracte.py
index 793702a..f2ea796 100644
--- a/plugins/bm29/subtracte.py
+++ b/plugins/bm29/subtracte.py
@@ -11,9 +11,9 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "16/09/2022"
+__date__ = "03/11/2022"
 __status__ = "development"
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 import os
 import json
@@ -58,7 +58,7 @@ class SubtractBuffer(Plugin):
     {
       "buffer_files": ["buffer_001.h5", "buffer_002.h5"],
       "sample_file": "sample.h5",
-      "output_file": "subtracted.h5"
+      "output_file": "subtracted.h5",
       "fidelity": 0.001,
       "ispyb": {
         "url": "http://ispyb.esrf.fr:1234",
@@ -291,8 +291,9 @@ def create_nexus(self):
         sum_signal = sum(self.buffer_juices[i].normalization * self.buffer_juices[i].signal2d for i in to_merge_idx)
         sum_variance = sum((self.buffer_juices[i].normalization * self.buffer_juices[i].error2d) ** 2 for i in to_merge_idx)
         sum_norm = sum(self.buffer_juices[i].normalization for i in to_merge_idx)
+        sum_norm2 = sum(self.buffer_juices[i].normalization**2 for i in to_merge_idx)
         buffer_average = sum_signal / sum_norm
-        buffer_variance = sum_variance / sum_norm
+        buffer_variance = sum_variance / sum_norm2
         sample_average = self.sample_juice.signal2d
         sample_variance = self.sample_juice.error2d ** 2
         sub_average = sample_average - buffer_average

From a0d3acf672c92bba335c68701acc062bae5cdf95 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 7 Feb 2023 14:05:17 +0100
Subject: [PATCH 64/98] polish minor bugs

---
 plugins/id27.py | 53 +++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 42 insertions(+), 11 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index e9f6ca6..db722aa 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -148,7 +148,16 @@ def create_rsync_file(filename, folder="esp"):
     :param folder: name of the folder to create. If None, just return the path of the script
     :return: the path of the crysalis directory  
     """
-    dest_dir = "/".join(filename.split("/")[3:-1])  # strip /data/visitor ... filename.h5
+    splitted = filename.split("/")
+    if "raw" in splitted:
+        splitted[splitted.index("raw")] = "processed"
+        destname =  "/".join(splitted)
+        ddir = os.path.dirname(destname)
+        if not os.path.isdir(ddir):
+            os.makedirs(ddir)
+    else:
+        destname = filename
+    dest_dir = "/".join(splitted[3:-1])  # strip /data/visitor ... filename.h5
     dest_dir = os.path.join(WORKDIR, dest_dir)
     script = os.path.join(dest_dir, "sync")
     if folder is None:
@@ -162,10 +171,10 @@ def create_rsync_file(filename, folder="esp"):
 
     if os.path.exists(script):
         with open(script, "a") as source:
-            source.write(os.linesep.join([f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(filename)}', ""]))
+            source.write(os.linesep.join([f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(destname)}', ""]))
     else:
         with open(script, "w") as source:
-            source.write(os.linesep.join(['#!/bin/sh', f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(filename)}', '']))
+            source.write(os.linesep.join(['#!/bin/sh', f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(destname)}', '']))
         os.chmod(script, 0o755)
 
     return crysalis_dir
@@ -279,6 +288,18 @@ def fabio_conversion(file_path,
     results = []
     filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
     files = sorted(glob.glob(filename))
+    file_path = file_path.rstrip("/")
+
+    splitted = file_path.split("/")
+    dset_name = splitted[-1]
+    if "raw" in splitted:
+        raw_pos = splitted.index("raw")
+        splitted[raw_pos] = "processed"
+        splitted.append(scan_number)
+        splitted.insert(0, "/")
+        dest_dir = os.path.join(*splitted)    
+    else:
+        dest_dir = os.path.join(file_path, scan_number)
 
     if len(files) == 0:
         raise RuntimeError(f"No such file {filename}")
@@ -286,15 +307,16 @@ def fabio_conversion(file_path,
     for idx_file, filename in enumerate(files):
         img_data_fabio = fabio.open(filename)
         basename = folder if len(files) == 1 else f"{folder}_{idx_file+1:04d}"
-        dest_dir = os.path.join(file_path, scan_number, basename)
-        if not os.path.exists(dest_dir):
-            os.makedirs(dest_dir)
+
+        dest_dir2 = os.path.join(dest_dir, basename)
+        if not os.path.exists(dest_dir2):
+            os.makedirs(dest_dir2)
         for i, frame in enumerate(img_data_fabio):
             conv = frame.convert(fabioimage)
             data = conv.data.astype('int32')
             data[data < 0 ] = 0
             conv.data = data
-            output = os.path.join(dest_dir, f"frame_{i+1:04d}.{extension}")
+            output = os.path.join(dest_dir2, f"{dset_name}_{i+1:04d}.{extension}")
             conv.write(output)
             results.append(output)
 
@@ -418,7 +440,7 @@ def process(self):
 
         results = {}
         outputs = []
-        basename = 'sum.edf'
+        basename = 'sum.h5'
         prefix = basename.split('.')[0]
 
         if not self.input:
@@ -436,16 +458,25 @@ def process(self):
             if output_directory:
                 dest_dir = output_directory
                 sample, dataset = file_path.strip().strip("/").split("/")[-2:]
-                tmpname = basename if len(filenames) == 1 else f'{prefix}_{idx_h5+1:04d}.edf'
+                tmpname = basename if len(filenames) == 1 else f'{prefix}_{idx_h5+1:04d}.h5'
                 basename = "_".join((sample, dataset, scan_number, tmpname))
             else:
                 tmpname = prefix if len(filenames) == 1 else f'{prefix}_{idx_h5+1:04d}'
-                dest_dir = os.path.join(file_path, scan_number, tmpname)
+                splitted = file_path.split("/")
+                if "raw" in splitted:
+                    raw_pos = splitted.index("raw")
+                    splitted[raw_pos] = "processed"
+                    splitted.append(scan_number)
+                    splitted.append(tmpname)
+                    splitted.insert(0, "/")
+                    dest_dir = os.path.join(*splitted)    
+                else:
+                    dest_dir = os.path.join(file_path, scan_number, tmpname)
             if not os.path.exists(dest_dir):
                 os.makedirs(dest_dir)
 
             output = os.path.join(dest_dir, basename)
-            command = ['pyFAI-average', '-m', 'sum', '-o', output, filename]
+            command = ['pyFAI-average', '-m', 'sum', '-F', 'lima', '-o', output, filename]
             results[filename] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
             outputs.append(output)
         self.output["output_filename"] = outputs

From d927276b774aada9d963961657ec3c1bdc3ded7d Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 28 Feb 2023 11:45:19 +0100
Subject: [PATCH 65/98] work in progress

---
 plugins/id27.py | 94 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 82 insertions(+), 12 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index db722aa..9138d80 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -10,11 +10,15 @@
 import glob
 import subprocess
 import fabio
+import pyFAI
 from dahu.plugin import Plugin
 from dahu.factory import register
 
 logger = logging.getLogger("id27")
 
+RAW = "raw"
+PROCESSED = "processed"
+
 try:
     from cryio import crysalis
 except ImportError:
@@ -106,6 +110,7 @@ def unpack_processed(completed_process):
 
 
 def createCrysalis(scans, crysalis_dir, basename):
+    """Unused"""
     runHeader = crysalis.RunHeader(basename.encode(), crysalis_dir.encode(), 1)
     runname = os.path.join(crysalis_dir, basename)
     runFile = []
@@ -149,8 +154,8 @@ def create_rsync_file(filename, folder="esp"):
     :return: the path of the crysalis directory  
     """
     splitted = filename.split("/")
-    if "raw" in splitted:
-        splitted[splitted.index("raw")] = "processed"
+    if RAW in splitted:
+        splitted[splitted.index(RAW)] = PROCESSED
         destname =  "/".join(splitted)
         ddir = os.path.dirname(destname)
         if not os.path.isdir(ddir):
@@ -188,6 +193,7 @@ def crysalis_conversion(wave_length=None, distance=None,
                         file_source_path=None,
                         scan_name=None,
                         calibration_path="", calibration_name="",
+                        update_mask=False, update_par=False,
                         **kwargs):
     """Run the `eiger2crysalis` script synchronously
      
@@ -215,13 +221,17 @@ def crysalis_conversion(wave_length=None, distance=None,
                   f"--omega={omega_start}+{omega_step}*index",
                   file_source_path,
                   "-o", os.path.join(crysalis_dir, f'{scan_name}_1_''{index}.esperanto')]
+    if update_mask:
+        parameters.append("--calc-mask")
     logger.info('starts with parameters: %s', parameters)
 
     crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points, omega_start, omega_step, center, distance, wave_length, exposure_time)
 
-    copy_set_ccd(crysalis_files, crysalis_dir, scan_name)
-    createCrysalis(scans, crysalis_dir, scan_name)
-    create_par_file(crysalis_files, crysalis_dir, scan_name)
+    if not update_mask:
+        copy_set_ccd(crysalis_files, crysalis_dir, scan_name)
+    #createCrysalis(scans, crysalis_dir, scan_name) # .run file already implemented in eiger2crysalis
+    if not update_par:
+        create_par_file(crysalis_files, crysalis_dir, scan_name)
 
     return subprocess.run(parameters, capture_output=True, check=False)
 
@@ -292,9 +302,9 @@ def fabio_conversion(file_path,
 
     splitted = file_path.split("/")
     dset_name = splitted[-1]
-    if "raw" in splitted:
-        raw_pos = splitted.index("raw")
-        splitted[raw_pos] = "processed"
+    if RAW in splitted:
+        raw_pos = splitted.index(RAW)
+        splitted[raw_pos] = PROCESSED
         splitted.append(scan_number)
         splitted.insert(0, "/")
         dest_dir = os.path.join(*splitted)    
@@ -345,7 +355,9 @@ class CrysalisConversion(Plugin):
  "scan_name": "scan0001",
  "calibration_path": "/data/id27/inhouse/blc13357/id27/Vanadinite_Weck/Vanadinite_Weck_0001/scan0001/esp",
  "calibration_name":"scan0001",
- "plugin_name": "id27.CrysalisConversion"
+ "plugin_name": "id27.CrysalisConversion",
+ "update_mask": 0,
+ "update_par": 0
 }    
     """
 
@@ -463,9 +475,9 @@ def process(self):
             else:
                 tmpname = prefix if len(filenames) == 1 else f'{prefix}_{idx_h5+1:04d}'
                 splitted = file_path.split("/")
-                if "raw" in splitted:
-                    raw_pos = splitted.index("raw")
-                    splitted[raw_pos] = "processed"
+                if RAW in splitted:
+                    raw_pos = splitted.index(RAW)
+                    splitted[raw_pos] = PROCESSED
                     splitted.append(scan_number)
                     splitted.append(tmpname)
                     splitted.insert(0, "/")
@@ -508,3 +520,61 @@ def process(self):
                          fabioimage="cbfimage",
                          extension="cbf")
         self.output["output"] = results
+
+@register
+class DiffMap(Plugin):
+    """
+    This plugin performs the azimuthal integration of a set of data and populate a diffraction map
+
+    Typical JSON input structure:
+    { "ponifile": "/tmp/geometry.poni",
+      "maskfile": "/tmp/mask.msk",
+      "npt": 2000,
+      "file_path": "/data/id27/inhouse/some/path",
+      "scan_number": "scan_0001"
+      "slow_scan" = 10,
+      "fast_scan" = 10
+    }
+    """
+    def process(self):
+        Plugin.process(self)
+        if not self.input:
+            logger.error("input is empty")
+
+        file_path = self.input["file_path"]
+        scan_number = self.input["scan_number"]
+        dest_dir = file_path.replace(RAW,PROCESSED) 
+        filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
+        files = sorted(glob.glob(filename))
+        if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
+        config = os.path.join(dest_dir, "diff_map.json")
+        dest = os.path.join(dest_dir, "diff_map.h5")
+        results = {}
+        param = {}
+        ai = pyFAI.load(self.input.get("ponifile")).get_config()
+        if "maskfile" in self.input:
+            ai["do_mask"] = True
+            ai["mask_file"] = self.input["maskfile"]
+        # some constants hardcoded for the beamline:
+        ai["unit"] = "q_nm^-1"
+        ai["do_polarization"] = True,
+        ai["polarization_factor"]= 0.99
+        ai["do_solid_angle"]= True
+        ai["error_model"]="poisson"
+	ai["application"] = "pyfai-integrate"
+        ai["version"] = 3
+        ai["method"] =[ "full", "csr", "opencl" ]
+        ai["opencl_device"] = "gpu"
+        param["ai"] = ai
+        param["experiment_title"] = os.path.join(os.path.basename(file_path),scan_number)
+        param["fast_motor_name": "fast",
+        param["slow_motor_name": "slow",
+        param["fast_motor_points"] = self.input.get("fast_scan" ,1)
+        param["slow_motor_points"] = self.input.get("slow_scan" ,1)
+        param["offset"]: None
+        param["output_file"] = dest
+        param["input_data"] =  [ files ]
+        with open(config, "w") as w:
+            w.write(json.dumps(param, indent=2))
+        results["config"] = config

From c4feff6b8c8736906a351141121ba6beb5199b25 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 28 Feb 2023 15:26:52 +0100
Subject: [PATCH 66/98] fix the path of Eiger2Crysalis

---
 plugins/id27.py | 83 +++++++++++++++++++++++++------------------------
 1 file changed, 42 insertions(+), 41 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 9138d80..9324b5a 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -2,6 +2,7 @@
 """
 
 import os
+import sys
 import shutil
 import collections
 import io
@@ -18,13 +19,14 @@
 
 RAW = "raw"
 PROCESSED = "processed"
+WORKDIR = "/scratch/shared"
+PREFIX = os.path.dirname(sys.executable) #where there scripts are
 
 try:
     from cryio import crysalis
 except ImportError:
     crysalis = None
 
-WORKDIR = "/scratch/shared"
 
 
 def crysalis_config(calibration_path, calibration_name, number_of_frames,
@@ -52,32 +54,31 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,
         'ccd_file': '/users/opid27/file_conversion/scan0001.ccd'}
 
     scans = collections.OrderedDict()
-    scans[0] = [{
-            'count': number_of_frames,
-            'omega': 0,
-            'omega_start': omega_start,
-            'omega_end': omega_start + number_of_frames * omega_step,
-            'pixel_size': 0.075,
-            'omega_runs': None,
-            'theta': 0,
-            'kappa': 0,
-            'phi': 0,
-            'domega': omega_step,
-            'dtheta': 0,
-            'dkappa': 0,
-            'dphi': 0,
-            'center_x': center[0],
-            'center_y': center[1],
-            'alpha': 50,
-            'dist': distance,
-            'l1': wave_length,
-            'l2': wave_length,
-            'l12': wave_length,
-            'b': wave_length,
-            'mono': 0.99,
-            'monotype': 'SYNCHROTRON',
-            'chip': [1024, 1024],
-            'Exposure_time': exposure_time,
+    scans[0] = [{'count': number_of_frames,
+                 'omega': 0,
+                 'omega_start': omega_start,
+                 'omega_end': omega_start + number_of_frames * omega_step,
+                 'pixel_size': 0.075,
+                 'omega_runs': None,
+                 'theta': 0,
+                 'kappa': 0,
+                 'phi': 0,
+                 'domega': omega_step,
+                 'dtheta': 0,
+                 'dkappa': 0,
+                 'dphi': 0,
+                 'center_x': center[0],
+                 'center_y': center[1],
+                 'alpha': 50,
+                 'dist': distance,
+                 'l1': wave_length,
+                 'l2': wave_length,
+                 'l12': wave_length,
+                 'b': wave_length,
+                 'mono': 0.99,
+                 'monotype': 'SYNCHROTRON',
+                 'chip': [1024, 1024],
+                 'Exposure_time': exposure_time,
             }]
 
     return crysalis_files, scans
@@ -210,7 +211,7 @@ def crysalis_conversion(wave_length=None, distance=None,
                 f'file_source_path={file_source_path}\n' +
                 f'scan_name={scan_name}')
 
-    script_name = 'eiger2crysalis'
+    script_name = os.path.join(PREFIX, 'eiger2crysalis')
 
     crysalis_dir = create_rsync_file(file_source_path)
 
@@ -250,7 +251,7 @@ def crysalis_conversion_fscannd(wave_length=None,
                                 calibration_name="",
                                 **kwargs):
     assert crysalis, "cryio is not installed"
-    script_name = 'eiger2crysalis'
+    script_name = os.path.join(PREFIX, 'eiger2crysalis')
     pattern = re.compile('eiger_([0-9]+).h5')
     filenames_to_convert = glob.glob(f'{dirname}/eiger_????.h5')
     results = {}
@@ -559,22 +560,22 @@ def process(self):
         # some constants hardcoded for the beamline:
         ai["unit"] = "q_nm^-1"
         ai["do_polarization"] = True,
-        ai["polarization_factor"]= 0.99
-        ai["do_solid_angle"]= True
-        ai["error_model"]="poisson"
-	ai["application"] = "pyfai-integrate"
+        ai["polarization_factor"] = 0.99
+        ai["do_solid_angle"] = True
+        ai["error_model"] = "poisson"
+        ai["application"] = "pyfai-integrate"
         ai["version"] = 3
-        ai["method"] =[ "full", "csr", "opencl" ]
+        ai["method"] = ["full", "csr", "opencl"]
         ai["opencl_device"] = "gpu"
         param["ai"] = ai
-        param["experiment_title"] = os.path.join(os.path.basename(file_path),scan_number)
-        param["fast_motor_name": "fast",
-        param["slow_motor_name": "slow",
-        param["fast_motor_points"] = self.input.get("fast_scan" ,1)
-        param["slow_motor_points"] = self.input.get("slow_scan" ,1)
-        param["offset"]: None
+        param["experiment_title"] = os.path.join(os.path.basename(file_path), scan_number)
+        param["fast_motor_name"] = "fast"
+        param["slow_motor_name"] = "slow"
+        param["fast_motor_points"] = self.input.get("fast_scan", 1)
+        param["slow_motor_points"] = self.input.get("slow_scan", 1)
+        param["offset"] = 0
         param["output_file"] = dest
-        param["input_data"] =  [ files ]
+        param["input_data"] =  files
         with open(config, "w") as w:
             w.write(json.dumps(param, indent=2))
         results["config"] = config

From 56fdb01ed93a6d3ac5eefd77429208969ff6fd48 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 2 Mar 2023 11:18:38 +0100
Subject: [PATCH 67/98] PAR-file not regenerated

---
 plugins/id27.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 9324b5a..26b0c56 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -45,11 +45,11 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,
     """
     calibration_name = calibration_name or ""
     calibration_path = calibration_path or ""
-    par_file = os.path.join(calibration_path, calibration_name + '.par')
-    if not os.path.exists(par_file):
-        par_file = "/users/opid27/file_conversion/scan0001.par"
+    # par_file = os.path.join(calibration_path, calibration_name + '.par')
+    # if not os.path.exists(par_file):
+    #     par_file = "/users/opid27/file_conversion/scan0001.par"
     crysalis_files = {
-        'par_file': par_file,
+        'par_file': "/users/opid27/file_conversion/scan0001.par",
         'set_file': '/users/opid27/file_conversion/scan0001.set',
         'ccd_file': '/users/opid27/file_conversion/scan0001.ccd'}
 

From af7c8338f8950011b4e3feddb6f413bce69a328c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 2 Mar 2023 14:17:19 +0100
Subject: [PATCH 68/98] Expose DiffMap plugin

---
 plugins/id27.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 26b0c56..07a12f6 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -10,6 +10,7 @@
 import re
 import glob
 import subprocess
+import json
 import fabio
 import pyFAI
 from dahu.plugin import Plugin
@@ -489,7 +490,7 @@ def process(self):
                 os.makedirs(dest_dir)
 
             output = os.path.join(dest_dir, basename)
-            command = ['pyFAI-average', '-m', 'sum', '-F', 'lima', '-o', output, filename]
+            command = [os.path.join(PREFIX,'pyFAI-average'), '-m', 'sum', '-F', 'lima', '-o', output, filename]
             results[filename] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
             outputs.append(output)
         self.output["output_filename"] = outputs
@@ -528,13 +529,15 @@ class DiffMap(Plugin):
     This plugin performs the azimuthal integration of a set of data and populate a diffraction map
 
     Typical JSON input structure:
-    { "ponifile": "/tmp/geometry.poni",
+    { "plugin_name": "id27.diffmap",
+      "ponifile": "/tmp/geometry.poni",
       "maskfile": "/tmp/mask.msk",
+      "unit": "2th_deg",
       "npt": 2000,
       "file_path": "/data/id27/inhouse/some/path",
-      "scan_number": "scan_0001"
-      "slow_scan" = 10,
-      "fast_scan" = 10
+      "scan_number": "scan_0001",
+      "slow_scan": 10,
+      "fast_scan": 10
     }
     """
     def process(self):
@@ -558,8 +561,7 @@ def process(self):
             ai["do_mask"] = True
             ai["mask_file"] = self.input["maskfile"]
         # some constants hardcoded for the beamline:
-        ai["unit"] = "q_nm^-1"
-        ai["do_polarization"] = True,
+        ai["do_polarization"] = True
         ai["polarization_factor"] = 0.99
         ai["do_solid_angle"] = True
         ai["error_model"] = "poisson"
@@ -567,6 +569,10 @@ def process(self):
         ai["version"] = 3
         ai["method"] = ["full", "csr", "opencl"]
         ai["opencl_device"] = "gpu"
+        ai["nbpt_rad"] = self.input.get("npt", 1)
+        ai["nbpt_azim"] = 1
+        ai["do_2D"] = False
+        ai["unit"] = self.input.get("unit", "q_nm^-1")
         param["ai"] = ai
         param["experiment_title"] = os.path.join(os.path.basename(file_path), scan_number)
         param["fast_motor_name"] = "fast"
@@ -575,7 +581,11 @@ def process(self):
         param["slow_motor_points"] = self.input.get("slow_scan", 1)
         param["offset"] = 0
         param["output_file"] = dest
-        param["input_data"] =  files
+        param["input_data"] =  [(i, None, None) for i in files]
         with open(config, "w") as w:
             w.write(json.dumps(param, indent=2))
         results["config"] = config
+        command = [os.path.join(PREFIX, 'pyFAI-diffmap'), '--no-gui', '--config', config]
+        results["processing"] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
+        self.output["output_filename"] = dest
+        self.output["diffmap"] = results

From cc8f4f0fceeac87969434fd8c7412476739fd84c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Thu, 2 Mar 2023 16:48:46 +0100
Subject: [PATCH 69/98] copy crysalis files after eiger2crysalis run

---
 plugins/id27.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 07a12f6..b00989f 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -227,6 +227,7 @@ def crysalis_conversion(wave_length=None, distance=None,
         parameters.append("--calc-mask")
     logger.info('starts with parameters: %s', parameters)
 
+    res = subprocess.run(parameters, capture_output=True, check=False)
     crysalis_files, scans = crysalis_config(calibration_path, calibration_name, number_of_points, omega_start, omega_step, center, distance, wave_length, exposure_time)
 
     if not update_mask:
@@ -235,7 +236,7 @@ def crysalis_conversion(wave_length=None, distance=None,
     if not update_par:
         create_par_file(crysalis_files, crysalis_dir, scan_name)
 
-    return subprocess.run(parameters, capture_output=True, check=False)
+    return res
 
 
 def crysalis_conversion_fscannd(wave_length=None,

From 71b05e6d90dd110e9871f74b48fcb967a1cf2139 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Mon, 6 Mar 2023 13:25:46 +0100
Subject: [PATCH 70/98] do not overwrite par-file if it already exists

---
 plugins/id27.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index b00989f..a278976 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -46,9 +46,9 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,
     """
     calibration_name = calibration_name or ""
     calibration_path = calibration_path or ""
-    # par_file = os.path.join(calibration_path, calibration_name + '.par')
-    # if not os.path.exists(par_file):
-    #     par_file = "/users/opid27/file_conversion/scan0001.par"
+    par_file = os.path.join(calibration_path, calibration_name + '.par')
+    if not os.path.exists(par_file):
+        par_file = "/users/opid27/file_conversion/scan0001.par"
     crysalis_files = {
         'par_file': "/users/opid27/file_conversion/scan0001.par",
         'set_file': '/users/opid27/file_conversion/scan0001.set',

From fea92096621a38a693ce88dd817ac58f1b647d59 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Fri, 10 Mar 2023 10:43:56 +0100
Subject: [PATCH 71/98] fix crysaliss export behavior

---
 plugins/id27.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index a278976..20b975d 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -50,7 +50,7 @@ def crysalis_config(calibration_path, calibration_name, number_of_frames,
     if not os.path.exists(par_file):
         par_file = "/users/opid27/file_conversion/scan0001.par"
     crysalis_files = {
-        'par_file': "/users/opid27/file_conversion/scan0001.par",
+        'par_file': par_file,
         'set_file': '/users/opid27/file_conversion/scan0001.set',
         'ccd_file': '/users/opid27/file_conversion/scan0001.ccd'}
 
@@ -134,11 +134,16 @@ def createCrysalis(scans, crysalis_dir, basename):
 
 
 def create_par_file(crysalis_files, crysalis_dir, basename):
-
+    """Create a new par-file by copying a reference one and 
+       changing the ....
+    """    
+    ref_par = crysalis_files['par_file']
     new_par = os.path.join(crysalis_dir, basename + '.par')
-
+    if os.path.exists(new_par):
+        "make a backup"
+        os.rename(new_par, new_par+".bak")
     with io.open(new_par, 'w', encoding='iso-8859-1') as new_file:
-        with io.open(crysalis_files['par_file'], 'r', encoding='iso-8859-1') as old_file:
+        with io.open(ref_par, 'r', encoding='iso-8859-1') as old_file:
             for line in old_file:
                 if line.startswith("FILE CHIP"):
                     new_file.write('FILE CHIP "' + basename + '.ccd" \n')

From d61e2b6fcf282d2150db974bbfb87b3419694514 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 31 Mar 2023 11:01:52 +0200
Subject: [PATCH 72/98] fix difmap plugin

---
 plugins/id27.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 20b975d..8305556 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -18,8 +18,8 @@
 
 logger = logging.getLogger("id27")
 
-RAW = "raw"
-PROCESSED = "processed"
+RAW = "RAW_DATA"
+PROCESSED = "PROCESSED_DATA"
 WORKDIR = "/scratch/shared"
 PREFIX = os.path.dirname(sys.executable) #where there scripts are
 
@@ -553,7 +553,7 @@ def process(self):
 
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
-        dest_dir = file_path.replace(RAW,PROCESSED) 
+        dest_dir = os.path.join(file_path.replace(RAW, PROCESSED),  scan_number)
         filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
         files = sorted(glob.glob(filename))
         if not os.path.isdir(dest_dir):

From 0775988af86c066262902104117e3eb886edac0c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 4 Apr 2023 08:04:40 +0200
Subject: [PATCH 73/98] protect with lock directory creation

---
 plugins/id27.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 8305556..73f78ec 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -15,7 +15,8 @@
 import pyFAI
 from dahu.plugin import Plugin
 from dahu.factory import register
-
+from threading import Semaphore
+lock = Semaphore()
 logger = logging.getLogger("id27")
 
 RAW = "RAW_DATA"
@@ -166,7 +167,9 @@ def create_rsync_file(filename, folder="esp"):
         destname =  "/".join(splitted)
         ddir = os.path.dirname(destname)
         if not os.path.isdir(ddir):
-            os.makedirs(ddir)
+            with lock:
+                if not os.path.isdir(ddir):
+                    os.makedirs(ddir)
     else:
         destname = filename
     dest_dir = "/".join(splitted[3:-1])  # strip /data/visitor ... filename.h5
@@ -177,7 +180,9 @@ def create_rsync_file(filename, folder="esp"):
 
     crysalis_dir = os.path.join(dest_dir, folder)
     if not os.path.exists(crysalis_dir):
-        os.makedirs(crysalis_dir)
+        with lock:
+            if not os.path.exists(crysalis_dir):
+                os.makedirs(crysalis_dir)
     with open(os.path.join(dest_dir, ".source"), "a") as source:
         source.write(filename + os.linesep)
 
@@ -328,7 +333,9 @@ def fabio_conversion(file_path,
 
         dest_dir2 = os.path.join(dest_dir, basename)
         if not os.path.exists(dest_dir2):
-            os.makedirs(dest_dir2)
+            with lock:
+                if not os.path.exists(dest_dir2):
+                    os.makedirs(dest_dir2)
         for i, frame in enumerate(img_data_fabio):
             conv = frame.convert(fabioimage)
             data = conv.data.astype('int32')
@@ -493,7 +500,9 @@ def process(self):
                 else:
                     dest_dir = os.path.join(file_path, scan_number, tmpname)
             if not os.path.exists(dest_dir):
-                os.makedirs(dest_dir)
+                with lock:
+                    if not os.path.exists(dest_dir):
+                        os.makedirs(dest_dir)
 
             output = os.path.join(dest_dir, basename)
             command = [os.path.join(PREFIX,'pyFAI-average'), '-m', 'sum', '-F', 'lima', '-o', output, filename]
@@ -557,7 +566,9 @@ def process(self):
         filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
         files = sorted(glob.glob(filename))
         if not os.path.isdir(dest_dir):
-            os.makedirs(dest_dir)
+            with lock:
+                if not os.path.isdir(dest_dir):
+                    os.makedirs(dest_dir)
         config = os.path.join(dest_dir, "diff_map.json")
         dest = os.path.join(dest_dir, "diff_map.h5")
         results = {}

From daa616a7a42d431961671e58520a6f7f4865fbae Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Tue, 4 Apr 2023 08:16:03 +0200
Subject: [PATCH 74/98] pylint

---
 plugins/id27.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 73f78ec..3e74182 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -339,7 +339,7 @@ def fabio_conversion(file_path,
         for i, frame in enumerate(img_data_fabio):
             conv = frame.convert(fabioimage)
             data = conv.data.astype('int32')
-            data[data < 0 ] = 0
+            data[data < 0] = 0
             conv.data = data
             output = os.path.join(dest_dir2, f"{dset_name}_{i+1:04d}.{extension}")
             conv.write(output)
@@ -443,10 +443,10 @@ def process(self):
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
         results = fabio_conversion(file_path,
-                         scan_number,
-                         folder="xdi",
-                         fabioimage="tifimage",
-                         extension="tif")
+                                   scan_number,
+                                   folder="xdi",
+                                   fabioimage="tifimage",
+                                   extension="tif")
         self.output["output"] = results
 
 
@@ -505,7 +505,8 @@ def process(self):
                         os.makedirs(dest_dir)
 
             output = os.path.join(dest_dir, basename)
-            command = [os.path.join(PREFIX,'pyFAI-average'), '-m', 'sum', '-F', 'lima', '-o', output, filename]
+            command = [os.path.join(PREFIX, 'pyFAI-average'), 
+                       '-m', 'sum', '-F', 'lima', '-o', output, filename]
             results[filename] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
             outputs.append(output)
         self.output["output_filename"] = outputs
@@ -532,10 +533,10 @@ def process(self):
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
         results = fabio_conversion(file_path,
-                         scan_number,
-                         folder="cbf",
-                         fabioimage="cbfimage",
-                         extension="cbf")
+                                   scan_number,
+                                   folder="cbf",
+                                   fabioimage="cbfimage",
+                                   extension="cbf")
         self.output["output"] = results
 
 @register
@@ -562,7 +563,7 @@ def process(self):
 
         file_path = self.input["file_path"]
         scan_number = self.input["scan_number"]
-        dest_dir = os.path.join(file_path.replace(RAW, PROCESSED),  scan_number)
+        dest_dir = os.path.join(file_path.replace(RAW, PROCESSED), scan_number)
         filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
         files = sorted(glob.glob(filename))
         if not os.path.isdir(dest_dir):
@@ -598,7 +599,7 @@ def process(self):
         param["slow_motor_points"] = self.input.get("slow_scan", 1)
         param["offset"] = 0
         param["output_file"] = dest
-        param["input_data"] =  [(i, None, None) for i in files]
+        param["input_data"] = [(i, None, None) for i in files]
         with open(config, "w") as w:
             w.write(json.dumps(param, indent=2))
         results["config"] = config

From bbbb7bafddf6684b575c6f6d4ab51748e1f0a237 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 22 Nov 2023 15:46:24 +0100
Subject: [PATCH 75/98] Function to deploy to iCat

---
 plugins/id27.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/plugins/id27.py b/plugins/id27.py
index 3e74182..04eaa5a 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -29,6 +29,11 @@
 except ImportError:
     crysalis = None
 
+try:
+    from pyicat_plus.client.main import IcatClient
+except ImportError:
+    print("iCat connection will no work")
+    IcatClient = None
 
 
 def crysalis_config(calibration_path, calibration_name, number_of_frames,
@@ -607,3 +612,26 @@ def process(self):
         results["processing"] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
         self.output["output_filename"] = dest
         self.output["diffmap"] = results
+
+
+def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", metadata=None):
+    "Function that sends to icat the processed data"
+    icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
+    metadata = metadata or {}
+    l = raw_dir.split("/")
+    if not proposal:
+        visitor_idx = l.find("visitor")
+        proposal = l[visitor_idx+1]
+    if not dataset:
+        dataset = l[-1]
+    kwargs = {"beamline":beamline, 
+              "proposal":proposal, 
+              "dataset":dataset, 
+              "path":processed_dir, 
+              "metadata":metadata, 
+              "raw":[raw_dir]}
+    icat_client.store_processed_data(**kwargs)
+    
+    
+    
+    

From 48efa9406dcd8d91d347d86d5e13c80db90c0ec4 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Wed, 22 Nov 2023 15:57:08 +0100
Subject: [PATCH 76/98] return sent data for inspection

---
 plugins/id27.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/id27.py b/plugins/id27.py
index 04eaa5a..cc759e4 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -631,6 +631,7 @@ def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="",
               "metadata":metadata, 
               "raw":[raw_dir]}
     icat_client.store_processed_data(**kwargs)
+    return kwargs
     
     
     

From 0c01643673115c5a7ab769c61ab75431ba44896b Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 24 Nov 2023 11:45:19 +0100
Subject: [PATCH 77/98] start to populate icat

---
 plugins/id27.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index cc759e4..c7c6659 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -311,7 +311,8 @@ def fabio_conversion(file_path,
                      scan_number,
                      folder="xdi",
                      fabioimage="tifimage",
-                     extension="tif"):
+                     extension="tif",
+                     export_icat=False):
     "Convert a set of eiger files to cbf or tiff"
     results = []
     filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
@@ -349,7 +350,11 @@ def fabio_conversion(file_path,
             output = os.path.join(dest_dir2, f"{dset_name}_{i+1:04d}.{extension}")
             conv.write(output)
             results.append(output)
-
+    if export_icat:
+        try:
+            send_icat(file_path, dest_dir2)
+        except Exception as err:
+            print(f"Error {type(err)}: {err}")
     return results
 
 ##########################
@@ -392,6 +397,7 @@ def process(self):
         script = create_rsync_file(self.input["file_source_path"], None)
         sync_results = subprocess.run([script], capture_output=True, check=False)
         self.output["sync"] = unpack_processed(sync_results)
+        #TODO: send to icat
 
 
 @register
@@ -426,6 +432,7 @@ def process(self):
         script = create_rsync_file(random_filename, None)
         sync_results = subprocess.run([script], capture_output=True, check=False)
         self.output["sync"] = unpack_processed(sync_results)
+        #TODO: send to icat
 
 
 @register
@@ -451,7 +458,8 @@ def process(self):
                                    scan_number,
                                    folder="xdi",
                                    fabioimage="tifimage",
-                                   extension="tif")
+                                   extension="tif",
+                                   export_icat=True)
         self.output["output"] = results
 
 
@@ -516,6 +524,7 @@ def process(self):
             outputs.append(output)
         self.output["output_filename"] = outputs
         self.output["conversion"] = results
+        #todo: send to icat
 
 
 @register
@@ -541,7 +550,8 @@ def process(self):
                                    scan_number,
                                    folder="cbf",
                                    fabioimage="cbfimage",
-                                   extension="cbf")
+                                   extension="cbf",
+                                   export_icat=True)
         self.output["output"] = results
 
 @register
@@ -612,6 +622,7 @@ def process(self):
         results["processing"] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
         self.output["output_filename"] = dest
         self.output["diffmap"] = results
+        #TODO: send to icat
 
 
 def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", metadata=None):

From 406e88fe70667740fd633e17ab5e28b17d719737 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <kieffer@lid27crystalbar.esrf.fr>
Date: Fri, 24 Nov 2023 14:06:24 +0100
Subject: [PATCH 78/98] corrected on beamline

---
 plugins/id27.py | 51 +++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 41 insertions(+), 10 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index c7c6659..77bec10 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -11,11 +11,11 @@
 import glob
 import subprocess
 import json
+from threading import Semaphore
 import fabio
 import pyFAI
 from dahu.plugin import Plugin
 from dahu.factory import register
-from threading import Semaphore
 lock = Semaphore()
 logger = logging.getLogger("id27")
 
@@ -327,8 +327,10 @@ def fabio_conversion(file_path,
         splitted.append(scan_number)
         splitted.insert(0, "/")
         dest_dir = os.path.join(*splitted)    
+        sample_name = splitted[raw_pos + 1]
     else:
         dest_dir = os.path.join(file_path, scan_number)
+        sample_name = "unknown sample"
 
     if len(files) == 0:
         raise RuntimeError(f"No such file {filename}")
@@ -351,10 +353,14 @@ def fabio_conversion(file_path,
             conv.write(output)
             results.append(output)
     if export_icat:
+        metadata = {"definition": "conversion",
+                    "Sample_name": sample_name}
         try:
-            send_icat(file_path, dest_dir2)
+            send_icat(file_path, dest_dir2, metadata=metadata)
         except Exception as err:
+            import traceback
             print(f"Error {type(err)}: {err}")
+            traceback.print_exc(err, file=sys.stdout)
     return results
 
 ##########################
@@ -493,7 +499,7 @@ def process(self):
 
         if len(filenames) == 0:
             raise RuntimeError(f"File does not exist {filename}")
-
+        sample_name = "undefined sample"
         for idx_h5, filename in enumerate(filenames):
             if output_directory:
                 dest_dir = output_directory
@@ -509,7 +515,8 @@ def process(self):
                     splitted.append(scan_number)
                     splitted.append(tmpname)
                     splitted.insert(0, "/")
-                    dest_dir = os.path.join(*splitted)    
+                    dest_dir = os.path.join(*splitted)
+                    sample_name = splitted[raw_pos+1]
                 else:
                     dest_dir = os.path.join(file_path, scan_number, tmpname)
             if not os.path.exists(dest_dir):
@@ -524,7 +531,15 @@ def process(self):
             outputs.append(output)
         self.output["output_filename"] = outputs
         self.output["conversion"] = results
-        #todo: send to icat
+        #send to icat
+        metadata = {"definition": "sum",
+                    "Sample_name": sample_name}
+        try:
+            send_icat(file_path, dest_dir, metadata=metadata)
+        except Exception as err:
+            import traceback
+            print(f"Error {type(err)}: {err}")
+            traceback.print_exc(err, file=sys.stdout)
 
 
 @register
@@ -600,7 +615,7 @@ def process(self):
         ai["error_model"] = "poisson"
         ai["application"] = "pyfai-integrate"
         ai["version"] = 3
-        ai["method"] = ["full", "csr", "opencl"]
+        ai["method"] = ["bbox", "csr", "opencl"]
         ai["opencl_device"] = "gpu"
         ai["nbpt_rad"] = self.input.get("npt", 1)
         ai["nbpt_azim"] = 1
@@ -622,17 +637,33 @@ def process(self):
         results["processing"] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
         self.output["output_filename"] = dest
         self.output["diffmap"] = results
-        #TODO: send to icat
+        #send to icat
+        if RAW in file_path:
+            l = file_path.split("/")
+            i = l.index(RAW)
+            sample_name = l[i+1]
+        metadata = {"definition": "diffmap",
+                    "Sample_name": sample_name}
+        try:
+            send_icat(file_path, dest_dir, metadata=metadata)
+        except Exception as err:
+            import traceback
+            print(f"Error {type(err)}: {err}")
+            traceback.print_exc(err, file=sys.stdout)
 
 
 def send_icat(raw_dir, processed_dir, beamline="id27", proposal="", dataset="", metadata=None):
     "Function that sends to icat the processed data"
     icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
-    metadata = metadata or {}
+    metadata = metadata or {"definition": "dummy processing", "Sample_name": "unknown sample"}
     l = raw_dir.split("/")
     if not proposal:
-        visitor_idx = l.find("visitor")
-        proposal = l[visitor_idx+1]
+        try:
+            visitor_idx = l.index("visitor")
+        except:
+            proposal = "undefined"
+        else:
+            proposal = l[visitor_idx+1]
     if not dataset:
         dataset = l[-1]
     kwargs = {"beamline":beamline, 

From d47f2827c0f70185e0ba2f8685632c066abaac06 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Fri, 24 Nov 2023 14:23:09 +0100
Subject: [PATCH 79/98] typo

---
 plugins/id27.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index c7c6659..0fbd119 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -32,7 +32,7 @@
 try:
     from pyicat_plus.client.main import IcatClient
 except ImportError:
-    print("iCat connection will no work")
+    print("iCat connection will not work")
     IcatClient = None
 
 

From dad59bd9a95e83c121f7dee439b35c6767766cfc Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Mon, 18 Dec 2023 13:35:23 +0100
Subject: [PATCH 80/98] quote path names in rsync

---
 plugins/id27.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 11bf16c..ebc411c 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -11,6 +11,7 @@
 import glob
 import subprocess
 import json
+import shlex
 from threading import Semaphore
 import fabio
 import pyFAI
@@ -193,10 +194,10 @@ def create_rsync_file(filename, folder="esp"):
 
     if os.path.exists(script):
         with open(script, "a") as source:
-            source.write(os.linesep.join([f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(destname)}', ""]))
+            source.write(os.linesep.join([f'rsync -avx {os.path.join(shlex.quote(dest_dir, folder))} {shlex.quote(os.path.dirname(destname))}', ""]))
     else:
         with open(script, "w") as source:
-            source.write(os.linesep.join(['#!/bin/sh', f'rsync -avx {os.path.join(dest_dir, folder)} {os.path.dirname(destname)}', '']))
+            source.write(os.linesep.join(['#!/bin/sh', f'rsync -avx {shlex.quote(os.path.join(dest_dir, folder))} {shlex.quote(os.path.dirname(destname))}', '']))
         os.chmod(script, 0o755)
 
     return crysalis_dir
@@ -465,7 +466,7 @@ def process(self):
                                    folder="xdi",
                                    fabioimage="tifimage",
                                    extension="tif",
-                                   export_icat=True)
+                                   export_icat=False)
         self.output["output"] = results
 
 
@@ -566,7 +567,7 @@ def process(self):
                                    folder="cbf",
                                    fabioimage="cbfimage",
                                    extension="cbf",
-                                   export_icat=True)
+                                   export_icat=False)
         self.output["output"] = results
 
 @register

From 0b1dc6b9b121c23bf3783388d8825fb22ef7bc88 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Mon, 18 Dec 2023 13:50:23 +0100
Subject: [PATCH 81/98] typo

---
 plugins/id27.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index ebc411c..5611e86 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -4,6 +4,7 @@
 import os
 import sys
 import shutil
+import shlex
 import collections
 import io
 import logging
@@ -11,7 +12,6 @@
 import glob
 import subprocess
 import json
-import shlex
 from threading import Semaphore
 import fabio
 import pyFAI
@@ -194,7 +194,7 @@ def create_rsync_file(filename, folder="esp"):
 
     if os.path.exists(script):
         with open(script, "a") as source:
-            source.write(os.linesep.join([f'rsync -avx {os.path.join(shlex.quote(dest_dir, folder))} {shlex.quote(os.path.dirname(destname))}', ""]))
+            source.write(os.linesep.join([f'rsync -avx {shlex.quote(os.path.join(dest_dir, folder))} {shlex.quote(os.path.dirname(destname))}', ""]))
     else:
         with open(script, "w") as source:
             source.write(os.linesep.join(['#!/bin/sh', f'rsync -avx {shlex.quote(os.path.join(dest_dir, folder))} {shlex.quote(os.path.dirname(destname))}', '']))

From 18583acedb4f7bb4c8bc675c158d7c26da3691b9 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 08:55:59 +0100
Subject: [PATCH 82/98] move away from numpy.distutils

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8abc2d6..f801a6d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,7 @@
 [build-system]
 requires = [
     "wheel",
-    "setuptools<60.0.0",
+    "setuptools",
     "oldest-supported-numpy",
     "sphinx",
     "nbsphinx",

From 5bdd081ad15cf735743daddc089c08cfbd472710 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 09:53:27 +0100
Subject: [PATCH 83/98] Start to move towards meson

---
 pyproject.toml                                |  66 +++++++
 {dahu => src/dahu}/__init__.py                |   0
 .../dahu/app/reprocess.py                     |   7 +-
 .../dahu/app/tango_server.py                  |   6 +-
 {dahu => src/dahu}/cache.py                   |   0
 {dahu => src/dahu}/factory.py                 |   0
 {dahu => src/dahu}/job.py                     |   0
 {dahu => src/dahu}/plugin.py                  |   0
 src/dahu/plugin.py.orig                       | 173 ++++++++++++++++++
 {dahu => src/dahu}/server.py                  |   0
 {dahu => src/dahu}/test/__init__.py           |   0
 {dahu => src/dahu}/test/test_all.py           |   0
 {dahu => src/dahu}/test/test_cache.py         |   0
 {dahu => src/dahu}/test/test_job.py           |   0
 {dahu => src/dahu}/test/test_plugin.py        |   0
 {dahu => src/dahu}/test/utilstest.py          |   0
 {dahu => src/dahu}/utils.py                   |   0
 version.py                                    |  99 ++++++++--
 18 files changed, 336 insertions(+), 15 deletions(-)
 create mode 100644 pyproject.toml
 rename {dahu => src/dahu}/__init__.py (100%)
 rename scripts/dahu-reprocess => src/dahu/app/reprocess.py (99%)
 rename scripts/dahu_server => src/dahu/app/tango_server.py (99%)
 rename {dahu => src/dahu}/cache.py (100%)
 rename {dahu => src/dahu}/factory.py (100%)
 rename {dahu => src/dahu}/job.py (100%)
 rename {dahu => src/dahu}/plugin.py (100%)
 create mode 100644 src/dahu/plugin.py.orig
 rename {dahu => src/dahu}/server.py (100%)
 rename {dahu => src/dahu}/test/__init__.py (100%)
 rename {dahu => src/dahu}/test/test_all.py (100%)
 rename {dahu => src/dahu}/test/test_cache.py (100%)
 rename {dahu => src/dahu}/test/test_job.py (100%)
 rename {dahu => src/dahu}/test/test_plugin.py (100%)
 rename {dahu => src/dahu}/test/utilstest.py (100%)
 rename {dahu => src/dahu}/utils.py (100%)
 mode change 100644 => 100755 version.py

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..3bcd574
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,66 @@
+[project]
+name = 'dahu'
+dynamic = ['version',]
+license = {file = 'copyright'}
+requires-python = '>=3.7'
+readme = 'README.rst'
+description = '"Python lightweight, plugin based, data analysis engine"'
+authors = [
+  { name = 'Jérôme Kieffer', email = 'jerome.kieffer@esrf.fr'},
+]
+
+# double check classifiers on https://pypi.python.org/pypi?%3Aaction=list_classifiers
+classifiers = ["Development Status :: 3 - Alpha",
+               "Environment :: Console",
+               "Environment :: MacOS X",
+               "Environment :: Win32 (MS Windows)",
+               "Environment :: X11 Applications :: Qt",
+               "Intended Audience :: Education",
+               "Intended Audience :: Science/Research",
+               "License :: OSI Approved :: MIT License",
+               "License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)",
+               "Natural Language :: English",
+               "Operating System :: MacOS",
+               "Operating System :: Microsoft :: Windows",
+               "Operating System :: POSIX",
+               "Programming Language :: Cython",
+               "Programming Language :: Python :: 3.6",
+               "Programming Language :: Python :: 3.7",
+               "Programming Language :: Python :: 3.8",
+               "Programming Language :: Python :: Implementation :: CPython",
+               "Topic :: Scientific/Engineering :: Physics",
+               "Topic :: Software Development :: Libraries :: Python Modules",
+               ]
+
+dependencies = [
+   'numpy>=1.10',
+]
+
+[build-system]
+build-backend = 'mesonpy'
+requires = [
+    'meson-python>=0.11',
+    "meson>=0.64; platform_system=='Windows'",
+    "meson>=0.60; platform_system!='Windows'",
+    'ninja',
+    'wheel',
+    'oldest-supported-numpy',
+    'pyproject-metadata>=0.5.0',
+    "tomli>=1.0.0; python_version < '3.12'"
+]
+
+[project.urls]
+homepage = 'http://www.silx.org'
+documentation = 'http://www.silx.org/doc/dahu/latest/'
+source = 'https://github.com/kif/dahu'
+download = 'https://github.com/kif/dahu/releases'
+tracker = 'https://github.com/kif/dahu/issues'
+
+[project.scripts]
+dahu_server = 'dahu.app.server:main'
+dahu-reprocess = 'dahu.app.reprocess:main'
+
+
+[tool.cibuildwheel]
+# Skip 32-bit builds and PyPy
+skip = ["*-win32", "*-manylinux_i686", "pp*", "*musllinux*"]
diff --git a/dahu/__init__.py b/src/dahu/__init__.py
similarity index 100%
rename from dahu/__init__.py
rename to src/dahu/__init__.py
diff --git a/scripts/dahu-reprocess b/src/dahu/app/reprocess.py
similarity index 99%
rename from scripts/dahu-reprocess
rename to src/dahu/app/reprocess.py
index b7d8313..9da7d2f 100644
--- a/scripts/dahu-reprocess
+++ b/src/dahu/app/reprocess.py
@@ -109,6 +109,11 @@ def process(args):
         with open(basename + ".inp", "w") as fp:
             json.dump(plugin.input, fp, indent=4, cls=NumpyEncoder)
 
-if __name__ == "__main__":
+
+def main():
     args = parse()
     process(args)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/dahu_server b/src/dahu/app/tango_server.py
similarity index 99%
rename from scripts/dahu_server
rename to src/dahu/app/tango_server.py
index 4b0535b..7ec7ff4 100644
--- a/scripts/dahu_server
+++ b/src/dahu/app/tango_server.py
@@ -40,7 +40,7 @@
 from argparse import ArgumentParser
 import PyTango
 
-if __name__ == '__main__':
+def main():
     logger.info("Starting Dahu Tango Device Server")
     description = """Data Analysis Tango device server 
 """
@@ -88,3 +88,7 @@
     except Exception as err:
         logger.error('PyTango --> An unforeseen exception occurred....%s' % err)
         sys.exit(-1)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/dahu/cache.py b/src/dahu/cache.py
similarity index 100%
rename from dahu/cache.py
rename to src/dahu/cache.py
diff --git a/dahu/factory.py b/src/dahu/factory.py
similarity index 100%
rename from dahu/factory.py
rename to src/dahu/factory.py
diff --git a/dahu/job.py b/src/dahu/job.py
similarity index 100%
rename from dahu/job.py
rename to src/dahu/job.py
diff --git a/dahu/plugin.py b/src/dahu/plugin.py
similarity index 100%
rename from dahu/plugin.py
rename to src/dahu/plugin.py
diff --git a/src/dahu/plugin.py.orig b/src/dahu/plugin.py.orig
new file mode 100644
index 0000000..0496ae9
--- /dev/null
+++ b/src/dahu/plugin.py.orig
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+
+"""
+Data Analysis RPC server over Tango: 
+
+Definiton of plugins
+"""
+from __future__ import with_statement, print_function, absolute_import, division
+
+__authors__ = ["Jérôme Kieffer"]
+__contact__ = "Jerome.Kieffer@ESRF.eu"
+__license__ = "MIT"
+__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
+<<<<<<< HEAD
+__date__ = "20/07/2020"
+=======
+__date__ = "17/03/2020"
+>>>>>>> origin/master
+__status__ = "production"
+
+from .factory import plugin_factory, register
+from .utils import fully_qualified_name, get_workdir
+import os
+import logging
+import cProfile
+logger = logging.getLogger("dahu.plugin")
+
+
+class Plugin(object):
+    """
+    A plugin is instanciated
+
+    * Gets its input parameters as a dictionary from the setup method
+    * Performs some work in the process
+    * Sets the result as output attribute, should be a dictionary
+    * The process can be an infinite loop or a server which can be aborted using the abort method
+
+    """
+    DEFAULT_SET_UP = "setup"  # name of the method used to set-up the plugin (close connection, files)
+    DEFAULT_PROCESS = "process"  # specify how to run the default processing
+    DEFAULT_TEAR_DOWN = "teardown"  # name of the method used to tear-down the plugin (close connection, files)
+    DEFAULT_ABORT = "abort"  # name of the method used to abort the plugin (if any. Tear_Down will be called)
+    REPROCESS_IGNORE = [] #list of keys from input to be ignored when reprocessing data 
+
+
+    def __init__(self):
+        """
+        We assume an empty constructor
+        """
+        self.input = {}
+        self.output = {}
+        self._logging = []  # stores the logging information to send back
+        self.is_aborted = False
+        self.__profiler = None
+
+    def get_name(self):
+        return self.__class__.__name__
+
+    def setup(self, kwargs=None):
+        """
+        This is the second constructor to setup
+        input variables and possibly initialize
+        some objects
+        """
+        if kwargs is not None:
+            self.input.update(kwargs)
+        if self.input.get("do_profiling"):
+            self.__profiler = cProfile.Profile()
+            self.__profiler.enable()
+
+    def process(self):
+        """
+        main processing of the plugin
+        """
+        pass
+
+    def teardown(self):
+        """
+        method used to tear-down the plugin (close connection, files)
+
+        This is always run, even if process fails
+        """
+        self.output["logging"] = self._logging
+        if self.input.get("do_profiling"):
+            self.__profiler.disable()
+            name = "%05i_%s.%s.profile" % (self.input.get("job_id", 0), self.__class__.__module__, self.__class__.__name__)
+            profile_file = os.path.join(get_workdir(), name)
+            self.log_error("Profiling information in %s" % profile_file, do_raise=False)
+            self.__profiler.dump_stats(profile_file)
+
+    def get_info(self):
+        """
+        """
+        return os.linesep.join(self._logging)
+
+    def abort(self):
+        """
+        Method called to stop a server process
+        """
+        self.is_aborted = True
+
+    def log_error(self, txt, do_raise=True):
+        """
+        Way to log errors and raise error
+        """
+        if do_raise:
+            err = "ERROR in %s: %s" % (self.get_name(), txt)
+            logger.error(err)
+        else:
+            err = "Warning in %s: %s" % (self.get_name(), txt)
+            logger.warning(err)
+        self._logging.append(err)
+        if do_raise:
+            raise RuntimeError(err)
+
+    def log_warning(self, txt):
+        """
+        Way to log warning
+        """
+        err = "Warning in %s: %s" % (self.get_name(), txt)
+        logger.warning(err)
+        self._logging.append(err)
+
+
+class PluginFromFunction(Plugin):
+    """
+    Template class to build  a plugin from a function
+    """
+
+    def __init__(self):
+        """
+        :param funct: function to be wrapped
+        """
+        Plugin.__init__(self)
+
+    def __call__(self, **kwargs):
+        """
+        Behaves like a normal function: for debugging
+        """
+        self.input.update(kwargs)
+        self.process()
+        self.teardown()
+        return self.output["result"]
+
+    def process(self):
+        if self.input is None:
+            logger.warning("PluginFromFunction.process: self.input is None !!! %s", self.input)
+        else:
+            funct_input = self.input.copy()
+            if "job_id" in funct_input:
+                funct_input.pop("job_id")
+            if "plugin_name" in funct_input:
+                funct_input.pop("plugin_name")
+            self.output["result"] = self.function(**funct_input)
+
+
+def plugin_from_function(function):
+    """
+    Create a plugin class from a given function and registers it into the
+
+    :param function: any function
+    :return: plugin name to be used by the plugin_factory to get an instance
+    """
+    logger.debug("creating plugin from function %s" % function.__name__)
+    class_name = function.__module__ + "." + function.__name__
+    klass = type(class_name, (PluginFromFunction,),
+                 {'function': staticmethod(function),
+                 "__doc__": function.__doc__})
+    plugin_factory.register(klass, class_name)
+    return class_name
+
diff --git a/dahu/server.py b/src/dahu/server.py
similarity index 100%
rename from dahu/server.py
rename to src/dahu/server.py
diff --git a/dahu/test/__init__.py b/src/dahu/test/__init__.py
similarity index 100%
rename from dahu/test/__init__.py
rename to src/dahu/test/__init__.py
diff --git a/dahu/test/test_all.py b/src/dahu/test/test_all.py
similarity index 100%
rename from dahu/test/test_all.py
rename to src/dahu/test/test_all.py
diff --git a/dahu/test/test_cache.py b/src/dahu/test/test_cache.py
similarity index 100%
rename from dahu/test/test_cache.py
rename to src/dahu/test/test_cache.py
diff --git a/dahu/test/test_job.py b/src/dahu/test/test_job.py
similarity index 100%
rename from dahu/test/test_job.py
rename to src/dahu/test/test_job.py
diff --git a/dahu/test/test_plugin.py b/src/dahu/test/test_plugin.py
similarity index 100%
rename from dahu/test/test_plugin.py
rename to src/dahu/test/test_plugin.py
diff --git a/dahu/test/utilstest.py b/src/dahu/test/utilstest.py
similarity index 100%
rename from dahu/test/utilstest.py
rename to src/dahu/test/utilstest.py
diff --git a/dahu/utils.py b/src/dahu/utils.py
similarity index 100%
rename from dahu/utils.py
rename to src/dahu/utils.py
diff --git a/version.py b/version.py
old mode 100644
new mode 100755
index f73e37e..05f4fc8
--- a/version.py
+++ b/version.py
@@ -1,7 +1,24 @@
+#!/usr/bin/env python3
 # coding: utf-8
-# Simple module to handle versions
-
-"""
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#  .
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#  .
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+"""Unique place where the version number is defined.
 
 Module for version handling:
 
@@ -27,11 +44,11 @@
 
 """
 
-__author__ = "Jerome Kieffer"
+__author__ = "Jérôme Kieffer"
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "01/09/2020"
+__date__ = "22/02/2024"
 __status__ = "producton"
 __docformat__ = 'restructuredtext'
 RELEASE_LEVEL_VALUE = {"dev": 0,
@@ -41,9 +58,9 @@
                        "rc": 12,
                        "final": 15}
 
-MAJOR = 1
-MINOR = 0
-MICRO = 1
+MAJOR = 2024
+MINOR = 3
+MICRO = 0
 RELEV = "final"  # <16
 SERIAL = 0  # <16
 
@@ -58,13 +75,69 @@
 
 if version_info.releaselevel != "final":
     version += "-%s%s" % version_info[-2:]
+    debianversion += "~adev%i" % version_info[-1] if RELEV == "dev" else "~%s%i" % version_info[-2:]
     prerel = "a" if RELEASE_LEVEL_VALUE.get(version_info[3], 0) < 10 else "b"
     if prerel not in "ab":
         prerel = "a"
     strictversion += prerel + str(version_info[-1])
 
-hexversion = version_info[4]
-hexversion |= RELEASE_LEVEL_VALUE.get(version_info[3], 0) * 1 << 4
-hexversion |= version_info[2] * 1 << 8
-hexversion |= version_info[1] * 1 << 16
-hexversion |= version_info[0] * 1 << 24
+_PATTERN = None
+
+
+def calc_hexversion(major=0, minor=0, micro=0, releaselevel="dev", serial=0, string=None):
+    """Calculate the hexadecimal version number from the tuple version_info:
+
+    :param major: integer
+    :param minor: integer
+    :param micro: integer
+    :param relev: integer or string
+    :param serial: integer
+    :param string: version number as a string
+    :return: integer always increasing with revision numbers
+    """
+    if string is not None:
+        global _PATTERN
+        if _PATTERN is None:
+            import re
+            _PATTERN = re.compile(r"(\d+)\.(\d+)\.(\d+)(\w+)?$")
+        result = _PATTERN.match(string)
+        if result is None:
+            raise ValueError("'%s' is not a valid version" % string)
+        result = result.groups()
+        major, minor, micro = int(result[0]), int(result[1]), int(result[2])
+        releaselevel = result[3]
+        if releaselevel is None:
+            releaselevel = 0
+
+    try:
+        releaselevel = int(releaselevel)
+    except ValueError:
+        releaselevel = RELEASE_LEVEL_VALUE.get(releaselevel, 0)
+
+    hex_version = int(serial)
+    hex_version |= releaselevel * 1 << 4
+    hex_version |= int(micro) * 1 << 8
+    hex_version |= int(minor) * 1 << 16
+    hex_version |= int(major) * 1 << 24
+    return hex_version
+
+
+hexversion = calc_hexversion(*version_info)
+
+citation = "https://doi.org/10.1107/S1600577522007238"
+
+if __name__ == "__main__":
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser(usage="print the version of the software")
+    parser.add_argument("--wheel", action="store_true", dest="wheel", default=None,
+                        help="print version formated for wheel")
+    parser.add_argument("--debian", action="store_true", dest="debian", default=None,
+                        help="print version formated for debian")
+    options = parser.parse_args()
+    if options.debian:
+        print(debianversion)
+    elif options.wheel:
+        print(strictversion)
+    else:
+        print(version)

From 1400316ee15fe60d7fe353364a0ccf5b1e4bbafe Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 10:07:45 +0100
Subject: [PATCH 84/98] fix issue with tif conversion when several files have
 only 1 frame

---
 plugins/id27.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 5611e86..f82d4bd 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -317,7 +317,8 @@ def fabio_conversion(file_path,
     "Convert a set of eiger files to cbf or tiff"
     results = []
     filename = os.path.join(file_path, scan_number, 'eiger_????.h5')
-    files = sorted(glob.glob(filename))
+    files = {f:fabio.open(f).nframes for f in sorted(glob.glob(filename))} #since python 3.7 dict are ordered !
+    all_single = max(files.values())==1
     file_path = file_path.rstrip("/")
 
     splitted = file_path.split("/")
@@ -336,21 +337,24 @@ def fabio_conversion(file_path,
     if len(files) == 0:
         raise RuntimeError(f"No such file {filename}")
 
+    cnt = 0
     for idx_file, filename in enumerate(files):
         img_data_fabio = fabio.open(filename)
-        basename = folder if len(files) == 1 else f"{folder}_{idx_file+1:04d}"
+        basename = folder if (len(files) == 1 or all_single) else f"{folder}_{idx_file+1:04d}"
 
         dest_dir2 = os.path.join(dest_dir, basename)
         if not os.path.exists(dest_dir2):
             with lock:
                 if not os.path.exists(dest_dir2):
                     os.makedirs(dest_dir2)
-        for i, frame in enumerate(img_data_fabio):
+        for i in range(img_data_fabio.nframes):
+            cnt += 1
+            frame = img_data_fabio.getframe(i)
             conv = frame.convert(fabioimage)
             data = conv.data.astype('int32')
             data[data < 0] = 0
             conv.data = data
-            output = os.path.join(dest_dir2, f"{dset_name}_{i+1:04d}.{extension}")
+            output = os.path.join(dest_dir2, f"{dset_name}_{cnt if all_single else i+1:04d}.{extension}")
             conv.write(output)
             results.append(output)
     if export_icat:

From 41c8d1669ffa27a8bcedd5ddd797c19fd7d1a3be Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 11:05:22 +0100
Subject: [PATCH 85/98] Implement fscan3d for diffmap

---
 plugins/id27.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index f82d4bd..6cda69a 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -593,6 +593,7 @@ class DiffMap(Plugin):
     """
     def process(self):
         Plugin.process(self)
+        results = {}
         if not self.input:
             logger.error("input is empty")
 
@@ -605,9 +606,24 @@ def process(self):
             with lock:
                 if not os.path.isdir(dest_dir):
                     os.makedirs(dest_dir)
+                    
+        fast_scan = self.input.get("fast_scan", 1)
+        slow_scan = self.input.get("slow_scan", 1)
+        
+        if len(files)>1 and fast_scan*slow_scan == len(files): 
+            # handle the special case of fscan3d -> each file is reduced prior to integration
+            reduced_files = []
+            for inputname in files:
+                output = os.path.join(dest_dir, os.path.basename(inputname))
+                command = [os.path.join(PREFIX, 'pyFAI-average'), 
+                           '-m', 'sum', '-F', 'lima', '-o', output, filename]
+                results[inputname] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
+                reduced_files.append(output)
+            files = reduced_files
+
         config = os.path.join(dest_dir, "diff_map.json")
         dest = os.path.join(dest_dir, "diff_map.h5")
-        results = {}
+        
         param = {}
         ai = pyFAI.load(self.input.get("ponifile")).get_config()
         if "maskfile" in self.input:
@@ -630,8 +646,8 @@ def process(self):
         param["experiment_title"] = os.path.join(os.path.basename(file_path), scan_number)
         param["fast_motor_name"] = "fast"
         param["slow_motor_name"] = "slow"
-        param["fast_motor_points"] = self.input.get("fast_scan", 1)
-        param["slow_motor_points"] = self.input.get("slow_scan", 1)
+        param["fast_motor_points"] = fast_scan
+        param["slow_motor_points"] = slow_scan
         param["offset"] = 0
         param["output_file"] = dest
         param["input_data"] = [(i, None, None) for i in files]

From b7e896d853ec8e8fd51cb53ba58a9ed914c2f845 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 11:21:21 +0100
Subject: [PATCH 86/98] Declare all meson files

---
 plugins/__init__.py       |  0
 plugins/bm29/meson.build  |  9 +++++++++
 plugins/id02/meson.build  |  9 +++++++++
 plugins/id31/meson.build  |  7 +++++++
 plugins/meson.build       | 12 ++++++++++++
 src/dahu/app/meson.build  |  6 ++++++
 src/dahu/meson.build      | 11 +++++++++++
 src/dahu/test/meson.build | 10 ++++++++++
 version.py                |  4 ++--
 9 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 plugins/__init__.py
 create mode 100644 plugins/bm29/meson.build
 create mode 100644 plugins/id02/meson.build
 create mode 100644 plugins/id31/meson.build
 create mode 100644 plugins/meson.build
 create mode 100644 src/dahu/app/meson.build
 create mode 100644 src/dahu/meson.build
 create mode 100644 src/dahu/test/meson.build

diff --git a/plugins/__init__.py b/plugins/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/plugins/bm29/meson.build b/plugins/bm29/meson.build
new file mode 100644
index 0000000..745a5aa
--- /dev/null
+++ b/plugins/bm29/meson.build
@@ -0,0 +1,9 @@
+py.install_sources( [
+   'common.py',  
+   'hplc.py',  
+   '__init__.py',  
+   'integrate.py',  
+   'ispyb.py',  
+   'nexus.py',  
+   'subtracte.py'
+    ])
\ No newline at end of file
diff --git a/plugins/id02/meson.build b/plugins/id02/meson.build
new file mode 100644
index 0000000..fa0cb38
--- /dev/null
+++ b/plugins/id02/meson.build
@@ -0,0 +1,9 @@
+py.install_sources( [
+    'single_detector.py',
+    '__init__.py',
+    'common.py',
+    'metadata.py',
+    'xpcs.py',
+    'nexus.py'
+    ]
+)
\ No newline at end of file
diff --git a/plugins/id31/meson.build b/plugins/id31/meson.build
new file mode 100644
index 0000000..c97fbd7
--- /dev/null
+++ b/plugins/id31/meson.build
@@ -0,0 +1,7 @@
+py.install_sources(
+[
+'__init__.py',
+'calculate_flux.py'
+]
+
+)
\ No newline at end of file
diff --git a/plugins/meson.build b/plugins/meson.build
new file mode 100644
index 0000000..2d40cd8
--- /dev/null
+++ b/plugins/meson.build
@@ -0,0 +1,12 @@
+subdir('bm29')
+subdir('id02')
+subdir('id31')
+py.install_sources(
+   [
+    'example.py',
+    'focus.py',
+    'id15.py',
+    'id15v2.py',
+    'id27.py',
+    'pyfai.py'
+    ]
\ No newline at end of file
diff --git a/src/dahu/app/meson.build b/src/dahu/app/meson.build
new file mode 100644
index 0000000..c5b6af1
--- /dev/null
+++ b/src/dahu/app/meson.build
@@ -0,0 +1,6 @@
+py.install_sources(
+[
+'reprocess.py',
+'tango_server.py'
+]
+)
\ No newline at end of file
diff --git a/src/dahu/meson.build b/src/dahu/meson.build
new file mode 100644
index 0000000..5cb9ee9
--- /dev/null
+++ b/src/dahu/meson.build
@@ -0,0 +1,11 @@
+py.install_sources(
+[
+'cache.py',
+'__init__.py',
+'job.py',
+'utils.py',
+'factory.py',
+'server.py',
+'plugin.py'
+]
+)
\ No newline at end of file
diff --git a/src/dahu/test/meson.build b/src/dahu/test/meson.build
new file mode 100644
index 0000000..a7007f7
--- /dev/null
+++ b/src/dahu/test/meson.build
@@ -0,0 +1,10 @@
+py.install_sources(
+[
+'__init__.py',
+'utilstest.py',
+'test_all.py',
+'test_plugin.py',
+'test_job.py',
+'test_cache.py'
+]
+)
\ No newline at end of file
diff --git a/version.py b/version.py
index 05f4fc8..e9755b1 100755
--- a/version.py
+++ b/version.py
@@ -59,7 +59,7 @@
                        "final": 15}
 
 MAJOR = 2024
-MINOR = 3
+MINOR = 2
 MICRO = 0
 RELEV = "final"  # <16
 SERIAL = 0  # <16
@@ -71,7 +71,7 @@
 
 version_info = _version_info(MAJOR, MINOR, MICRO, RELEV, SERIAL)
 
-strictversion = version = "%d.%d.%d" % version_info[:3]
+debianversion = strictversion = version = "%d.%d.%d" % version_info[:3]
 
 if version_info.releaselevel != "final":
     version += "-%s%s" % version_info[-2:]

From 956b5cbe1bf6dd55f03b7843495bb60f59d5ec08 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 11:26:32 +0100
Subject: [PATCH 87/98] declare installation path

---
 meson.build               | 33 +++++++++++++++++++++++++++++++++
 plugins/bm29/meson.build  |  5 ++++-
 plugins/id02/meson.build  |  5 ++++-
 plugins/id31/meson.build  |  5 +++--
 plugins/meson.build       |  5 ++++-
 src/dahu/app/meson.build  |  5 ++++-
 src/dahu/meson.build      |  5 ++++-
 src/dahu/test/meson.build |  5 ++++-
 8 files changed, 60 insertions(+), 8 deletions(-)
 create mode 100644 meson.build

diff --git a/meson.build b/meson.build
new file mode 100644
index 0000000..fe2232d
--- /dev/null
+++ b/meson.build
@@ -0,0 +1,33 @@
+project('dahu',
+  license: 'MIT',
+  meson_version: '>= 0.60',
+  version: run_command('version.py',
+  			           check:true).stdout().strip(),
+)
+
+
+# Seek the backend
+if meson.backend() != 'ninja'
+  error('Ninja backend required')
+endif
+
+cc = meson.get_compiler('c')
+m_dep = cc.find_library('m', required : false)
+if m_dep.found()
+  add_project_link_arguments('-lm', language : 'c')
+endif
+
+# https://mesonbuild.com/Python-module.html
+py_mod = import('python')
+py = py_mod.find_installation()
+py_dep = py.dependency()
+
+py.install_sources([
+    'version.py',
+],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'pyFAI'  # Folder relative to site-packages to install to
+)
+
+subdir('src/dahu')
+subdir('plugins')
diff --git a/plugins/bm29/meson.build b/plugins/bm29/meson.build
index 745a5aa..dc1832a 100644
--- a/plugins/bm29/meson.build
+++ b/plugins/bm29/meson.build
@@ -6,4 +6,7 @@ py.install_sources( [
    'ispyb.py',  
    'nexus.py',  
    'subtracte.py'
-    ])
\ No newline at end of file
+    ],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu/plugins/bm29'  # Folder relative to site-packages to install to
+)
\ No newline at end of file
diff --git a/plugins/id02/meson.build b/plugins/id02/meson.build
index fa0cb38..2514b35 100644
--- a/plugins/id02/meson.build
+++ b/plugins/id02/meson.build
@@ -5,5 +5,8 @@ py.install_sources( [
     'metadata.py',
     'xpcs.py',
     'nexus.py'
-    ]
+    ],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu/plugins/id02'  # Folder relative to site-packages to install to
+
 )
\ No newline at end of file
diff --git a/plugins/id31/meson.build b/plugins/id31/meson.build
index c97fbd7..f9164d1 100644
--- a/plugins/id31/meson.build
+++ b/plugins/id31/meson.build
@@ -2,6 +2,7 @@ py.install_sources(
 [
 '__init__.py',
 'calculate_flux.py'
-]
-
+],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu/plugins/id31'  # Folder relative to site-packages to install to
 )
\ No newline at end of file
diff --git a/plugins/meson.build b/plugins/meson.build
index 2d40cd8..6b18ce4 100644
--- a/plugins/meson.build
+++ b/plugins/meson.build
@@ -9,4 +9,7 @@ py.install_sources(
     'id15v2.py',
     'id27.py',
     'pyfai.py'
-    ]
\ No newline at end of file
+    ],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu/plugins'  # Folder relative to site-packages to install to
+)
\ No newline at end of file
diff --git a/src/dahu/app/meson.build b/src/dahu/app/meson.build
index c5b6af1..d5da9de 100644
--- a/src/dahu/app/meson.build
+++ b/src/dahu/app/meson.build
@@ -2,5 +2,8 @@ py.install_sources(
 [
 'reprocess.py',
 'tango_server.py'
-]
+],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu/app'  # Folder relative to site-packages to install to
+
 )
\ No newline at end of file
diff --git a/src/dahu/meson.build b/src/dahu/meson.build
index 5cb9ee9..c312dab 100644
--- a/src/dahu/meson.build
+++ b/src/dahu/meson.build
@@ -7,5 +7,8 @@ py.install_sources(
 'factory.py',
 'server.py',
 'plugin.py'
-]
+],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu'  # Folder relative to site-packages to install to
+
 )
\ No newline at end of file
diff --git a/src/dahu/test/meson.build b/src/dahu/test/meson.build
index a7007f7..72910af 100644
--- a/src/dahu/test/meson.build
+++ b/src/dahu/test/meson.build
@@ -6,5 +6,8 @@ py.install_sources(
 'test_plugin.py',
 'test_job.py',
 'test_cache.py'
-]
+],
+  pure: false,    # Will be installed next to binaries
+  subdir: 'dahu/test'  # Folder relative to site-packages to install to
+
 )
\ No newline at end of file

From 8c429302c32e7dd00b0ef901a3ca4f05fda5aa16 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 11:29:08 +0100
Subject: [PATCH 88/98] create empty file

---
 meson.build     | 8 +-------
 plugins/id27.py | 0
 2 files changed, 1 insertion(+), 7 deletions(-)
 create mode 100644 plugins/id27.py

diff --git a/meson.build b/meson.build
index fe2232d..6c70a67 100644
--- a/meson.build
+++ b/meson.build
@@ -11,12 +11,6 @@ if meson.backend() != 'ninja'
   error('Ninja backend required')
 endif
 
-cc = meson.get_compiler('c')
-m_dep = cc.find_library('m', required : false)
-if m_dep.found()
-  add_project_link_arguments('-lm', language : 'c')
-endif
-
 # https://mesonbuild.com/Python-module.html
 py_mod = import('python')
 py = py_mod.find_installation()
@@ -26,7 +20,7 @@ py.install_sources([
     'version.py',
 ],
   pure: false,    # Will be installed next to binaries
-  subdir: 'pyFAI'  # Folder relative to site-packages to install to
+  subdir: 'dahu'  # Folder relative to site-packages to install to
 )
 
 subdir('src/dahu')
diff --git a/plugins/id27.py b/plugins/id27.py
new file mode 100644
index 0000000..e69de29

From 3ce1ccb81f4d0da0e235605db04064c07023c8ce Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 11:42:30 +0100
Subject: [PATCH 89/98] tests are running

---
 bootstrap.py               | 193 ++++++++++++-------------
 run_tests.py               | 281 ++++++++++++++++++-------------------
 src/dahu/__init__.py       |  10 +-
 src/dahu/meson.build       |   3 +
 src/dahu/test/utilstest.py |   4 +-
 5 files changed, 240 insertions(+), 251 deletions(-)

diff --git a/bootstrap.py b/bootstrap.py
index c0cea59..6ee8bd6 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-
 """
 Bootstrap helps you to test scripts without installing them
 by patching your PYTHONPATH on the fly
@@ -11,72 +10,94 @@
 __authors__ = ["Frédéric-Emmanuel Picca", "Jérôme Kieffer"]
 __contact__ = "jerome.kieffer@esrf.eu"
 __license__ = "MIT"
-__date__ = "06/02/2020"
-
+__date__ = "03/03/2023"
 
 import sys
 import os
-import shutil
-import distutils.util
 import subprocess
 import logging
-
+if sys.version_info[:2] < (3, 11):
+    import tomli
+else:
+    import tomllib as tomli
 logging.basicConfig()
 logger = logging.getLogger("bootstrap")
 
 
-def is_debug_python():
-    """Returns true if the Python interpreter is in debug mode."""
-    try:
-        import sysconfig
-    except ImportError:  # pragma nocover
-        # Python < 2.7
-        import distutils.sysconfig as sysconfig
+def get_project_name(root_dir):
+    """Retrieve project name by running python setup.py --name in root_dir.
 
-    if sysconfig.get_config_var("Py_DEBUG"):
-        return True
+    :param str root_dir: Directory where to run the command.
+    :return: The name of the project stored in root_dir
+    """
+    logger.debug("Getting project name in %s", root_dir)
+    with open("pyproject.toml") as f:
+        pyproject = tomli.loads(f.read())
+    return pyproject.get("project", {}).get("name")
 
-    return hasattr(sys, "gettotalrefcount")
 
+def build_project(name, root_dir):
+    """Build locally the project using meson
 
-def _distutils_dir_name(dname="lib"):
-    """
-    Returns the name of a distutils build directory
+    :param str name: Name of the project.
+    :param str root_dir: Root directory of the project
+    :return: The path to the directory were build was performed
     """
-    platform = distutils.util.get_platform()
-    architecture = "%s.%s-%i.%i" % (dname, platform,
-                                    sys.version_info[0], sys.version_info[1])
-    if is_debug_python():
-        architecture += "-pydebug"
-    return architecture
-
+    extra = []
+    libdir = "lib"
+    if sys.platform == "win32":
+        libdir = "Lib"
+        # extra = ["--buildtype", "plain"]
+
+    build = os.path.join(root_dir, "build")
+    if not(os.path.isdir(build) and os.path.isdir(os.path.join(build, name))):
+        p = subprocess.Popen(["meson", "setup", "build"],
+                         shell=False, cwd=root_dir, env=os.environ)
+        p.wait()
+    p = subprocess.Popen(["meson", "configure", "--prefix", "/"] + extra,
+                     shell=False, cwd=build, env=os.environ)
+    p.wait()
+    p = subprocess.Popen(["meson", "install", "--destdir", "."],
+                     shell=False, cwd=build, env=os.environ)
+    logger.debug("meson install ended with rc= %s", p.wait())
+
+    home = None
+    if os.environ.get("PYBUILD_NAME") == name:
+        # we are in the debian packaging way
+        home = os.environ.get("PYTHONPATH", "").split(os.pathsep)[-1]
+    if not home:
+        if os.environ.get("BUILDPYTHONPATH"):
+            home = os.path.abspath(os.environ.get("BUILDPYTHONPATH", ""))
+        else:
+            if sys.platform == "win32":
+                home = os.path.join(build, libdir, "site-packages")
+            else:
+                python_version = f"python{sys.version_info.major}.{sys.version_info.minor}"
+                home = os.path.join(build, libdir, python_version, "site-packages")
+            home = os.path.abspath(home)
 
-def _distutils_scripts_name():
-    """Return the name of the distrutils scripts sirectory"""
-    f = "scripts-{version[0]}.{version[1]}"
-    return f.format(version=sys.version_info)
+    cnt = 0
+    while not os.path.isdir(home):
+        cnt += 1
+        home = os.path.split(home)[0]
+    for _ in range(cnt):
+        n = os.listdir(home)[0]
+        home = os.path.join(home, n)
 
+    logger.warning("Building %s to %s", name, home)
 
-def _get_available_scripts(path):
-    res = []
-    try:
-        res = " ".join([s.rstrip('.py') for s in os.listdir(path)])
-    except OSError:
-        res = ["no script available, did you ran "
-               "'python setup.py build' before bootstrapping ?"]
-    return res
+    return home
 
 
-if sys.version_info[0] >= 3:  # Python3
-    def execfile(fullpath, globals=None, locals=None):
-        "Python3 implementation for execfile"
-        with open(fullpath) as f:
-            try:
-                data = f.read()
-            except UnicodeDecodeError:
-                raise SyntaxError("Not a Python script")
-            code = compile(data, fullpath, 'exec')
-            exec(code, globals, locals)
+def execfile(fullpath, globals=None, locals=None):
+    "Python3 implementation for execfile"
+    with open(fullpath) as f:
+        try:
+            data = f.read()
+        except UnicodeDecodeError:
+            raise SyntaxError("Not a Python script")
+        code = compile(data, fullpath, 'exec')
+        exec(code, globals, locals)
 
 
 def run_file(filename, argv):
@@ -117,18 +138,16 @@ def run_file(filename, argv):
         run.wait()
 
 
-def run_entry_point(entry_point, argv):
+def run_entry_point(target_name, entry_point, argv):
     """
     Execute an entry_point using the current python context
-    (http://setuptools.readthedocs.io/en/latest/setuptools.html#automatic-script-creation)
 
     :param str entry_point: A string identifying a function from a module
         (NAME = PACKAGE.MODULE:FUNCTION)
+    :param argv: list of arguments
     """
     import importlib
-    elements = entry_point.split("=")
-    target_name = elements[0].strip()
-    elements = elements[1].split(":")
+    elements = entry_point.split(":")
     module_name = elements[0].strip()
     function_name = elements[1].strip()
 
@@ -162,54 +181,27 @@ def find_executable(target):
     if os.path.isfile(target):
         return ("path", os.path.abspath(target))
 
-    # search the file from setup.py
-    import setup
-    config = setup.get_project_configuration(dry_run=True)
-    # scripts from project configuration
-    if "scripts" in config:
-        for script_name in config["scripts"]:
-            if os.path.basename(script) == target:
-                return ("path", os.path.abspath(script_name))
-    # entry-points from project configuration
-    if "entry_points" in config:
-        for kind in config["entry_points"]:
-            for entry_point in config["entry_points"][kind]:
-                elements = entry_point.split("=")
-                name = elements[0].strip()
-                if name == target:
-                    return ("entry_point", entry_point)
-
-    # search the file from env PATH
-    for dirname in os.environ.get("PATH", "").split(os.pathsep):
-        path = os.path.join(dirname, target)
-        if os.path.isfile(path):
-            return ("path", path)
+    # search the executable in pyproject.toml
+    with open(os.path.join(PROJECT_DIR, "pyproject.toml")) as f:
+        pyproject = tomli.loads(f.read())
 
-    return None, None
+    scripts = {}
+    scripts.update(pyproject.get("project", {}).get("scripts", {}))
+    scripts.update(pyproject.get("project", {}).get("gui-scripts", {}))
 
+    for script, entry_point in scripts.items():
+        if script == target:
+            print(script, entry_point)
+            return ("entry_point", target, entry_point)
+    return None, None
 
-home = os.path.dirname(os.path.abspath(__file__))
-LIBPATH = os.path.join(home, 'build', _distutils_dir_name('lib'))
-cwd = os.getcwd()
-os.chdir(home)
-build = subprocess.Popen([sys.executable, "setup.py", "build"],
-                         shell=False, cwd=os.path.dirname(os.path.abspath(__file__)))
-build_rc = build.wait()
-if not os.path.exists(LIBPATH):
-    logger.warning("`lib` directory does not exist, trying common Python3 lib")
-    LIBPATH = os.path.join(os.path.split(LIBPATH)[0], "lib")
-os.chdir(cwd)
-
-if build_rc == 0:
-    logger.info("Build process ended.")
-else:
-    logger.error("Build process ended with rc=%s", build_rc)
-    sys.exit(-1)
 
-dahu_root = os.path.dirname(os.path.abspath(__file__))
-os.environ["DAHU_PLUGINS"] = os.path.join(dahu_root, "plugins")
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_NAME = get_project_name(PROJECT_DIR)
+logger.info("Project name: %s", PROJECT_NAME)
 
 if __name__ == "__main__":
+    LIBPATH = build_project(PROJECT_NAME, PROJECT_DIR)
     if len(sys.argv) < 2:
         logger.warning("usage: ./bootstrap.py <script>\n")
         script = None
@@ -225,14 +217,15 @@ def find_executable(target):
 
     if script:
         argv = sys.argv[2:]
-        kind, target = find_executable(script)
-        if kind == "path":
-            run_file(target, argv)
-        elif kind == "entry_point":
-            run_entry_point(target, argv)
+        res = find_executable(script)
+        if res[0] == "path":
+            run_file(res[1], argv)
+        elif res[0] == "entry_point":
+            run_entry_point(res[1], res[2], argv)
         else:
             logger.error("Script %s not found", script)
     else:
+        logging.info("Running IPython by default")
         logger.info("Patch the sys.argv: %s", sys.argv)
         sys.path.insert(2, "")
         try:
diff --git a/run_tests.py b/run_tests.py
index cfe538a..24a2372 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -2,7 +2,7 @@
 # coding: utf-8
 # /*##########################################################################
 #
-# Copyright (c) 2015-2017 European Synchrotron Radiation Facility
+# Copyright (C) 2015-2023 European Synchrotron Radiation Facility
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -32,19 +32,17 @@
 """
 
 __authors__ = ["Jérôme Kieffer", "Thomas Vincent"]
-__date__ = "07/02/2020"
+__date__ = "22/02/2024"
 __license__ = "MIT"
 
-import distutils.util
+import sys
 import logging
 import os
-import shutil
-import subprocess
-import sys
+from argparse import ArgumentParser
 import time
 import unittest
 import collections
-from argparse import ArgumentParser
+import tempfile
 
 
 class StreamHandlerUnittestReady(logging.StreamHandler):
@@ -77,13 +75,18 @@ def createBasicHandler():
 
 # Use an handler compatible with unittests, else use_buffer is not working
 logging.root.addHandler(createBasicHandler())
+
+# Capture all default warnings
 logging.captureWarnings(True)
+import warnings
+warnings.simplefilter('default')
 
 logger = logging.getLogger("run_tests")
 logger.setLevel(logging.WARNING)
 
 logger.info("Python %s %s", sys.version, tuple.__itemsize__ * 8)
-
+if sys.version_info.major < 3:
+    logger.error("pyFAI no more support Python2")
 
 try:
     import resource
@@ -95,6 +98,7 @@ def createBasicHandler():
     import importlib
     importer = importlib.import_module
 except ImportError:
+
     def importer(name):
         module = __import__(name)
         # returns the leaf module, instead of the root module
@@ -104,7 +108,6 @@ def importer(name):
             module = getattr(module, subname)
             return module
 
-
 try:
     import numpy
 except Exception as error:
@@ -113,26 +116,10 @@ def importer(name):
     logger.info("Numpy %s", numpy.version.version)
 
 
-try:
-    import h5py
-except Exception as error:
-    logger.warning("h5py missing: %s", error)
-else:
-    logger.info("h5py %s", h5py.version.version)
-
-
-def get_project_name(root_dir):
-    """Retrieve project name by running python setup.py --name in root_dir.
-
-    :param str root_dir: Directory where to run the command.
-    :return: The name of the project stored in root_dir
-    """
-    logger.debug("Getting project name in %s", root_dir)
-    p = subprocess.Popen([sys.executable, "setup.py", "--name"],
-                         shell=False, cwd=root_dir, stdout=subprocess.PIPE)
-    name, _stderr_data = p.communicate()
-    logger.debug("subprocess ended with rc= %s", p.returncode)
-    return name.split()[-1].decode('ascii')
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+from bootstrap import get_project_name, build_project
+PROJECT_NAME = get_project_name(PROJECT_DIR)
+logger.info("Project name: %s", PROJECT_NAME)
 
 
 class TextTestResultWithSkipList(unittest.TextTestResult):
@@ -175,8 +162,7 @@ def startTest(self, test):
         if resource:
             self.__mem_start = \
                 resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-        self.__time_start = time.time()
-        unittest.TestResult.startTest(self, test)
+        self.__time_start = time.perf_counter()
 
     def stopTest(self, test):
         unittest.TextTestRunner.resultclass.stopTest(self, test)
@@ -191,11 +177,63 @@ def stopTest(self, test):
         else:
             memusage = 0
         self.logger.info("Time: %.3fs \t RAM: %.3f Mb\t%s",
-                         time.time() - self.__time_start,
+                         time.perf_counter() - self.__time_start,
                          memusage, test.id())
 
 
-def report_rst(cov, package, version="0.0.0", base=""):
+def report_uncovered_files(cov, build_dir, inject_xml=None):
+    """
+    Generate a report of all modules uncovered by the tests
+
+    :param cov: test coverage instance
+    :param str build_dir: Build directory
+    :return: Text report
+    """
+    if build_dir.endswith("pyFAI"):
+        build_dir = os.path.dirname(build_dir)
+    import fnmatch
+
+    existing_files = []
+    for root, _dirnames, filenames in os.walk(PROJECT_NAME):
+        for filename in fnmatch.filter(filenames, '*.py'):
+            existing_files.append(os.path.join(root, filename))
+    existing_files = filter(lambda x: not x.endswith("/setup.py"), existing_files)
+    existing_files = filter(lambda x: "/test/" not in x, existing_files)
+    existing_files = filter(lambda x: "/third_party/" not in x, existing_files)
+
+    if inject_xml is None:
+        fd, fn = tempfile.mkstemp(suffix=".xml")
+        os.close(fd)
+        cov.xml_report(outfile=fn)
+    else:
+        fn = inject_xml
+    from lxml import etree
+    xml = etree.parse(fn)
+    classes = xml.xpath("//class")
+
+    build_dir = os.path.abspath(build_dir)
+
+    covered_files = []
+
+    for cl in classes:
+        filename = cl.get("filename")
+        filename = os.path.abspath(filename)
+        if filename.startswith(build_dir):
+            filename = filename[len(build_dir) + 1:]
+            covered_files.append(filename)
+
+    uncovered_files = set(existing_files) - set(covered_files)
+    uncovered_files = sorted(list(uncovered_files))
+    text = ""
+    text += "Uncovered files (%d)\n" % len(uncovered_files)
+    text += "-" * (len(text) - 1) + "\n"
+    for filename in uncovered_files:
+        text += "* %s\n" % filename
+
+    return text
+
+
+def report_rst(cov, package, version="0.0.0", base="", inject_xml=None):
     """
     Generate a report of test coverage in RST (for Sphinx inclusion)
 
@@ -204,12 +242,14 @@ def report_rst(cov, package, version="0.0.0", base=""):
     :param str base: base directory of modules to include in the report
     :return: RST string
     """
-    import tempfile
-    fd, fn = tempfile.mkstemp(suffix=".xml")
-    os.close(fd)
-    cov.xml_report(outfile=fn)
-
+    if inject_xml is None:
+        fd, fn = tempfile.mkstemp(suffix=".xml")
+        os.close(fd)
+        cov.xml_report(outfile=fn)
+    else:
+        fn = inject_xml
     from lxml import etree
+
     xml = etree.parse(fn)
     classes = xml.xpath("//class")
 
@@ -228,23 +268,22 @@ def report_rst(cov, package, version="0.0.0", base=""):
     for cl in classes:
         name = cl.get("name")
         fname = cl.get("filename")
-        if not os.path.abspath(fname).startswith(base):
-            continue
-        lines = cl.find("lines").getchildren()
-        hits = [int(i.get("hits")) for i in lines]
+        if os.path.abspath(fname).startswith(base):
+            lines = cl.find("lines").getchildren()
+            hits = [int(i.get("hits")) for i in lines]
 
-        sum_hits = sum(hits)
-        sum_lines = len(lines)
+            sum_hits = sum(hits)
+            sum_lines = len(lines)
 
-        cover = 100.0 * sum_hits / sum_lines if sum_lines else 0
+            cover = 100.0 * sum_hits / sum_lines if sum_lines else 0
 
-        if base:
-            name = os.path.relpath(fname, base)
+            if base:
+                name = os.path.relpath(fname, base)
 
-        res.append('   "%s", "%s", "%s", "%.1f %%"' %
-                   (name, sum_lines, sum_hits, cover))
-        tot_sum_lines += sum_lines
-        tot_sum_hits += sum_hits
+            res.append('   "%s", "%s", "%s", "%.1f %%"' %
+                       (name, sum_lines, sum_hits, cover))
+            tot_sum_lines += sum_lines
+            tot_sum_hits += sum_hits
     res.append("")
     res.append('   "%s total", "%s", "%s", "%.1f %%"' %
                (package, tot_sum_lines, tot_sum_hits,
@@ -267,68 +306,15 @@ def is_debug_python():
     return hasattr(sys, "gettotalrefcount")
 
 
-def build_project(name, root_dir):
-    """Run python setup.py build for the project.
-
-    Build directory can be modified by environment variables.
-
-    :param str name: Name of the project.
-    :param str root_dir: Root directory of the project
-    :return: The path to the directory were build was performed
-    """
-    platform = distutils.util.get_platform()
-    architecture = "lib.%s-%i.%i" % (platform,
-                                     sys.version_info[0], sys.version_info[1])
-    if is_debug_python():
-        architecture += "-pydebug"
-
-    if os.environ.get("PYBUILD_NAME") == name:
-        # we are in the debian packaging way
-        home = os.environ.get("PYTHONPATH", "").split(os.pathsep)[-1]
-    elif os.environ.get("BUILDPYTHONPATH"):
-        home = os.path.abspath(os.environ.get("BUILDPYTHONPATH", ""))
-    else:
-        home = os.path.join(root_dir, "build", architecture)
-
-    logger.warning("Building %s to %s", name, home)
-    p = subprocess.Popen([sys.executable, "setup.py", "build"],
-                         shell=False, cwd=root_dir)
-    logger.debug("subprocess ended with rc= %s", p.wait())
-
-    if os.path.isdir(home):
-        return home
-    alt_home = os.path.join(os.path.dirname(home), "lib")
-    if os.path.isdir(alt_home):
-        return alt_home
-
-
-def import_project_module(project_name, project_dir):
-    """Import project module, from the system of from the project directory"""
-    # Prevent importing from source directory
-    if (os.path.dirname(os.path.abspath(__file__)) == os.path.abspath(sys.path[0])):
-        removed_from_sys_path = sys.path.pop(0)
-        logger.info("Patched sys.path, removed: '%s'", removed_from_sys_path)
-
-    if "--installed" in sys.argv:
-        try:
-            module = importer(project_name)
-        except ImportError:
-            raise ImportError(
-                "%s not installed: Cannot run tests on installed version" %
-                PROJECT_NAME)
-    else:  # Use built source
-        build_dir = build_project(project_name, project_dir)
-        if build_dir is None:
-            logging.error("Built project is not available !!! investigate")
-        sys.path.insert(0, build_dir)
-        logger.warning("Patched sys.path, added: '%s'", build_dir)
-        module = importer(project_name)
-    return module
+# Prevent importing from source directory
+if os.path.dirname(os.path.abspath(__file__)) == os.path.abspath(sys.path[0]):
+    removed_from_sys_path = sys.path.pop(0)
+    logger.info("Patched sys.path, removed: '%s'", removed_from_sys_path)
 
 
 def get_test_options(project_module):
     """Returns the test options if available, else None"""
-    module_name = project_module.__name__ + '.test.utils'
+    module_name = project_module.__name__ + '.test.utilstest'
     logger.info('Import %s', module_name)
     try:
         test_utils = importer(module_name)
@@ -340,28 +326,43 @@ def get_test_options(project_module):
     return test_options
 
 
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
-PROJECT_NAME = get_project_name(PROJECT_DIR)
-logger.info("Project name: %s", PROJECT_NAME)
-
-project_module = import_project_module(PROJECT_NAME, PROJECT_DIR)
-PROJECT_VERSION = getattr(project_module, 'version', '')
-PROJECT_PATH = project_module.__path__[0]
-
-test_options = get_test_options(project_module)
-"""Contains extra configuration for the tests."""
-
+if "-i" in sys.argv or "--installed" in sys.argv:
+    for bad_path in (".", os.getcwd(), PROJECT_DIR):
+        if bad_path in sys.path:
+            sys.path.remove(bad_path)
+    try:
+        module = importer(PROJECT_NAME)
+    except Exception:
+        logger.error("Cannot run tests on installed version: %s not installed or raising error.",
+                     PROJECT_NAME)
+        raise
+    else:
+        print("Running tests on system-wide installed project")
+else:
+    build_dir = build_project(PROJECT_NAME, PROJECT_DIR)
+    sys.path.insert(0, build_dir)
+    logger.warning("Patched sys.path, added: '%s'", build_dir)
+    module = importer(PROJECT_NAME)
 
 epilog = """Environment variables:
 WITH_QT_TEST=False to disable graphical tests
-SILX_OPENCL=False to disable OpenCL tests
-SILX_TEST_LOW_MEM=True to disable tests taking large amount of memory
-GPU=False to disable the use of a GPU with OpenCL test
+PYFAI_OPENCL=False to disable OpenCL tests.
+PYFAI_LOW_MEM: set to True to skip all tests >100Mb
 WITH_GL_TEST=False to disable tests using OpenGL
 """
 parser = ArgumentParser(description='Run the tests.',
                         epilog=epilog)
 
+
+test_options = get_test_options(module)
+# """Contains extra configuration for the tests."""
+# test_options.add_parser_argument(parser)
+
+default_test_name = f"{PROJECT_NAME}.test.suite"
+parser.add_argument("test_name", nargs='*',
+                    default=(default_test_name,),
+                    help="Test names to run (Default: %s)" % default_test_name)
+
 parser.add_argument("--installed",
                     action="store_true", dest="installed", default=False,
                     help=("Test the installed version instead of" +
@@ -380,17 +381,10 @@ def get_test_options(project_module):
                          "including debug messages and test help strings.")
 parser.add_argument("--qt-binding", dest="qt_binding", default=None,
                     help="Force using a Qt binding, from 'PyQt4', 'PyQt5', or 'PySide'")
-if test_options is not None:
-    test_options.add_parser_argument(parser)
 
-default_test_name = "%s.test.suite" % PROJECT_NAME
-parser.add_argument("test_name", nargs='*',
-                    default=(default_test_name,),
-                    help="Test names to run (Default: %s)" % default_test_name)
 options = parser.parse_args()
 sys.argv = [sys.argv[0]]
 
-
 test_verbosity = 1
 use_buffer = True
 if options.verbose == 1:
@@ -411,22 +405,18 @@ def get_test_options(project_module):
              # temporary test modules (silx.math.fit.test.test_fitmanager)
              "*customfun.py", ]
     try:
-        cov = coverage.Coverage(omit=omits)
+        coverage_class = coverage.Coverage
     except AttributeError:
-        cov = coverage.coverage(omit=omits)
+        coverage_class = coverage.coverage
+    print(f"|{PROJECT_NAME}|")
+    cov = coverage_class(include=[f"*/{PROJECT_NAME}/*"],
+                         omit=omits)
     cov.start()
 
 if options.qt_binding:
     binding = options.qt_binding.lower()
     if binding == "pyqt4":
         logger.info("Force using PyQt4")
-        if sys.version < "3.0.0":
-            try:
-                import sip
-                sip.setapi("QString", 2)
-                sip.setapi("QVariant", 2)
-            except Exception:
-                logger.warning("Cannot set sip API")
         import PyQt4.QtCore  # noqa
     elif binding == "pyqt5":
         logger.info("Force using PyQt5")
@@ -440,6 +430,9 @@ def get_test_options(project_module):
     else:
         raise ValueError("Qt binding '%s' is unknown" % options.qt_binding)
 
+PROJECT_VERSION = getattr(module, 'version', '')
+PROJECT_PATH = module.__path__[0]
+
 # Run the tests
 runnerArgs = {}
 runnerArgs["verbosity"] = test_verbosity
@@ -456,15 +449,16 @@ def get_test_options(project_module):
 test_module_name = PROJECT_NAME + '.test'
 logger.info('Import %s', test_module_name)
 test_module = importer(test_module_name)
+
 test_suite = unittest.TestSuite()
 
 if test_options is not None:
     # Configure the test options according to the command lines and the the environment
     test_options.configure(options)
+    print(test_options)
 else:
     logger.warning("No test options available.")
 
-
 if not options.test_name:
     # Do not use test loader to avoid cryptic exception
     # when an error occur during import
@@ -481,15 +475,18 @@ def get_test_options(project_module):
 
 if result.wasSuccessful():
     exit_status = 0
+    import pyFAI.test.utilstest
+    pyFAI.test.utilstest.UtilsTest.clean_up()
 else:
-    logger.warning("Test suite failed")
     exit_status = 1
 
-
 if options.coverage:
     cov.stop()
     cov.save()
     with open("coverage.rst", "w") as fn:
         fn.write(report_rst(cov, PROJECT_NAME, PROJECT_VERSION, PROJECT_PATH))
+    print(cov.report())
+    print("")
+    print(report_uncovered_files(cov, PROJECT_PATH))
 
 sys.exit(exit_status)
diff --git a/src/dahu/__init__.py b/src/dahu/__init__.py
index 9fda1e0..e09f977 100644
--- a/src/dahu/__init__.py
+++ b/src/dahu/__init__.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 # /*##########################################################################
 #
-# Copyright (c) 2013-2016 European Synchrotron Radiation Facility
+# Copyright (c) 2013-2024 European Synchrotron Radiation Facility
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -33,16 +33,14 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "17/03/2020"
+__date__ = "22/02/2024"
 __status__ = "production"
 
-from ._version import version, version_info, hexversion, date
-
 import os
 project = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
 try:
-    from ._version import __date__ as date  # noqa
-    from ._version import version, version_info, hexversion, strictversion  # noqa
+    from .version import __date__ as date  # noqa
+    from .version import version, version_info, hexversion, strictversion, citation  # noqa
 except ImportError:
     raise RuntimeError("Do NOT use %s from its sources: build it and use the built version" % project)
 
diff --git a/src/dahu/meson.build b/src/dahu/meson.build
index c312dab..56a2b54 100644
--- a/src/dahu/meson.build
+++ b/src/dahu/meson.build
@@ -1,3 +1,6 @@
+subdir('app')
+subdir('test')
+
 py.install_sources(
 [
 'cache.py',
diff --git a/src/dahu/test/utilstest.py b/src/dahu/test/utilstest.py
index 54c11b1..16d8293 100644
--- a/src/dahu/test/utilstest.py
+++ b/src/dahu/test/utilstest.py
@@ -27,7 +27,7 @@
 __contact__ = "jerome.kieffer@esrf.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "07/02/2020"
+__date__ = "22/02/2024"
 
 PACKAGE = "dahu"
 DATA_KEY = "DAHU_DATA"
@@ -36,12 +36,10 @@
     __name__ = "dahu.test"
 
 import os
-import imp
 import sys
 import getpass
 import subprocess
 import threading
-import distutils.util
 import unittest
 import logging
 try:  # Python3

From d491c4f86fe46ab0c68f065785b5c7174bbf91d2 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 11:53:33 +0100
Subject: [PATCH 90/98] Merge upstream

---
 dahu/app/server.py                 | 93 ------------------------------
 src/dahu/app/meson.build           |  3 +-
 {dahu => src/dahu}/app/register.py |  0
 src/dahu/app/tango_server.py       | 15 +++--
 4 files changed, 11 insertions(+), 100 deletions(-)
 delete mode 100644 dahu/app/server.py
 rename {dahu => src/dahu}/app/register.py (100%)

diff --git a/dahu/app/server.py b/dahu/app/server.py
deleted file mode 100644
index c80ef11..0000000
--- a/dahu/app/server.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-#Example on how to launch it:
-#PYTHONPATH=build/lib.linux-x86_64-2.6/ TANGO_HOST=saxs1:20000 python scripts/dahu_server.py GPU1
-#Example for Jive:
-#"example.square",{\"x\":5}
-"""
-
-Data analysis Tango device server ... for UPBL09a
-
-"""
-__author__ = "Jérôme Kieffer"
-__contact__ = "Jerome.Kieffer@ESRF.eu"
-__license__ = "MIT"
-__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "17/12/2021"
-__status__ = "beta"
-__docformat__ = 'restructuredtext'
-
-import sys
-import os
-import tempfile
-import logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger("dahu_server")
-try:
-    from rfoo.utils import rconsole
-    rconsole.spawn_server()
-except ImportError:
-    logger.debug("No socket opened for debugging. Please install rfoo")
-
-
-import dahu.utils
-from dahu.server import DahuDSClass, DahuDS
-
-# set loglevel at least at INFO
-if logger.getEffectiveLevel() > logging.INFO:
-    logger.setLevel(logging.INFO)
-from argparse import ArgumentParser
-import PyTango
-
-def main(argv=None):
-    if argv is None:
-        argv = sys.argv
-    logger.info("Starting Dahu Tango Device Server")
-    description = """Data Analysis Tango device server 
-"""
-    epilog = """ Provided by the Data analysis unit - ESRF 
-        """
-    parser = ArgumentParser(description=description, epilog=epilog, add_help=True)
-    parser.add_argument("-V", "--version", action='version', version='%(prog)s 0.0')
-    parser.add_argument("-d", "--debug", dest="debug", default=False,
-                        action="store_true", help="Switch to debug mode ",)
-    parser.add_argument("-v", "--verbose", dest="tango_verbose", default=None,
-                        help="tango trace level")
-    parser.add_argument("-f", "--file", dest="tango_file", default=None,
-                        help="tango log filename")
-    parser.add_argument("-l", "--log", dest="dahu_log", 
-                        default=os.path.join(os.environ.get("HOME",tempfile.gettempdir()),"log"),
-                        help="directory where dahu stores logs ")
-#    parser.add_argument("-n", "--nbcpu", dest="nbcpu", type=int,
-#                  help="Maximum bumber of processing threads to be started", default=None)
-
-    options, tango_options = parser.parse_known_args(argv[1:])
-    dahu.utils.get_workdir(options.dahu_log)
-    if options.debug:
-        logging.root.setLevel(logging.DEBUG)
-
-    tangoParam = ["DahuDS"] + tango_options
-    if options.tango_verbose:
-        tangoParam += ["-v%s" % options.tango_verbose]
-    if options.tango_file:
-        tangoParam += ["-file=%s" % options.tango_file]
-
-    try:
-        print(tangoParam)
-        py = PyTango.Util(tangoParam)
-        py.add_TgClass(DahuDSClass, DahuDS, 'DahuDS')
-        U = py.instance() #PyTango.Util.instance()
-        U.server_init()
-        U.server_run()
-    except PyTango.DevFailed as err:
-        logger.error('PyTango --> Received a DevFailed exception: %s' % err)
-        return -1
-    except Exception as err:
-        logger.error('PyTango --> An unforeseen exception occurred....%s' % err)
-        return -1
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main())
diff --git a/src/dahu/app/meson.build b/src/dahu/app/meson.build
index d5da9de..334b6a1 100644
--- a/src/dahu/app/meson.build
+++ b/src/dahu/app/meson.build
@@ -1,7 +1,8 @@
 py.install_sources(
 [
 'reprocess.py',
-'tango_server.py'
+'tango_server.py',
+'register.py'
 ],
   pure: false,    # Will be installed next to binaries
   subdir: 'dahu/app'  # Folder relative to site-packages to install to
diff --git a/dahu/app/register.py b/src/dahu/app/register.py
similarity index 100%
rename from dahu/app/register.py
rename to src/dahu/app/register.py
diff --git a/src/dahu/app/tango_server.py b/src/dahu/app/tango_server.py
index 7ec7ff4..2100b4e 100644
--- a/src/dahu/app/tango_server.py
+++ b/src/dahu/app/tango_server.py
@@ -40,7 +40,9 @@
 from argparse import ArgumentParser
 import PyTango
 
-def main():
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
     logger.info("Starting Dahu Tango Device Server")
     description = """Data Analysis Tango device server 
 """
@@ -65,9 +67,9 @@ def main():
     dahu.utils.get_workdir(options.dahu_log)
     tangoParam = ["DahuDS"] + options.tango
     if options.tango_verbose:
-        tangoParam += ["-v%s" % options.tango_verbose]
+        tangoParam.append(f"-v{options.tango_verbose}"
     if options.tango_file:
-        tangoParam += ["-file=%s" % options.tango_file]
+        tangoParam.append(f"-file={options.tango_file}"
 
     # Analyse arguments and options
     if options.debug:
@@ -84,11 +86,12 @@ def main():
         U.server_run()
     except PyTango.DevFailed as err:
         logger.error('PyTango --> Received a DevFailed exception: %s' % err)
-        sys.exit(-1)
+        return -1
     except Exception as err:
         logger.error('PyTango --> An unforeseen exception occurred....%s' % err)
-        sys.exit(-1)
+        return -1
+    return 0
 
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())

From 70ca13f83df275b6dbfcc89b53a8679408943a9c Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 12:00:34 +0100
Subject: [PATCH 91/98] the code is actually MIT licensed, not GPL

---
 CHANGELOG.txt  |   7 +
 LICENSE        | 356 +++----------------------------------------------
 pyproject.toml |   2 +-
 3 files changed, 25 insertions(+), 340 deletions(-)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index f9d7b7f..b453a72 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,3 +1,10 @@
+dahu 2024.02
+============
+* Move to meson-build
+* Update BM29
+* Update ID02
+* Update ID27
+
 from version 0.4 -> 0.5
 =======================
 
diff --git a/LICENSE b/LICENSE
index d7f1051..2746584 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,339 +1,17 @@
-GNU GENERAL PUBLIC LICENSE
-                       Version 2, June 1991
-
- Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-                            NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    {description}
-    Copyright (C) {year}  {fullname}
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  {signature of Ty Coon}, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
diff --git a/pyproject.toml b/pyproject.toml
index 3bcd574..242e74c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = 'dahu'
 dynamic = ['version',]
-license = {file = 'copyright'}
+license = {file = 'LICENSE'}
 requires-python = '>=3.7'
 readme = 'README.rst'
 description = '"Python lightweight, plugin based, data analysis engine"'

From 9830d7844410f651783a4a4d293bf7b42516e1b9 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 12:02:53 +0100
Subject: [PATCH 92/98] rename files related to tango

---
 pyproject.toml                                  | 4 ++--
 src/dahu/app/meson.build                        | 2 +-
 src/dahu/app/{register.py => tango_register.py} | 0
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename src/dahu/app/{register.py => tango_register.py} (100%)

diff --git a/pyproject.toml b/pyproject.toml
index 242e74c..480dac8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,9 +57,9 @@ download = 'https://github.com/kif/dahu/releases'
 tracker = 'https://github.com/kif/dahu/issues'
 
 [project.scripts]
-dahu_server = 'dahu.app.server:main'
+dahu_server = 'dahu.app.tango_server:main'
 dahu-reprocess = 'dahu.app.reprocess:main'
-
+dahu-register = 'dahu.app.tango_register:main'
 
 [tool.cibuildwheel]
 # Skip 32-bit builds and PyPy
diff --git a/src/dahu/app/meson.build b/src/dahu/app/meson.build
index 334b6a1..63cd6d3 100644
--- a/src/dahu/app/meson.build
+++ b/src/dahu/app/meson.build
@@ -2,7 +2,7 @@ py.install_sources(
 [
 'reprocess.py',
 'tango_server.py',
-'register.py'
+'tango_register.py'
 ],
   pure: false,    # Will be installed next to binaries
   subdir: 'dahu/app'  # Folder relative to site-packages to install to
diff --git a/src/dahu/app/register.py b/src/dahu/app/tango_register.py
similarity index 100%
rename from src/dahu/app/register.py
rename to src/dahu/app/tango_register.py

From cb9bd10bb4d254d844265f0f269163b319e8cb3e Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 13:35:08 +0100
Subject: [PATCH 93/98] fix bootstrap

---
 bootstrap.py                 | 7 ++++---
 src/dahu/app/tango_server.py | 8 ++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/bootstrap.py b/bootstrap.py
index 6ee8bd6..39d07db 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -10,7 +10,7 @@
 __authors__ = ["Frédéric-Emmanuel Picca", "Jérôme Kieffer"]
 __contact__ = "jerome.kieffer@esrf.eu"
 __license__ = "MIT"
-__date__ = "03/03/2023"
+__date__ = "22/02/2024"
 
 import sys
 import os
@@ -30,8 +30,9 @@ def get_project_name(root_dir):
     :param str root_dir: Directory where to run the command.
     :return: The name of the project stored in root_dir
     """
-    logger.debug("Getting project name in %s", root_dir)
-    with open("pyproject.toml") as f:
+    print(f"Getting project name in {root_dir}")
+    
+    with open(os.path.join(root_dir, "pyproject.toml")) as f:
         pyproject = tomli.loads(f.read())
     return pyproject.get("project", {}).get("name")
 
diff --git a/src/dahu/app/tango_server.py b/src/dahu/app/tango_server.py
index 2100b4e..96581ee 100644
--- a/src/dahu/app/tango_server.py
+++ b/src/dahu/app/tango_server.py
@@ -14,7 +14,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "17/12/2021"
+__date__ = "22/02/2024"
 __status__ = "beta"
 __docformat__ = 'restructuredtext'
 
@@ -67,15 +67,15 @@ def main(argv=None):
     dahu.utils.get_workdir(options.dahu_log)
     tangoParam = ["DahuDS"] + options.tango
     if options.tango_verbose:
-        tangoParam.append(f"-v{options.tango_verbose}"
+        tangoParam.append(f"-v{options.tango_verbose}")
     if options.tango_file:
-        tangoParam.append(f"-file={options.tango_file}"
+        tangoParam.append(f"-file={options.tango_file}")
 
     # Analyse arguments and options
     if options.debug:
         logger.debug("Switch logger to debug level")
         logger.setLevel(logging.DEBUG)
-        logging.root.etLevel(logging.DEBUG)
+        logging.root.setLevel(logging.DEBUG)
 
     try:
         print(tangoParam)

From b86b3509a9ad95a3aa72c096aebd3085c9d02269 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 13:58:54 +0100
Subject: [PATCH 94/98] work on ID27 ... fscan3d + diffmap

---
 plugins/id27.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 6cda69a..8252980 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -614,8 +614,9 @@ def process(self):
             # handle the special case of fscan3d -> each file is reduced prior to integration
             reduced_files = []
             for inputname in files:
-                output = os.path.join(dest_dir, os.path.basename(inputname))
-                command = [os.path.join(PREFIX, 'pyFAI-average'), 
+                output = os.path.join(dest_dir, os.path.basename(inputname).replace(".h5",".sum.h5"))
+                # print(inputname, output)
+                command = [os.path.join(PREFIX, 'pyFAI-average'), "--quiet",
                            '-m', 'sum', '-F', 'lima', '-o', output, filename]
                 results[inputname] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
                 reduced_files.append(output)

From 74cf58bfc3165978336f35d96d80b51bc7609caf Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 14:10:02 +0100
Subject: [PATCH 95/98] fix pipeline for ID27

---
 plugins/id27.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/plugins/id27.py b/plugins/id27.py
index 8252980..47a6d19 100644
--- a/plugins/id27.py
+++ b/plugins/id27.py
@@ -610,14 +610,15 @@ def process(self):
         fast_scan = self.input.get("fast_scan", 1)
         slow_scan = self.input.get("slow_scan", 1)
         
+        # handle the special case of fscan3d -> each file is reduced prior to integration
         if len(files)>1 and fast_scan*slow_scan == len(files): 
-            # handle the special case of fscan3d -> each file is reduced prior to integration
+            
             reduced_files = []
             for inputname in files:
                 output = os.path.join(dest_dir, os.path.basename(inputname).replace(".h5",".sum.h5"))
                 # print(inputname, output)
                 command = [os.path.join(PREFIX, 'pyFAI-average'), "--quiet",
-                           '-m', 'sum', '-F', 'lima', '-o', output, filename]
+                           '-m', 'sum', '-F', 'lima', '-o', output, inputname]
                 results[inputname] = unpack_processed(subprocess.run(command, capture_output=True, check=False))
                 reduced_files.append(output)
             files = reduced_files
@@ -664,6 +665,8 @@ def process(self):
             l = file_path.split("/")
             i = l.index(RAW)
             sample_name = l[i+1]
+        else:
+            sample_name = "unknown"
         metadata = {"definition": "diffmap",
                     "Sample_name": sample_name}
         try:

From 753af963ddd3a1fc1eca4490cea1214fba408f5b Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 14:21:38 +0100
Subject: [PATCH 96/98] requirements on meson

---
 ci/requirements_rtd.txt    | 3 ---
 ci/requirements_travis.txt | 2 --
 requirements.txt           | 3 ++-
 3 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/ci/requirements_rtd.txt b/ci/requirements_rtd.txt
index c829587..26db981 100644
--- a/ci/requirements_rtd.txt
+++ b/ci/requirements_rtd.txt
@@ -1,9 +1,6 @@
 numpy
 h5py
-fabio
 sphinx
-scipy
-cython
 sphinxcontrib-programoutput
 silx
 nbsphinx
diff --git a/ci/requirements_travis.txt b/ci/requirements_travis.txt
index 0863247..baf2adb 100644
--- a/ci/requirements_travis.txt
+++ b/ci/requirements_travis.txt
@@ -2,7 +2,6 @@
 
 wheel
 numpy
-cython
 pillow
 lxml
 unittest2
@@ -11,4 +10,3 @@ fabio
 scipy
 matplotlib
 numexpr
-silx
diff --git a/requirements.txt b/requirements.txt
index 9027bd8..b704074 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
 numpy
-six
\ No newline at end of file
+six
+meson-python

From 14bd2566b9e51db276ca748672bdda7c07781bc4 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 14:23:10 +0100
Subject: [PATCH 97/98] request meson & ninja as well

---
 requirements.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/requirements.txt b/requirements.txt
index b704074..1187a54 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
 numpy
 six
 meson-python
+meson
+ninja
\ No newline at end of file

From 66975856fa078692af564a2b9983ca2be8ea0720 Mon Sep 17 00:00:00 2001
From: Jerome Kieffer <jerome.kieffer@esrf.fr>
Date: Thu, 22 Feb 2024 14:25:29 +0100
Subject: [PATCH 98/98] typo in dev tools

---
 run_tests.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/run_tests.py b/run_tests.py
index 24a2372..88a0e76 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -86,7 +86,7 @@ def createBasicHandler():
 
 logger.info("Python %s %s", sys.version, tuple.__itemsize__ * 8)
 if sys.version_info.major < 3:
-    logger.error("pyFAI no more support Python2")
+    logger.error("dahu no more support Python2")
 
 try:
     import resource
@@ -189,7 +189,7 @@ def report_uncovered_files(cov, build_dir, inject_xml=None):
     :param str build_dir: Build directory
     :return: Text report
     """
-    if build_dir.endswith("pyFAI"):
+    if build_dir.endswith("dahu"):
         build_dir = os.path.dirname(build_dir)
     import fnmatch
 
@@ -475,8 +475,8 @@ def get_test_options(project_module):
 
 if result.wasSuccessful():
     exit_status = 0
-    import pyFAI.test.utilstest
-    pyFAI.test.utilstest.UtilsTest.clean_up()
+    import dahu.test.utilstest
+    dahu.test.utilstest.UtilsTest.clean_up()
 else:
     exit_status = 1