From 17d9a5938a9527bf88517348c64a54630982222f Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Tue, 26 Dec 2023 22:57:50 +0100 Subject: [PATCH 1/5] disable file locking like pymca --- pyEvalData/io/sardana_nexus.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyEvalData/io/sardana_nexus.py b/pyEvalData/io/sardana_nexus.py index ad30613..36bd049 100644 --- a/pyEvalData/io/sardana_nexus.py +++ b/pyEvalData/io/sardana_nexus.py @@ -25,7 +25,7 @@ from numpy.core.records import fromarrays import nexusformat.nexus as nxs -import os.path as path +import os from .source import Source from .scan import Scan @@ -95,8 +95,9 @@ def parse_raw(self): """ self.log.info('parse_raw') - nxs_file_path = path.join(self.file_path, self.file_name) + nxs_file_path = os.path.join(self.file_path, self.file_name) try: + os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" nxs_file = nxs.nxload(nxs_file_path, mode='r') except nxs.NeXusError: raise nxs.NeXusError('Sardana NeXus file \'{:s}\' does not exist!'.format( @@ -133,6 +134,8 @@ def parse_raw(self): if self.read_all_data: self.read_scan_data(self.scan_dict[entry_number]) + nxs_file.close() + def read_raw_scan_data(self, scan): """read_raw_scan_data @@ -144,8 +147,9 @@ def read_raw_scan_data(self, scan): """ self.log.info('read_raw_scan_data for scan #{:d}'.format(scan.number)) # try to open the file - nxs_file_path = path.join(self.file_path, self.file_name) + nxs_file_path = os.path.join(self.file_path, self.file_name) try: + os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" nxs_file = nxs.nxload(nxs_file_path, mode='r') except nxs.NeXusError: raise nxs.NeXusError('Sardana NeXus file \'{:s}\' does not exist!'.format( @@ -172,3 +176,5 @@ def read_raw_scan_data(self, scan): scan.data = fromarrays(data_list, dtype=dtype_list) else: scan.date = None + + nxs_file.close() From b817cd432c895902511647298c10dc44016c8114 Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Wed, 27 Dec 2023 21:13:17 +0100 Subject: [PATCH 2/5] use with statement for nexus file --- pyEvalData/io/sardana_nexus.py | 118 ++++++++++++++++----------------- 1 file changed, 56 insertions(+), 62 deletions(-) diff --git a/pyEvalData/io/sardana_nexus.py b/pyEvalData/io/sardana_nexus.py index 36bd049..115c530 100644 --- a/pyEvalData/io/sardana_nexus.py +++ b/pyEvalData/io/sardana_nexus.py @@ -98,43 +98,38 @@ def parse_raw(self): nxs_file_path = os.path.join(self.file_path, self.file_name) try: os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" - nxs_file = nxs.nxload(nxs_file_path, mode='r') - except nxs.NeXusError: - raise nxs.NeXusError('Sardana NeXus file \'{:s}\' does not exist!'.format( - nxs_file_path)) - - with nxs_file.nxfile: - for entry in nxs_file: - # check for scan number in given range - entry_number = int(nxs_file[entry].entry_identifier) - if (entry_number >= self.start_scan_number) and \ - ((entry_number <= self.stop_scan_number) or - (self.stop_scan_number == -1)): - last_scan_number = self.get_last_scan_number() - # check if Scan needs to be re-created - # if scan is not present, its the last one, or force overwrite - if (entry_number not in self.scan_dict.keys()) or \ - (entry_number >= last_scan_number) or \ - self.force_overwrite: - # create scan object - init_mopo = {} - for field in nxs_file[entry].measurement.pre_scan_snapshot: - init_mopo[field] = \ - nxs_file[entry]['measurement/pre_scan_snapshot'][field] - - scan = Scan(int(entry_number), - cmd=nxs_file[entry].title, - date=nxs_file[entry].start_time, - time=nxs_file[entry].start_time, - int_time=float(0), - header='', - init_mopo=init_mopo) - self.scan_dict[entry_number] = scan - # check if the data needs to be read as well - if self.read_all_data: - self.read_scan_data(self.scan_dict[entry_number]) - - nxs_file.close() + with nxs.nxload(nxs_file_path, mode='r') as nxs_file: + for entry in nxs_file: + # check for scan number in given range + entry_number = int(nxs_file[entry].entry_identifier) + if (entry_number >= self.start_scan_number) and \ + ((entry_number <= self.stop_scan_number) or + (self.stop_scan_number == -1)): + last_scan_number = self.get_last_scan_number() + # check if Scan needs to be re-created + # if scan is not present, its the last one, or force overwrite + if (entry_number not in self.scan_dict.keys()) or \ + (entry_number >= last_scan_number) or \ + self.force_overwrite: + # create scan object + init_mopo = {} + for field in nxs_file[entry].measurement.pre_scan_snapshot: + init_mopo[field] = \ + nxs_file[entry]['measurement/pre_scan_snapshot'][field] + + scan = Scan(int(entry_number), + cmd=nxs_file[entry].title, + date=nxs_file[entry].start_time, + time=nxs_file[entry].start_time, + int_time=float(0), + header='', + init_mopo=init_mopo) + self.scan_dict[entry_number] = scan + # check if the data needs to be read as well + if self.read_all_data: + self.read_scan_data(self.scan_dict[entry_number]) + except nxs.NeXusError as e: + raise nxs.NeXusError(e) def read_raw_scan_data(self, scan): """read_raw_scan_data @@ -146,35 +141,34 @@ def read_raw_scan_data(self, scan): """ self.log.info('read_raw_scan_data for scan #{:d}'.format(scan.number)) + entry_name = 'entry{:d}'.format(scan.number) # try to open the file nxs_file_path = os.path.join(self.file_path, self.file_name) try: os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" - nxs_file = nxs.nxload(nxs_file_path, mode='r') + with nxs.nxload(nxs_file_path, mode='r') as nxs_file: + # try to enter entry + try: + entry = nxs_file[entry_name] + except nxs.NeXusError: + self.log.exception('Entry #{:d} not present in NeXus file!'.format( + scan.number)) + return + # iterate through data fields + data_list = [] + dtype_list = [] + for field in entry.measurement: + # do not add data which is already in the pre-scan snapshot + # that is tricky if it is in the snapshot and scanned ... + if field != 'pre_scan_snapshot': + data_list.append(entry.measurement[field]) + dtype_list.append((field, + entry.measurement[field].dtype, + entry.measurement[field].shape)) + if len(data_list) > 0: + scan.data = fromarrays(data_list, dtype=dtype_list) + else: + scan.date = None except nxs.NeXusError: raise nxs.NeXusError('Sardana NeXus file \'{:s}\' does not exist!'.format( nxs_file_path)) - entry_name = 'entry{:d}'.format(scan.number) - # try to enter entry - try: - entry = nxs_file[entry_name] - except nxs.NeXusError: - self.log.exception('Entry #{:d} not present in NeXus file!'.format(scan.number)) - return - # iterate through data fields - data_list = [] - dtype_list = [] - for field in entry.measurement: - # do not add data which is already in the pre-scan snapshot - # that is tricky if it is in the snapshot and scanned ... - if field != 'pre_scan_snapshot': - data_list.append(entry.measurement[field]) - dtype_list.append((field, - entry.measurement[field].dtype, - entry.measurement[field].shape)) - if len(data_list) > 0: - scan.data = fromarrays(data_list, dtype=dtype_list) - else: - scan.date = None - - nxs_file.close() From 080239225d85d2b77a81afa84738fe968544f354 Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Wed, 27 Dec 2023 21:35:50 +0100 Subject: [PATCH 3/5] Revert "use with statement for nexus file" This reverts commit b817cd432c895902511647298c10dc44016c8114. --- pyEvalData/io/sardana_nexus.py | 118 +++++++++++++++++---------------- 1 file changed, 62 insertions(+), 56 deletions(-) diff --git a/pyEvalData/io/sardana_nexus.py b/pyEvalData/io/sardana_nexus.py index 115c530..36bd049 100644 --- a/pyEvalData/io/sardana_nexus.py +++ b/pyEvalData/io/sardana_nexus.py @@ -98,38 +98,43 @@ def parse_raw(self): nxs_file_path = os.path.join(self.file_path, self.file_name) try: os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" - with nxs.nxload(nxs_file_path, mode='r') as nxs_file: - for entry in nxs_file: - # check for scan number in given range - entry_number = int(nxs_file[entry].entry_identifier) - if (entry_number >= self.start_scan_number) and \ - ((entry_number <= self.stop_scan_number) or - (self.stop_scan_number == -1)): - last_scan_number = self.get_last_scan_number() - # check if Scan needs to be re-created - # if scan is not present, its the last one, or force overwrite - if (entry_number not in self.scan_dict.keys()) or \ - (entry_number >= last_scan_number) or \ - self.force_overwrite: - # create scan object - init_mopo = {} - for field in nxs_file[entry].measurement.pre_scan_snapshot: - init_mopo[field] = \ - nxs_file[entry]['measurement/pre_scan_snapshot'][field] - - scan = Scan(int(entry_number), - cmd=nxs_file[entry].title, - date=nxs_file[entry].start_time, - time=nxs_file[entry].start_time, - int_time=float(0), - header='', - init_mopo=init_mopo) - self.scan_dict[entry_number] = scan - # check if the data needs to be read as well - if self.read_all_data: - self.read_scan_data(self.scan_dict[entry_number]) - except nxs.NeXusError as e: - raise nxs.NeXusError(e) + nxs_file = nxs.nxload(nxs_file_path, mode='r') + except nxs.NeXusError: + raise nxs.NeXusError('Sardana NeXus file \'{:s}\' does not exist!'.format( + nxs_file_path)) + + with nxs_file.nxfile: + for entry in nxs_file: + # check for scan number in given range + entry_number = int(nxs_file[entry].entry_identifier) + if (entry_number >= self.start_scan_number) and \ + ((entry_number <= self.stop_scan_number) or + (self.stop_scan_number == -1)): + last_scan_number = self.get_last_scan_number() + # check if Scan needs to be re-created + # if scan is not present, its the last one, or force overwrite + if (entry_number not in self.scan_dict.keys()) or \ + (entry_number >= last_scan_number) or \ + self.force_overwrite: + # create scan object + init_mopo = {} + for field in nxs_file[entry].measurement.pre_scan_snapshot: + init_mopo[field] = \ + nxs_file[entry]['measurement/pre_scan_snapshot'][field] + + scan = Scan(int(entry_number), + cmd=nxs_file[entry].title, + date=nxs_file[entry].start_time, + time=nxs_file[entry].start_time, + int_time=float(0), + header='', + init_mopo=init_mopo) + self.scan_dict[entry_number] = scan + # check if the data needs to be read as well + if self.read_all_data: + self.read_scan_data(self.scan_dict[entry_number]) + + nxs_file.close() def read_raw_scan_data(self, scan): """read_raw_scan_data @@ -141,34 +146,35 @@ def read_raw_scan_data(self, scan): """ self.log.info('read_raw_scan_data for scan #{:d}'.format(scan.number)) - entry_name = 'entry{:d}'.format(scan.number) # try to open the file nxs_file_path = os.path.join(self.file_path, self.file_name) try: os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" - with nxs.nxload(nxs_file_path, mode='r') as nxs_file: - # try to enter entry - try: - entry = nxs_file[entry_name] - except nxs.NeXusError: - self.log.exception('Entry #{:d} not present in NeXus file!'.format( - scan.number)) - return - # iterate through data fields - data_list = [] - dtype_list = [] - for field in entry.measurement: - # do not add data which is already in the pre-scan snapshot - # that is tricky if it is in the snapshot and scanned ... - if field != 'pre_scan_snapshot': - data_list.append(entry.measurement[field]) - dtype_list.append((field, - entry.measurement[field].dtype, - entry.measurement[field].shape)) - if len(data_list) > 0: - scan.data = fromarrays(data_list, dtype=dtype_list) - else: - scan.date = None + nxs_file = nxs.nxload(nxs_file_path, mode='r') except nxs.NeXusError: raise nxs.NeXusError('Sardana NeXus file \'{:s}\' does not exist!'.format( nxs_file_path)) + entry_name = 'entry{:d}'.format(scan.number) + # try to enter entry + try: + entry = nxs_file[entry_name] + except nxs.NeXusError: + self.log.exception('Entry #{:d} not present in NeXus file!'.format(scan.number)) + return + # iterate through data fields + data_list = [] + dtype_list = [] + for field in entry.measurement: + # do not add data which is already in the pre-scan snapshot + # that is tricky if it is in the snapshot and scanned ... + if field != 'pre_scan_snapshot': + data_list.append(entry.measurement[field]) + dtype_list.append((field, + entry.measurement[field].dtype, + entry.measurement[field].shape)) + if len(data_list) > 0: + scan.data = fromarrays(data_list, dtype=dtype_list) + else: + scan.date = None + + nxs_file.close() From c19d1499081b6f4e4a9013d8d5bad2736109ecad Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Wed, 27 Dec 2023 21:39:28 +0100 Subject: [PATCH 4/5] remove close statement --- pyEvalData/io/sardana_nexus.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyEvalData/io/sardana_nexus.py b/pyEvalData/io/sardana_nexus.py index 36bd049..0be6f71 100644 --- a/pyEvalData/io/sardana_nexus.py +++ b/pyEvalData/io/sardana_nexus.py @@ -134,8 +134,6 @@ def parse_raw(self): if self.read_all_data: self.read_scan_data(self.scan_dict[entry_number]) - nxs_file.close() - def read_raw_scan_data(self, scan): """read_raw_scan_data @@ -176,5 +174,3 @@ def read_raw_scan_data(self, scan): scan.data = fromarrays(data_list, dtype=dtype_list) else: scan.date = None - - nxs_file.close() From e3c8d03c6cbfc1b8ffd6c306faed7eebcec08fa9 Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Wed, 27 Dec 2023 21:43:07 +0100 Subject: [PATCH 5/5] disable h5 file locking for PAL FEL source --- pyEvalData/io/palxfel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyEvalData/io/palxfel.py b/pyEvalData/io/palxfel.py index c80c039..9c70000 100644 --- a/pyEvalData/io/palxfel.py +++ b/pyEvalData/io/palxfel.py @@ -127,6 +127,7 @@ def parse_raw(self): self.file_name.format(scan_number) + '.h5') try: + os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" with h5py.File(h5_file, 'r') as h5: header = h5['R{0:04d}/header'.format(scan_number)] @@ -171,6 +172,7 @@ def read_raw_scan_data(self, scan): self.file_name.format(scan.number), self.file_name.format(scan.number) + '.h5') + os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" with h5py.File(h5_file, 'r') as h5: entry = h5['R{0:04d}'.format(scan.number)] # iterate through data fields