From 65888925d216160d72edaac8fc48df53534daee0 Mon Sep 17 00:00:00 2001 From: RubelMozumder <32923026+RubelMozumder@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:57:18 +0200 Subject: [PATCH] app def missing. (#108) * Implement write nexus section based on the populated nomad archive * app def missing. * mapping nomad_measurement. * All concept are connected, creates nexus file and subsection. * adding links in hdf5 file. * Remove the nxs file. * back to the previous design. * Include pynxtools plugins in nomad.yaml and extend dependencies including pynxtools ans pnxtools-xrd. * PR review correction. * Remove the entry_type overwtitten. * Remove comments. * Replace __str__ function. * RUFF * Update pyproject.toml Co-authored-by: Sarthak Kapoor <57119427+ka-sarthak@users.noreply.github.com> * Update src/nomad_measurements/xrd/schema.py Co-authored-by: Sarthak Kapoor <57119427+ka-sarthak@users.noreply.github.com> * Update src/nomad_measurements/xrd/nx.py * Replace Try-block. --------- Co-authored-by: Sarthak Kapoor Co-authored-by: Sarthak Kapoor <57119427+ka-sarthak@users.noreply.github.com> --- pyproject.toml | 2 +- src/nomad_measurements/xrd/nx.py | 182 +++++++++++++++++++++++++++ src/nomad_measurements/xrd/schema.py | 47 ++----- tests/test_parser.py | 5 +- 4 files changed, 193 insertions(+), 43 deletions(-) create mode 100644 src/nomad_measurements/xrd/nx.py diff --git a/pyproject.toml b/pyproject.toml index 1a250165..7a825689 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "nomad-lab>=1.3.4dev", "xmltodict==0.13.0", "fairmat-readers-xrd~=0.0.3", - "pynxtools", + "pynxtools>=0.6.1", ] [project.urls] "Homepage" = "https://github.com/FAIRmat-NFDI/nomad-measurements" diff --git a/src/nomad_measurements/xrd/nx.py b/src/nomad_measurements/xrd/nx.py new file mode 100644 index 00000000..e1b41fcf --- /dev/null +++ b/src/nomad_measurements/xrd/nx.py @@ -0,0 +1,182 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import TYPE_CHECKING + +from pynxtools import dataconverter +from pynxtools.nomad.dataconverter import populate_nexus_subsection + +if TYPE_CHECKING: + from nomad.datamodel.datamodel import EntryArchive + from structlog.stdlib import ( + BoundLogger, + ) + + +def walk_through_object(parent_obj, attr_chain, default=None): + """ + Walk though the object until reach the leaf. + + Args: + parent_obj: This is a python obj. + attr_chain: Dot separated obj chain. + default: A value to be returned by default, if not data is found. + """ + expected_parts = 2 + if isinstance(attr_chain, str): + parts = attr_chain.split('.', 1) + + if len(parts) == expected_parts: + child_nm, rest_part = parts + if '[' in child_nm: + child_nm, index = child_nm.split('[') + index = int(index[:-1]) + child_obj = getattr(parent_obj, child_nm)[index] + else: + child_obj = getattr(parent_obj, child_nm) + return walk_through_object(child_obj, rest_part, default=default) + else: + return getattr(parent_obj, attr_chain, default) + + +def connect_concepts(template, archive: 'EntryArchive', scan_type: str): # noqa: PLR0912 + """ + Connect the concepts between `ELNXrayDiffraction` and `NXxrd_pan` schema. + + Args: + template (Template): The pynxtools template, a inherited class from python dict. + archive (EntryArchive): Nomad archive contains secttions, subsections and + quantities. + scan_type (str): Name of the scan type such as line and RSM. + """ + + # General concepts + # ruff: noqa: E501 + concept_map = { + '/ENTRY[entry]/method': 'archive.data.method', + '/ENTRY[entry]/measurement_type': 'archive.data.diffraction_method_name', + '/ENTRY[entry]/experiment_result/intensity': 'archive.data.results[0].intensity.magnitude', + '/ENTRY[entry]/experiment_result/two_theta': 'archive.data.results[0].two_theta.magnitude', + '/ENTRY[entry]/experiment_result/two_theta/@units': 'archive.data.results[0].two_theta.units', + '/ENTRY[entry]/experiment_result/omega': 'archive.data.results[0].omega.magnitude', + '/ENTRY[entry]/experiment_result/omega/@units': 'archive.data.results[0].omega.units', + '/ENTRY[entry]/experiment_result/chi': 'archive.data.results[0].chi.magnitude', + '/ENTRY[entry]/experiment_result/chi/@units': 'archive.data.results[0].chi.units', + '/ENTRY[entry]/experiment_result/phi': 'archive.data.results[0].phi.magnitude', + '/ENTRY[entry]/experiment_result/phi/@units': 'archive.data.results[0].phi.units', + '/ENTRY[entry]/INSTRUMENT[instrument]/DETECTOR[detector]/scan_axis': 'archive.data.results[0].scan_axis', + '/ENTRY[entry]/experiment_config/count_time': 'archive.data.results[0].count_time.magnitude', + 'line': '', # For future implementation + 'rsm': { + '/ENTRY[entry]/experiment_result/q_parallel': 'archive.data.results[0].q_parallel', + '/ENTRY[entry]/experiment_result/q_parallel/@units': 'archive.data.results[0].q_parallel.units', + '/ENTRY[entry]/experiment_result/q_perpendicular': 'archive.data.results[0].q_perpendicular.magnitude', + '/ENTRY[entry]/experiment_result/q_perpendicular/@units': 'archive.data.results[0].q_perpendicular.units', + '/ENTRY[entry]/experiment_result/q_norm': 'archive.data.results[0].q_norm.magnitude', + '/ENTRY[entry]/experiment_result/q_norm/@units': 'archive.data.results[0].q_norm.units', + }, + # Source + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_material': 'archive.data.xrd_settings.source.xray_tube_material', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_current': 'archive.data.xrd_settings.source.xray_tube_current.magnitude', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_current/@units': 'archive.data.xrd_settings.source.xray_tube_current.units', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_voltage': 'archive.data.xrd_settings.source.xray_tube_voltage.magnitude', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/xray_tube_voltage/@units': 'archive.data.xrd_settings.source.xray_tube_voltage.units', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_one': 'archive.data.xrd_settings.source.kalpha_one.magnitude', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_one/@units': 'archive.data.xrd_settings.source.kalpha_one.units', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_two': 'archive.data.xrd_settings.source.kalpha_two.magnitude', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_two/@units': 'archive.data.xrd_settings.source.kalpha_two.units', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/ratio_k_alphatwo_k_alphaone': 'archive.data.xrd_settings.source.ratio_kalphatwo_kalphaone', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/kbeta': 'archive.data.xrd_settings.source.kbeta.magnitude', + '/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/kbeta/@units': 'archive.data.xrd_settings.source.kbeta.units', + } + + for key, archive_concept in concept_map.items(): + if isinstance(archive_concept, dict): + if key == scan_type: + for sub_key, sub_archive_concept in archive_concept.items(): + _, arch_attr = sub_archive_concept.split('.', 1) + value = None + try: + value = walk_through_object(archive, arch_attr) + except (AttributeError, IndexError, KeyError, ValueError): + pass + finally: + if value is not None: + template[sub_key] = ( + str(value) if sub_key.endswith('units') else value + ) + else: + continue + elif archive_concept: + _, arch_attr = archive_concept.split('.', 1) + value = None + try: + value = walk_through_object(archive, arch_attr) + # Use multiple excepts to avoid catching all exceptions + except (AttributeError, IndexError, KeyError, ValueError): + pass + finally: + if value is not None: + template[key] = str(value) if key.endswith('units') else value + + template['/ENTRY[entry]/definition'] = 'NXxrd_pan' + + # Links to the data and concepts + template['/ENTRY[entry]/@default'] = 'experiment_result' + template['/ENTRY[entry]/experiment_result/@signal'] = 'intensity' + template['/ENTRY[entry]/experiment_result/@axes'] = 'two_theta' + template['/ENTRY[entry]/q_data/q'] = { + 'link': '/ENTRY[entry]/experiment_result/q_norm' + } + template['/ENTRY[entry]/q_data/intensity'] = { + 'link': '/ENTRY[entry]/experiment_result/intensity' + } + template['/ENTRY[entry]/q_data/q_parallel'] = { + 'link': '/ENTRY[entry]/experiment_result/q_parallel' + } + template['/ENTRY[entry]/q_data/q_perpendicular'] = { + 'link': '/ENTRY[entry]/experiment_result/q_perpendicular' + } + + +def write_nx_section_and_create_file( + archive: 'EntryArchive', logger: 'BoundLogger', scan_type: str = 'line' +): + """ + Uses the archive to generate the NeXus section and .nxs file. + + Args: + archive (EntryArchive): The archive containing the section. + logger (BoundLogger): A structlog logger. + generate_nexus_file (boolean): If True, the function will generate a .nxs file. + nxs_as_entry (boolean): If True, the function will generate a .nxs file + as a nomad entry. + """ + nxdl_root, _ = dataconverter.helpers.get_nxdl_root_and_path('NXxrd_pan') + template = dataconverter.template.Template() + dataconverter.helpers.generate_template_from_nxdl(nxdl_root, template) + connect_concepts(template, archive, scan_type=scan_type) + archive_name = archive.metadata.mainfile.split('.')[0] + nexus_output = f'{archive_name}.nxs' + + populate_nexus_subsection( + template=template, + app_def='NXxrd_pan', + archive=archive, + logger=logger, + output_file_path=nexus_output, + ) diff --git a/src/nomad_measurements/xrd/schema.py b/src/nomad_measurements/xrd/schema.py index 8804139e..a7addd80 100644 --- a/src/nomad_measurements/xrd/schema.py +++ b/src/nomad_measurements/xrd/schema.py @@ -68,20 +68,17 @@ NOMADMeasurementsCategory, ) from nomad_measurements.utils import get_bounding_range_2d, merge_sections +from nomad_measurements.xrd.nx import write_nx_section_and_create_file if TYPE_CHECKING: import pint from nomad.datamodel.datamodel import ( EntryArchive, ) - from pynxtools.dataconverter.template import Template from structlog.stdlib import ( BoundLogger, ) - import pint -from pynxtools.nomad.dataconverter import populate_nexus_subsection -from pynxtools import dataconverter from nomad.config import config @@ -862,6 +859,7 @@ class ELNXRayDiffraction(XRayDiffraction, EntryData, PlotSection): generate_nexus_file = Quantity( type=bool, description='Whether or not to generate a NeXus output file (if possible).', + default=True, a_eln=ELNAnnotation( component=ELNComponentEnum.BoolEditQuantity, label='Generate NeXus file', @@ -957,41 +955,6 @@ def write_xrd_data( ) merge_sections(self, xrd, logger) - def write_nx_section_and_create_file( - self, archive: 'EntryArchive', logger: 'BoundLogger' - ): - """ - Uses the archive to generate the NeXus section and .nxs file. - - Args: - archive (EntryArchive): The archive containing the section. - logger (BoundLogger): A structlog logger. - """ - nxdl_root, _ = dataconverter.helpers.get_nxdl_root_and_path('NXxrd_pan') - template = dataconverter.template.Template() - dataconverter.helpers.generate_template_from_nxdl(nxdl_root, template) - - template['/ENTRY[entry]/2theta_plot/intensity'] = archive.data.results[ - 0 - ].intensity.magnitude - template['/ENTRY[entry]/2theta_plot/two_theta'] = archive.data.results[ - 0 - ].two_theta.magnitude - template['/ENTRY[entry]/2theta_plot/two_theta/@units'] = str( - archive.data.results[0].two_theta.units - ) - archive_name = archive.metadata.mainfile.split('.')[0] - nexus_output = f'{archive_name}_output.nxs' - - populate_nexus_subsection( - template=template, - app_def='NXxrd_pan', - archive=archive, - logger=logger, - output_file_path=nexus_output, - on_temp_file=self.generate_nexus_file, - ) - def normalize(self, archive: 'EntryArchive', logger: 'BoundLogger'): """ The normalize function of the `ELNXRayDiffraction` section. @@ -1014,7 +977,11 @@ def normalize(self, archive: 'EntryArchive', logger: 'BoundLogger'): super().normalize(archive, logger) if not self.results: return - self.write_nx_section_and_create_file(archive, logger) + + scan_type = xrd_dict.get('metadata', {}).get('scan_type', None) + if self.generate_nexus_file and self.data_file is not None: + write_nx_section_and_create_file(archive, logger, scan_type=scan_type) + self.figures = self.results[0].generate_plots(archive, logger) diff --git a/tests/test_parser.py b/tests/test_parser.py index 34e93a13..84ebf657 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -47,8 +47,9 @@ def parsed_archive(request): yield measurement_archive - if os.path.exists(measurement): - os.remove(measurement) + for file_path in [measurement, measurement.replace('archive.json', 'nxs')]: + if os.path.exists(file_path): + os.remove(file_path) def test_normalize_all(parsed_archive):