From 13bf3d6e769e869fb3932d0279e3d98d02dd0390 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 4 Oct 2024 11:09:51 +0200 Subject: [PATCH 1/5] Cleanup --- src/ess/reduce/nexus/json_generator.py | 80 ++++++++++++++++++++++++++ tests/nexus/json_generator_test.py | 38 ++++++++++++ tests/nexus/json_nexus_test.py | 2 + 3 files changed, 120 insertions(+) create mode 100644 src/ess/reduce/nexus/json_generator.py create mode 100644 tests/nexus/json_generator_test.py diff --git a/src/ess/reduce/nexus/json_generator.py b/src/ess/reduce/nexus/json_generator.py new file mode 100644 index 00000000..4b6ef49f --- /dev/null +++ b/src/ess/reduce/nexus/json_generator.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +"""Generators for "JSON" data from a NeXus file, for the purpose of testing.""" + +from collections.abc import Generator +from typing import Any + +import scipp as sc + + +def _as_attr(value: Any, name: str) -> dict: + val = str(value) + return {"string_size": len(val), "type": "string", "name": name, "values": val} + + +def _variable_to_json(var: sc.Variable, name: str): + attrs = [] + if var.dtype == sc.DType.datetime64: + offset = var.min() + attrs.append(_as_attr(offset.value, "offset")) + var = var - offset + if var.unit is not None: + attrs.append(_as_attr(var.unit, "units")) + return { + "module": "dataset", + "config": { + "name": name, + "values": var.values, + "size": list(var.shape), + "type": str(var.dtype), + }, + "attributes": attrs, + } + + +_event_index_0 = sc.array(dims=('dummy',), values=[0], unit=None) + + +def _event_data_pulse_to_json(pulse: sc.DataArray) -> dict: + content = pulse.value.coords + event_time_zero = sc.concat([pulse.coords['event_time_zero']], 'dummy') + event_time_offset = content['event_time_offset'] + event_id = content.get('event_id') + # I think we always have a pixel_id in the flatbuffer, so monitors just get ones? + if event_id is None: + event_id = sc.ones(sizes=event_time_offset.sizes, dtype='int32', unit=None) + children = [ + _variable_to_json(event_time_zero, name='event_time_zero'), + _variable_to_json(event_time_offset, name='event_time_offset'), + _variable_to_json(event_id, name='event_id'), + _variable_to_json(_event_index_0, name='event_index'), + ] + group = { + "type": "group", + "name": "events_0", + "children": children, + "attributes": [ + { + "string_size": 12, + "type": "string", + "name": "NX_class", + "values": "NXevent_data", + } + ], + } + return group + + +def event_data_generator(data: sc.DataArray) -> Generator[dict, None, None]: + """ + Generate JSON data for event data from a NeXus file. + + Parameters + ---------- + data: + A data array with event data, equivalent to what ScippNexus would load from an + NXevent_data group in a NeXus file. + """ + for pulse in data: + yield _event_data_pulse_to_json(pulse) diff --git a/tests/nexus/json_generator_test.py b/tests/nexus/json_generator_test.py new file mode 100644 index 00000000..7bc3032e --- /dev/null +++ b/tests/nexus/json_generator_test.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import scippnexus as snx +from scipp.testing import assert_identical + +from ess.reduce import data +from ess.reduce.nexus.json_generator import event_data_generator +from ess.reduce.nexus.json_nexus import json_nexus_group + + +def test_event_data_generator_monitor_events_round_trip() -> None: + filename = data.loki_tutorial_sample_run_60250() + monitor = snx.load(filename, root='entry/instrument/monitor_1/monitor_1_events') + generator = event_data_generator(monitor) + for i in range(len(monitor)): + group = json_nexus_group(next(generator)) + assert_identical(group[()], monitor[i : i + 1]) + + +def test_event_data_generator_detector_events_round_trip() -> None: + filename = data.loki_tutorial_sample_run_60250() + detector = snx.load( + filename, root='entry/instrument/larmor_detector/larmor_detector_events' + ) + generator = event_data_generator(detector) + for i in range(100): + group = json_nexus_group(next(generator)) + assert_identical(group[()], detector[i : i + 1]) + + +def test_event_data_generator_without_event_id_yields_ones() -> None: + filename = data.loki_tutorial_sample_run_60250() + base = snx.load(filename, root='entry/instrument/monitor_1/monitor_1_events') + monitor = base.bins.drop_coords('event_id') + generator = event_data_generator(monitor) + for i in range(100): + group = json_nexus_group(next(generator)) + assert_identical(group[()], base[i : i + 1]) diff --git a/tests/nexus/json_nexus_test.py b/tests/nexus/json_nexus_test.py index 71514083..d8d86c9d 100644 --- a/tests/nexus/json_nexus_test.py +++ b/tests/nexus/json_nexus_test.py @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import json from copy import deepcopy from functools import lru_cache From 90a00ac6ce9f95a1ec3ef6d3006139e5d87bbcbf Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 4 Oct 2024 11:21:39 +0200 Subject: [PATCH 2/5] More cleanup --- src/ess/reduce/nexus/json_generator.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/ess/reduce/nexus/json_generator.py b/src/ess/reduce/nexus/json_generator.py index 4b6ef49f..ed66f4fb 100644 --- a/src/ess/reduce/nexus/json_generator.py +++ b/src/ess/reduce/nexus/json_generator.py @@ -8,7 +8,7 @@ import scipp as sc -def _as_attr(value: Any, name: str) -> dict: +def _as_str_attr(value: Any, name: str) -> dict: val = str(value) return {"string_size": len(val), "type": "string", "name": name, "values": val} @@ -17,10 +17,10 @@ def _variable_to_json(var: sc.Variable, name: str): attrs = [] if var.dtype == sc.DType.datetime64: offset = var.min() - attrs.append(_as_attr(offset.value, "offset")) + attrs.append(_as_str_attr(offset.value, "offset")) var = var - offset if var.unit is not None: - attrs.append(_as_attr(var.unit, "units")) + attrs.append(_as_str_attr(var.unit, "units")) return { "module": "dataset", "config": { @@ -54,14 +54,7 @@ def _event_data_pulse_to_json(pulse: sc.DataArray) -> dict: "type": "group", "name": "events_0", "children": children, - "attributes": [ - { - "string_size": 12, - "type": "string", - "name": "NX_class", - "values": "NXevent_data", - } - ], + "attributes": [_as_str_attr("NXevent_data", name="NX_class")], } return group From 3ce9cb029b1ddf790115f0d949454f1093f280e7 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 7 Oct 2024 14:39:11 +0200 Subject: [PATCH 3/5] Finish docstring --- src/ess/reduce/nexus/json_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ess/reduce/nexus/json_generator.py b/src/ess/reduce/nexus/json_generator.py index ed66f4fb..922e5aa7 100644 --- a/src/ess/reduce/nexus/json_generator.py +++ b/src/ess/reduce/nexus/json_generator.py @@ -68,6 +68,12 @@ def event_data_generator(data: sc.DataArray) -> Generator[dict, None, None]: data: A data array with event data, equivalent to what ScippNexus would load from an NXevent_data group in a NeXus file. + + Yields + ------ + : + A dict of data for a single event data pulse that can be wrapped in a + :py:class:`ess.reduce.nexus.json_nexus.JSONGroup`. """ for pulse in data: yield _event_data_pulse_to_json(pulse) From 307d3e70cf2df2a87a58e1b3576a8f84602f791c Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 7 Oct 2024 14:39:26 +0200 Subject: [PATCH 4/5] Test iterator stops --- tests/nexus/json_generator_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/nexus/json_generator_test.py b/tests/nexus/json_generator_test.py index 7bc3032e..d6b53ce1 100644 --- a/tests/nexus/json_generator_test.py +++ b/tests/nexus/json_generator_test.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import pytest import scippnexus as snx from scipp.testing import assert_identical @@ -15,6 +16,8 @@ def test_event_data_generator_monitor_events_round_trip() -> None: for i in range(len(monitor)): group = json_nexus_group(next(generator)) assert_identical(group[()], monitor[i : i + 1]) + with pytest.raises(StopIteration): + next(generator) def test_event_data_generator_detector_events_round_trip() -> None: From 1cb902c4a3c0f6afb339eb9ea6fdd9b3fa551e60 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 7 Oct 2024 14:46:28 +0200 Subject: [PATCH 5/5] Whitespace error --- src/ess/reduce/nexus/json_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ess/reduce/nexus/json_generator.py b/src/ess/reduce/nexus/json_generator.py index 922e5aa7..2cd1f145 100644 --- a/src/ess/reduce/nexus/json_generator.py +++ b/src/ess/reduce/nexus/json_generator.py @@ -67,7 +67,7 @@ def event_data_generator(data: sc.DataArray) -> Generator[dict, None, None]: ---------- data: A data array with event data, equivalent to what ScippNexus would load from an - NXevent_data group in a NeXus file. + NXevent_data group in a NeXus file. Yields ------