From 389e46782ed8fd071ca382766099f8939cbfa4e0 Mon Sep 17 00:00:00 2001 From: Serwan Asaad Date: Sat, 18 May 2024 05:54:59 +0200 Subject: [PATCH 1/6] Refactor: Move DEFAULT_DATA_PROCESSORS --- .../results/data_handler/__init__.py | 19 ++++++++++++++++++- .../results/data_handler/data_processors.py | 19 +------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/qualang_tools/results/data_handler/__init__.py b/qualang_tools/results/data_handler/__init__.py index cb496969..bf5f1d99 100644 --- a/qualang_tools/results/data_handler/__init__.py +++ b/qualang_tools/results/data_handler/__init__.py @@ -1,6 +1,23 @@ +from abc import ABC + from .data_folder_tools import * from . import data_processors -from .data_processors import DEFAULT_DATA_PROCESSORS + from .data_handler import * + +DEFAULT_DATA_PROCESSORS = [ + data_processors.MatplotlibPlotSaver, + data_processors.NumpyArraySaver, +] + + +try: + import xarray # noqa: F401 + + DEFAULT_DATA_PROCESSORS.append(data_processors.XarraySaver) +except ImportError: + pass + + __all__ = [*data_folder_tools.__all__, data_processors, DEFAULT_DATA_PROCESSORS, *data_handler.__all__] diff --git a/qualang_tools/results/data_handler/data_processors.py b/qualang_tools/results/data_handler/data_processors.py index 5807c5a3..bbc0eed9 100644 --- a/qualang_tools/results/data_handler/data_processors.py +++ b/qualang_tools/results/data_handler/data_processors.py @@ -4,10 +4,7 @@ from matplotlib import pyplot as plt import numpy as np -__all__ = ["DEFAULT_DATA_PROCESSORS", "DataProcessor", "MatplotlibPlotSaver", "NumpyArraySaver", "XarraySaver"] - - -DEFAULT_DATA_PROCESSORS = [] +__all__ = ["DataProcessor", "MatplotlibPlotSaver", "NumpyArraySaver", "XarraySaver"] def iterate_nested_dict( @@ -78,9 +75,6 @@ def post_process(self, data_folder: Path): fig.savefig(data_folder / path) -DEFAULT_DATA_PROCESSORS.append(MatplotlibPlotSaver) - - class NumpyArraySaver(DataProcessor): merge_arrays: bool = True merged_array_name: str = "arrays.npz" @@ -118,9 +112,6 @@ def post_process(self, data_folder: Path): np.save(data_folder / path.with_suffix(".npy"), arr) -DEFAULT_DATA_PROCESSORS.append(NumpyArraySaver) - - class XarraySaver(DataProcessor): merge_arrays: bool = False merged_array_name: str = "xarrays" @@ -184,11 +175,3 @@ def post_process(self, data_folder: Path): else: for path, array in self.data_arrays.items(): array.to_netcdf(data_folder / path.with_suffix(self.file_suffix)) - - -try: - import xarray # noqa: F401 - - DEFAULT_DATA_PROCESSORS.append(XarraySaver) -except ImportError: - pass From e6dc4fc7c0e72429460fc7f87ffbe4d2343f77fb Mon Sep 17 00:00:00 2001 From: Serwan Asaad Date: Sat, 18 May 2024 06:11:20 +0200 Subject: [PATCH 2/6] remove unnecessary import --- qualang_tools/results/data_handler/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qualang_tools/results/data_handler/__init__.py b/qualang_tools/results/data_handler/__init__.py index bf5f1d99..65c5992c 100644 --- a/qualang_tools/results/data_handler/__init__.py +++ b/qualang_tools/results/data_handler/__init__.py @@ -1,5 +1,3 @@ -from abc import ABC - from .data_folder_tools import * from . import data_processors From 86f5abe37eec270b98ea01e277c77f681b8419b1 Mon Sep 17 00:00:00 2001 From: Serwan Asaad Date: Sat, 18 May 2024 06:16:57 +0200 Subject: [PATCH 3/6] remove data_processors import --- qualang_tools/results/data_handler/data_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qualang_tools/results/data_handler/data_handler.py b/qualang_tools/results/data_handler/data_handler.py index c518a88b..a475ec3f 100644 --- a/qualang_tools/results/data_handler/data_handler.py +++ b/qualang_tools/results/data_handler/data_handler.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Optional, Sequence, Union import warnings -from .data_processors import DEFAULT_DATA_PROCESSORS, DataProcessor +from .data_processors import DataProcessor from .data_folder_tools import ( DEFAULT_FOLDER_PATTERN, create_data_folder, @@ -106,7 +106,9 @@ class DataHandler: data_handler.save_data(data) """ - default_data_processors = DEFAULT_DATA_PROCESSORS + from . import DEFAULT_DATA_PROCESSORS as _DEFAULT_DATA_PROCESSORS + + default_data_processors = _DEFAULT_DATA_PROCESSORS root_data_folder: Path = None folder_pattern: str = DEFAULT_FOLDER_PATTERN data_filename: str = "data.json" From 830467d9706f8086c30b44efc27d92f3862dc705 Mon Sep 17 00:00:00 2001 From: Yoav Romach Date: Tue, 21 May 2024 09:16:45 +0300 Subject: [PATCH 4/6] attempt on circulator import --- qualang_tools/results/__init__.py | 1 - qualang_tools/results/data_handler/__init__.py | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/qualang_tools/results/__init__.py b/qualang_tools/results/__init__.py index 0b145c32..372d3461 100644 --- a/qualang_tools/results/__init__.py +++ b/qualang_tools/results/__init__.py @@ -1,7 +1,6 @@ from qualang_tools.results.results import fetching_tool from qualang_tools.results.results import progress_counter from qualang_tools.results.results import wait_until_job_is_paused - from qualang_tools.results.data_handler import DataHandler, data_processors __all__ = ["fetching_tool", "progress_counter", "wait_until_job_is_paused", "DataHandler", "data_processors"] diff --git a/qualang_tools/results/data_handler/__init__.py b/qualang_tools/results/data_handler/__init__.py index 65c5992c..dee71005 100644 --- a/qualang_tools/results/data_handler/__init__.py +++ b/qualang_tools/results/data_handler/__init__.py @@ -1,15 +1,10 @@ -from .data_folder_tools import * from . import data_processors -from .data_handler import * - - DEFAULT_DATA_PROCESSORS = [ data_processors.MatplotlibPlotSaver, data_processors.NumpyArraySaver, ] - try: import xarray # noqa: F401 @@ -17,5 +12,7 @@ except ImportError: pass +from .data_folder_tools import * +from .data_handler import * __all__ = [*data_folder_tools.__all__, data_processors, DEFAULT_DATA_PROCESSORS, *data_handler.__all__] From 5f2ee3ceb538ab260595a3c7997ceec03062b0da Mon Sep 17 00:00:00 2001 From: Yoav Romach Date: Tue, 21 May 2024 09:26:04 +0300 Subject: [PATCH 5/6] fix init file --- .../results/data_handler/__init__.py | 25 ++++++++++++++++--- .../results/data_handler/data_folder_tools.py | 3 --- .../results/data_handler/data_handler.py | 2 -- .../results/data_handler/data_processors.py | 2 -- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/qualang_tools/results/data_handler/__init__.py b/qualang_tools/results/data_handler/__init__.py index dee71005..7f10ffb7 100644 --- a/qualang_tools/results/data_handler/__init__.py +++ b/qualang_tools/results/data_handler/__init__.py @@ -12,7 +12,26 @@ except ImportError: pass -from .data_folder_tools import * -from .data_handler import * +from .data_folder_tools import ( # noqa: E402 + DEFAULT_FOLDER_PATTERN, + extract_data_folder_properties, + get_latest_data_folder, + create_data_folder, +) +from .data_handler import save_data, DataHandler # noqa: E402 +from .data_processors import DataProcessor, MatplotlibPlotSaver, NumpyArraySaver, XarraySaver # noqa: E402 -__all__ = [*data_folder_tools.__all__, data_processors, DEFAULT_DATA_PROCESSORS, *data_handler.__all__] + +__all__ = [ + "DEFAULT_FOLDER_PATTERN", + "extract_data_folder_properties", + "get_latest_data_folder", + "create_data_folder", + "DataProcessor", + "MatplotlibPlotSaver", + "NumpyArraySaver", + "XarraySaver", + "DEFAULT_DATA_PROCESSORS", + "save_data", + "DataHandler", +] diff --git a/qualang_tools/results/data_handler/data_folder_tools.py b/qualang_tools/results/data_handler/data_folder_tools.py index 725d2d21..4441306a 100644 --- a/qualang_tools/results/data_handler/data_folder_tools.py +++ b/qualang_tools/results/data_handler/data_folder_tools.py @@ -10,9 +10,6 @@ from datetime import datetime -__all__ = ["DEFAULT_FOLDER_PATTERN", "extract_data_folder_properties", "get_latest_data_folder", "create_data_folder"] - - DEFAULT_FOLDER_PATTERN = "%Y-%m-%d/#{idx}_{name}_%H%M%S" diff --git a/qualang_tools/results/data_handler/data_handler.py b/qualang_tools/results/data_handler/data_handler.py index a475ec3f..661d8138 100644 --- a/qualang_tools/results/data_handler/data_handler.py +++ b/qualang_tools/results/data_handler/data_handler.py @@ -14,8 +14,6 @@ ) -__all__ = ["save_data", "DataHandler"] - NODE_FILENAME = "node.json" diff --git a/qualang_tools/results/data_handler/data_processors.py b/qualang_tools/results/data_handler/data_processors.py index bbc0eed9..b5cbc42e 100644 --- a/qualang_tools/results/data_handler/data_processors.py +++ b/qualang_tools/results/data_handler/data_processors.py @@ -4,8 +4,6 @@ from matplotlib import pyplot as plt import numpy as np -__all__ = ["DataProcessor", "MatplotlibPlotSaver", "NumpyArraySaver", "XarraySaver"] - def iterate_nested_dict( d: Dict[str, Any], parent_keys: Optional[List[str]] = None From e684be0574dcf1370023d0f6146ab68144264e5f Mon Sep 17 00:00:00 2001 From: Yoav Romach Date: Tue, 21 May 2024 09:28:48 +0300 Subject: [PATCH 6/6] fix test --- tests/data_handler/test_numpy_array_saver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data_handler/test_numpy_array_saver.py b/tests/data_handler/test_numpy_array_saver.py index c6f38910..0a888d77 100644 --- a/tests/data_handler/test_numpy_array_saver.py +++ b/tests/data_handler/test_numpy_array_saver.py @@ -1,6 +1,6 @@ import numpy as np -from qualang_tools.results.data_handler.data_processors import DEFAULT_DATA_PROCESSORS, NumpyArraySaver +from qualang_tools.results.data_handler import NumpyArraySaver def test_numpy_array_saver_process_merged():