From ce22fc5505d60a68def5df14310555d9706f72d6 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 2 Jan 2023 13:23:56 -0800 Subject: [PATCH 1/5] RF: Move generic json read/write functions into data.utils. Beginning to work towards s3bids deprecation. --- AFQ/api/participant.py | 2 +- AFQ/data/s3bids.py | 38 -------------------------------------- AFQ/data/utils.py | 38 ++++++++++++++++++++++++++++++++++++++ AFQ/definitions/mapping.py | 6 +++--- AFQ/tasks/decorators.py | 2 +- AFQ/tasks/mapping.py | 2 +- AFQ/tasks/segmentation.py | 2 +- AFQ/tasks/viz.py | 2 +- AFQ/utils/streamlines.py | 2 +- 9 files changed, 47 insertions(+), 47 deletions(-) diff --git a/AFQ/api/participant.py b/AFQ/api/participant.py index b5a6e4458..09c3d05eb 100644 --- a/AFQ/api/participant.py +++ b/AFQ/api/participant.py @@ -15,7 +15,7 @@ from AFQ.tasks.segmentation import get_segmentation_plan from AFQ.tasks.viz import get_viz_plan from AFQ.utils.path import drop_extension -from AFQ.data.s3bids import read_json +from AFQ.data.utils import read_json __all__ = ["ParticipantAFQ"] diff --git a/AFQ/data/s3bids.py b/AFQ/data/s3bids.py index 65a6d2d20..0fcf919a7 100644 --- a/AFQ/data/s3bids.py +++ b/AFQ/data/s3bids.py @@ -1222,44 +1222,6 @@ def s3fs_nifti_read(fname, fs=None, anon=False): return img -def write_json(fname, data): - """ - Write data to JSON file. - - Parameters - ---------- - fname : str - Full path to the file to write. - - data : dict - A dict containing the data to write. - - Returns - ------- - None - """ - with open(fname, 'w') as ff: - json.dump(data, ff, default=lambda obj: "Not Serializable") - - -def read_json(fname): - """ - Read data from a JSON file. - - Parameters - ---------- - fname : str - Full path to the data-containing file - - Returns - ------- - dict - """ - with open(fname, 'r') as ff: - out = json.load(ff) - return out - - def s3fs_json_read(fname, fs=None, anon=False): """ Reads json directly from S3 diff --git a/AFQ/data/utils.py b/AFQ/data/utils.py index 43913c7a9..9ea1397f5 100644 --- a/AFQ/data/utils.py +++ b/AFQ/data/utils.py @@ -31,3 +31,41 @@ 'RightILF': 'ILF_R', 'LeftILF': 'ILF_L', 'Right SLF': 'SLF_R', 'Left SLF': 'SLF_L', 'RightSLF': 'SLF_R', 'LeftSLF': 'SLF_L'} + + +def write_json(fname, data): + """ + Write data to JSON file. + + Parameters + ---------- + fname : str + Full path to the file to write. + + data : dict + A dict containing the data to write. + + Returns + ------- + None + """ + with open(fname, 'w') as ff: + json.dump(data, ff, default=lambda obj: "Not Serializable") + + +def read_json(fname): + """ + Read data from a JSON file. + + Parameters + ---------- + fname : str + Full path to the data-containing file + + Returns + ------- + dict + """ + with open(fname, 'r') as ff: + out = json.load(ff) + return out diff --git a/AFQ/definitions/mapping.py b/AFQ/definitions/mapping.py index 2d2a15a1c..3205c22dc 100644 --- a/AFQ/definitions/mapping.py +++ b/AFQ/definitions/mapping.py @@ -7,7 +7,7 @@ from AFQ.definitions.utils import Definition, find_file from dipy.align import syn_registration, affine_registration import AFQ.registration as reg -import AFQ.data.s3bids as afs +from AFQ.data.utils import write_json from AFQ.tasks.utils import get_fname from dipy.align.imaffine import AffineMap @@ -320,7 +320,7 @@ def prealign(self, base_fname, reg_subject, reg_template, save=True): np.save(prealign_file, aff) meta_fname = get_fname( base_fname, f'{prealign_file_desc}.json') - afs.write_json(meta_fname, meta) + write_json(meta_fname, meta) return prealign_file if save else np.load(prealign_file) def get_for_subses(self, base_fname, dwi, bids_info, reg_subject, @@ -350,7 +350,7 @@ def get_for_subses(self, base_fname, dwi, bids_info, reg_subject, meta["dependent"] = "dwi" else: meta["dependent"] = "trk" - afs.write_json(meta_fname, meta) + write_json(meta_fname, meta) reg_prealign_inv = np.linalg.inv(reg_prealign) if self.use_prealign\ else None mapping = reg.read_mapping( diff --git a/AFQ/tasks/decorators.py b/AFQ/tasks/decorators.py index 3a6c23652..65f233bcb 100644 --- a/AFQ/tasks/decorators.py +++ b/AFQ/tasks/decorators.py @@ -7,7 +7,7 @@ import nibabel as nib from dipy.io.streamline import save_tractogram from dipy.io.stateful_tractogram import StatefulTractogram -from AFQ.data.s3bids import write_json +from AFQ.data.utils import write_json import numpy as np diff --git a/AFQ/tasks/mapping.py b/AFQ/tasks/mapping.py index 99f79a748..7e8916e5d 100644 --- a/AFQ/tasks/mapping.py +++ b/AFQ/tasks/mapping.py @@ -8,7 +8,7 @@ from AFQ.tasks.decorators import as_file from AFQ.tasks.utils import get_fname, with_name, str_to_desc import AFQ.data.fetch as afd -from AFQ.data.s3bids import write_json +from AFQ.data.utils import write_json from AFQ.utils.path import drop_extension import AFQ.utils.volume as auv from AFQ.definitions.mapping import SynMap diff --git a/AFQ/tasks/segmentation.py b/AFQ/tasks/segmentation.py index 3fc54e5d7..7e0c845e0 100644 --- a/AFQ/tasks/segmentation.py +++ b/AFQ/tasks/segmentation.py @@ -14,7 +14,7 @@ from AFQ.utils.path import drop_extension import AFQ.utils.streamlines as aus from AFQ.tasks.utils import get_default_args -from AFQ.data.s3bids import write_json +from AFQ.data.utils import write_json import AFQ.api.bundle_dict as abd import AFQ.utils.streamlines as aus import AFQ.utils.volume as auv diff --git a/AFQ/tasks/viz.py b/AFQ/tasks/viz.py index be0f04726..e2a56add7 100644 --- a/AFQ/tasks/viz.py +++ b/AFQ/tasks/viz.py @@ -12,7 +12,7 @@ from AFQ.tasks.utils import get_fname, with_name, str_to_desc import AFQ.utils.volume as auv -from AFQ.data.s3bids import write_json +from AFQ.data.utils import write_json from AFQ.viz.utils import Viz from plotly.subplots import make_subplots diff --git a/AFQ/utils/streamlines.py b/AFQ/utils/streamlines.py index f35129f6d..e54ea30e3 100644 --- a/AFQ/utils/streamlines.py +++ b/AFQ/utils/streamlines.py @@ -2,7 +2,7 @@ from dipy.io.streamline import load_tractogram import numpy as np from dipy.io.stateful_tractogram import StatefulTractogram, Space -from AFQ.data.s3bids import read_json +from AFQ.data.utils import read_json import os.path as op from AFQ.utils.path import drop_extension From d85b85b55b92d129d5003242f6bb7ff4b7918b43 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 2 Jan 2023 13:40:54 -0800 Subject: [PATCH 2/5] Import stdlib dependency. --- AFQ/data/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AFQ/data/utils.py b/AFQ/data/utils.py index 9ea1397f5..116d8bd92 100644 --- a/AFQ/data/utils.py +++ b/AFQ/data/utils.py @@ -1,3 +1,5 @@ +import json + BUNDLE_RECO_2_AFQ = \ { "AF_L": "ARC_L", "AF_R": "ARC_R", From eca0074f25728a83f7b5efafa32da5d2b436e7c1 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 2 Jan 2023 13:56:26 -0800 Subject: [PATCH 3/5] Add the deprecation warning. --- AFQ/data/s3bids.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/AFQ/data/s3bids.py b/AFQ/data/s3bids.py index 0fcf919a7..40d9692c3 100644 --- a/AFQ/data/s3bids.py +++ b/AFQ/data/s3bids.py @@ -11,7 +11,6 @@ from dask.diagnostics import ProgressBar from pathlib import Path -import os import os.path as op import logging @@ -24,6 +23,10 @@ from bids import BIDSLayout from AFQ.data.fetch import to_bids_description +import warnings +msg = "The `s3bids` module will be deprecated " +msg += " in a future version of pyAFQ." +warnings.warn(msg, DeprecationWarning, stacklevel=2) # +----------------------------------------------------+ # | Begin S3BIDSStudy classes and supporting functions | From a2c5c9ba25611608bdc25b78a24d3d1137c0b731 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 2 Jan 2023 13:57:28 -0800 Subject: [PATCH 4/5] Reword the warning to be more precise/communicative. --- AFQ/data/s3bids.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AFQ/data/s3bids.py b/AFQ/data/s3bids.py index 40d9692c3..cb1a94442 100644 --- a/AFQ/data/s3bids.py +++ b/AFQ/data/s3bids.py @@ -24,10 +24,11 @@ from AFQ.data.fetch import to_bids_description import warnings -msg = "The `s3bids` module will be deprecated " +msg = "The `s3bids` module will be removed " msg += " in a future version of pyAFQ." warnings.warn(msg, DeprecationWarning, stacklevel=2) + # +----------------------------------------------------+ # | Begin S3BIDSStudy classes and supporting functions | # +----------------------------------------------------+ From cc0e3c0ea21ec031cc168b1c65ea9fab53e4f8de Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 2 Jan 2023 20:33:54 -0800 Subject: [PATCH 5/5] More systematic deprecation with decorators. This means that the warning will only be raised upon calling a deprecated function, or initialization of a deprecated class. --- AFQ/data/s3bids.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/AFQ/data/s3bids.py b/AFQ/data/s3bids.py index cb1a94442..e03428366 100644 --- a/AFQ/data/s3bids.py +++ b/AFQ/data/s3bids.py @@ -24,14 +24,40 @@ from AFQ.data.fetch import to_bids_description import warnings -msg = "The `s3bids` module will be removed " -msg += " in a future version of pyAFQ." -warnings.warn(msg, DeprecationWarning, stacklevel=2) +import functools + + +# +---------------------+ +# | Deprecation helpers | +# +---------------------+ +def deprecate_function(func): + @functools.wraps(func) + def new_func(*args, **kwargs): + msg = f"The function {func.__name__} is part of the " + msg += "AFQ.data.s3bids module, which will be removed " + msg += "in version 2.0 of pyAFQ." + warnings.warn(msg, DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) + + return new_func + + +def deprecate_class(klass): + class new_klass(klass): + def __init__(self, *args, **kwargs): + msg = f"The class {klass.__name__} is part of the " + msg += "AFQ.data.s3bids module, which will be removed " + msg += "in version 2.0 of pyAFQ." + warnings.warn(msg, DeprecationWarning, stacklevel=2) + + return klass.__init__(*args, **kwargs) + return new_klass # +----------------------------------------------------+ # | Begin S3BIDSStudy classes and supporting functions | # +----------------------------------------------------+ +@deprecate_function def get_s3_client(anon=True): """Return a boto3 s3 client @@ -197,6 +223,7 @@ def _download_from_s3(fname, bucket, key, overwrite=False, anon=True): fs.get("/".join([bucket, key]), fname) +@deprecate_class class S3BIDSSubject: """A single study subject hosted on AWS S3""" @@ -446,6 +473,7 @@ def split_key(key): progress.close() +@deprecate_class class HBNSubject(S3BIDSSubject): """A subject in the HBN study @@ -529,6 +557,7 @@ def get_deriv_type(s3_key): self._s3_keys = s3_keys +@deprecate_class class S3BIDSStudy: """A BIDS-compliant study hosted on AWS S3""" @@ -957,6 +986,7 @@ def download(self, directory, compute(*results, scheduler='threads') +@deprecate_class class HBNSite(S3BIDSStudy): """An HBN study site @@ -1188,6 +1218,7 @@ def s3fs_nifti_write(img, fname, fs=None): ff.write(data) +@deprecate_function def s3fs_nifti_read(fname, fs=None, anon=False): """ Lazily reads a nifti image from S3. @@ -1226,6 +1257,7 @@ def s3fs_nifti_read(fname, fs=None, anon=False): return img +@deprecate_function def s3fs_json_read(fname, fs=None, anon=False): """ Reads json directly from S3 @@ -1249,6 +1281,7 @@ def s3fs_json_read(fname, fs=None, anon=False): return data +@deprecate_function def s3fs_json_write(data, fname, fs=None): """ Writes json from a dict directly into S3