From fc232fa2c7558ccb3b2359172596aa242bf6801b Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 09:54:41 -0500 Subject: [PATCH] improve the editing tutorial and have it cross-reference with the add_remove_containers tutorial --- docs/gallery/advanced_io/plot_editing.py | 83 ++++++------------- docs/gallery/general/add_remove_containers.py | 28 ++----- 2 files changed, 32 insertions(+), 79 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index b7dff1bd0..6fbfee283 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -1,21 +1,23 @@ """ +.. _editing: + Editing NWB files ================= -This tutorial demonstrates how to edit NWB files manually (i.e., without using PyNWB). -How and whether it is possible to edit an NWB file depends on the storage backend -and the type of edit. Here, we go through the common types of manual edits for HDF5 files -using :py:mod:`h5py`. We call this "doing surgery" on the NWB file. +This tutorial demonstrates how to edit NWB files in-place. For information on how to add +or remove containers from an NWB file, see :ref:`modifying_data`. How and whether it is +possible to edit an NWB file depends on the storage backend +and the type of edit. .. warning:: - Manually editing an existing NWB file can make the file invalid if you are not careful. - We highly recommend making a copy before editing and running a validation check on the - file after editing it. See :ref:`validating`. + Manually editing an existing NWB file can make the file invalid if you are not + careful. We highly recommend making a copy before editing and running a validation + check on the file after editing it. See :ref:`validating`. -Editing datasets in place --------------------------- +Editing datasets +---------------- When reading an HDF5 NWB file, PyNWB exposes :py:class:`h5py.Dataset` objects, which can be edited in place. For this to work, you must open the file in read/write mode (``"r+"`` or ``"a"``). @@ -48,9 +50,6 @@ io.write(nwbfile) ############################################## -# Editing values of datasets -# ~~~~~~~~~~~~~~~~~~~~~~~~~~ -# # Now, let's edit the values of the dataset with NWBHDF5IO("test_edit.nwb", "r+") as io: @@ -66,8 +65,9 @@ nwbfile.acquisition["synthetic_timeseries"].data.attrs["unit"] = "volts" ############################################## -# If you want to edit the attributes of a group, you will need to open the file and edit -# it using :py:mod:`h5py`: +# Editing groups +# -------------- +# To edit the attributes of a group, open the file and edit it using :py:mod:`h5py`: import h5py @@ -76,12 +76,12 @@ ############################################## # .. warning:: -# You can edit values that will bring the file out of compliance with -# the NWB specification. +# You can edit values that will bring the file out of compliance with the NWB +# specification. # # Renaming groups and datasets # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Rename groups and datasets in-place using the ``move`` method. For example, to rename +# Rename groups and datasets in-place using the :py:meth:`~h5py.Group.move` method. For example, to rename # the ``"synthetic_timeseries"`` group: with h5py.File("test_edit.nwb", "r+") as f: @@ -93,7 +93,10 @@ # ``"analysis"`` group: with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"].move("synthetic_timeseries_renamed", "/analysis/synthetic_timeseries_renamed") + f["acquisition"].move( + "synthetic_timeseries_renamed", + "/analysis/synthetic_timeseries_renamed", + ) ############################################## # Changing the shape of dataset @@ -141,46 +144,14 @@ with NWBHDF5IO("test_edit2.nwb", "r+") as io: nwbfile = io.read() - nwbfile.acquisition["synthetic_timeseries"].resize((200, 100)) + nwbfile.acquisition["synthetic_timeseries"].data.resize((200, 100)) ############################################## # This will change the shape of the dataset in-place. If you try to change the shape of # a dataset with a fixed shape, you will get an error. # -# Replacing a dataset using h5py -# ------------------------------ -# There are several types of dataset edits that cannot be done in-place: -# -# * Changing the shape of a dataset with a fixed shape -# * Changing the datatype of a dataset -# * Changing the compression of a dataset -# * Changing the chunking of a dataset -# * Changing the max-shape of a dataset -# * Changing the fill-value of a dataset -# -# For any of these, you will need to create a new dataset with the new shape, copying -# the data from the old dataset to the new dataset, and deleting the old dataset. - -with h5py.File("test_edit2.nwb", "r+") as f: - data_values = f["acquisition"]["synthetic_timeseries"]["data"][:] - del f["acquisition"]["synthetic_timeseries"]["data"] - f["acquisition"]["synthetic_timeseries"].create_dataset( - name="data", - data=data_values, - maxshape=(None, 100), - chunks=(100, 100), - compression="gzip", - compression_opts=3, - fillvalue=0.0, - dtype=np.float64, - ) - -############################################## -# .. note:: -# Because of the way HDF5 works, the ``del`` action will not actually free up any -# space in the HDF5 file. To free up space in the file, you will need to run the -# ``h5repack`` command line tool. See the `h5repack documentation -# `_ for more -# information. - - +# There are several types of dataset edits that cannot be done in-place: changing the +# shape of a dataset with a fixed shape, or changing the datatype, compression, +# chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using +# the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See +# :ref:`modifying_data` for more information. diff --git a/docs/gallery/general/add_remove_containers.py b/docs/gallery/general/add_remove_containers.py index 80c5cb032..0e4c545aa 100644 --- a/docs/gallery/general/add_remove_containers.py +++ b/docs/gallery/general/add_remove_containers.py @@ -70,31 +70,13 @@ # file path, and it is not possible to remove objects from an NWB file. You can use the # :py:meth:`NWBHDF5IO.export ` method, detailed below, to modify an NWB file in these ways. # -# .. warning:: -# -# NWB datasets that have been written to disk are read as :py:class:`h5py.Dataset ` objects. -# Directly modifying the data in these :py:class:`h5py.Dataset ` objects immediately -# modifies the data on disk -# (the :py:meth:`NWBHDF5IO.write ` method does not need to be called and the -# :py:class:`~pynwb.NWBHDF5IO` instance does not need to be closed). Directly modifying datasets in this way -# can lead to files that do not validate or cannot be opened, so exercise caution when using this method. -# Note: only chunked datasets or datasets with ``maxshape`` set can be resized. -# See the `h5py chunked storage documentation `_ -# for more details. - -############################################################################### -# .. note:: -# -# It is not possible to modify the attributes (fields) of an NWB container in memory. - -############################################################################### # Exporting a written NWB file to a new file path -# --------------------------------------------------- +# ----------------------------------------------- # Use the :py:meth:`NWBHDF5IO.export ` method to read data from an existing NWB file, # modify the data, and write the modified data to a new file path. Modifications to the data can be additions or # removals of objects, such as :py:class:`~pynwb.base.TimeSeries` objects. This is especially useful if you -# have raw data and processed data in the same NWB file and you want to create a new NWB file with all of the -# contents of the original file except for the raw data for sharing with collaborators. +# have raw data and processed data in the same NWB file and you want to create a new NWB file with all the contents of +# the original file except for the raw data for sharing with collaborators. # # To remove existing containers, use the :py:class:`~hdmf.utils.LabelledDict.pop` method on any # :py:class:`~hdmf.utils.LabelledDict` object, such as ``NWBFile.acquisition``, ``NWBFile.processing``, @@ -200,7 +182,7 @@ export_io.export(src_io=read_io, nwbfile=read_nwbfile) ############################################################################### -# More information about export -# --------------------------------- # For more information about the export functionality, see :ref:`export` # and the PyNWB documentation for :py:meth:`NWBHDF5IO.export `. +# +# For more options about editing a file in place, see :ref:`editing`.