From 7d19af187c985e2c38f334ad6b44fbb429ee8e37 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 8 Aug 2024 14:25:13 -0700 Subject: [PATCH 01/29] Add outline for adding user and developer documentation --- docs/pages/about.dox | 7 ------- docs/pages/devdocs.dox | 7 +++++++ docs/pages/devdocs/testing.dox | 5 +++++ docs/pages/userdocs.dox | 8 ++++++++ docs/pages/userdocs/hdf5io.dox | 5 +++++ 5 files changed, 25 insertions(+), 7 deletions(-) delete mode 100644 docs/pages/about.dox create mode 100644 docs/pages/devdocs.dox create mode 100644 docs/pages/devdocs/testing.dox create mode 100644 docs/pages/userdocs.dox create mode 100644 docs/pages/userdocs/hdf5io.dox diff --git a/docs/pages/about.dox b/docs/pages/about.dox deleted file mode 100644 index 2efbda93..00000000 --- a/docs/pages/about.dox +++ /dev/null @@ -1,7 +0,0 @@ -/** - * @page about About - * @section about-doxygen Doxygen documentation - * This page is auto generated using - * Doxygen, making use of some useful - * special commands. - */ diff --git a/docs/pages/devdocs.dox b/docs/pages/devdocs.dox new file mode 100644 index 00000000..cde46058 --- /dev/null +++ b/docs/pages/devdocs.dox @@ -0,0 +1,7 @@ +/** + * @page devdocs For Developers + * + * This documentation is intended for developers of AqNWB. + * + * @subpage testing + */ diff --git a/docs/pages/devdocs/testing.dox b/docs/pages/devdocs/testing.dox new file mode 100644 index 00000000..4925b3e0 --- /dev/null +++ b/docs/pages/devdocs/testing.dox @@ -0,0 +1,5 @@ +/** + * @page testing Testing + * + * Coming soon + */ diff --git a/docs/pages/userdocs.dox b/docs/pages/userdocs.dox new file mode 100644 index 00000000..a2078142 --- /dev/null +++ b/docs/pages/userdocs.dox @@ -0,0 +1,8 @@ +/** + * @page userdocs User Documentation + * + * This documentation is intended for users of AqNWB, e.g., developers seeking to integrate NWB + * with a particular data acquisition software via AqNWB. + * + * @subpage hdf5io + */ diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox new file mode 100644 index 00000000..4d13be38 --- /dev/null +++ b/docs/pages/userdocs/hdf5io.dox @@ -0,0 +1,5 @@ +/** + * @page hdf5io HDF5 I/O + * + * Coming soon + */ From 6aff062eb39d59a66db3ccc67c9d06ac51c626f3 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 8 Aug 2024 17:54:13 -0700 Subject: [PATCH 02/29] Add flush function for the IO and fix test build warnings for testHDF5IO --- src/BaseIO.hpp | 6 ++++++ src/hdf5/HDF5IO.cpp | 8 +++++++- src/hdf5/HDF5IO.hpp | 6 ++++++ tests/testHDF5IO.cpp | 13 ++++++++++--- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 71eab503..83d1c9f2 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -132,6 +132,12 @@ class BaseIO */ virtual Status close() = 0; + /** + * @brief Flush data to disk + * @return The status of the flush operation. + */ + virtual Status flush() = 0; + /** * @brief Creates an attribute at a given location in the file. * @param type The base data type of the attribute. diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index e505b217..707cf808 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -84,6 +84,12 @@ Status checkStatus(int status) return Status::Success; } +Status HDF5IO::flush() +{ + int status = H5Fflush(this->file->getId(), H5F_SCOPE_GLOBAL); + return checkStatus(status); +} + Status HDF5IO::createAttribute(const BaseDataType& type, const void* data, const std::string& path, @@ -407,7 +413,7 @@ Status HDF5IO::stopRecording() if (!disableSWMRMode) { close(); // SWMR mode cannot be disabled so close the file } else { - H5Fflush(this->file->getId(), H5F_SCOPE_GLOBAL); // flush all data to disk + this->flush(); } return Status::Success; } diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 8b1edbc2..1842714f 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -74,6 +74,12 @@ class HDF5IO : public BaseIO */ Status close() override; + /** + * @brief Flush data to disk + * @return The status of the flush operation. + */ + Status flush() override; + /** * @brief Creates an attribute at a given location in the file. * @param type The base data type of the attribute. diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index e4a6a3fa..c49be19f 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -90,6 +90,7 @@ TEST_CASE("writeDataset", "[hdf5io]") dataPath); Status status = dataset->writeDataBlock( dataShape, positionOffset, BaseDataType::I32, testData.data()); + REQUIRE(status == Status::Success); // Read back the 1D data block from 3D dataset std::unique_ptr dataRead1D = @@ -129,6 +130,7 @@ TEST_CASE("writeDataset", "[hdf5io]") // Write 2D data block Status status = dataset->writeDataBlock( dataShape, positionOffset, BaseDataType::I32, testData.data()); + REQUIRE(status == Status::Success); // Read back the 2D data block std::unique_ptr dsetRead2D = @@ -165,6 +167,7 @@ TEST_CASE("writeDataset", "[hdf5io]") dataPath); Status status = dataset->writeDataBlock( dataShape, positionOffset, BaseDataType::I32, testData.data()); + REQUIRE(status == Status::Success); // Read back the 1D data block from 3D dataset std::unique_ptr dataRead1D = @@ -201,6 +204,7 @@ TEST_CASE("writeDataset", "[hdf5io]") dataPath); Status status = dataset->writeDataBlock( dataShape, positionOffset, BaseDataType::I32, testData.data()); + REQUIRE(status == Status::Success); // Read back the 2D data block from 3D dataset std::unique_ptr dataRead2D = @@ -312,7 +316,7 @@ TEST_CASE("SWMRmode", "[hdf5io]") std::move(promise)); // write to file - for (int b = 0; b <= numBlocks; b++) { + for (SizeType b = 0; b <= numBlocks; b++) { // write data block and flush to file std::vector dataShape = {numSamples}; dataset->writeDataBlock(dataShape, BaseDataType::I32, &testData[0]); @@ -336,6 +340,9 @@ TEST_CASE("SWMRmode", "[hdf5io]") REQUIRE(retSWMREnabled == 0); // process should succeed if data was written // and read successfully + // test flush data to disk + REQUIRE(hdf5io->flush() == Status::Success); + // stop recording, check that file is closed and recording cannot be // restarted status = hdf5io->stopRecording(); @@ -365,7 +372,7 @@ TEST_CASE("SWMRmode", "[hdf5io]") REQUIRE(hdf5io->canModifyObjects() == true); // write to file - for (int b = 0; b <= numBlocks; b++) { + for (SizeType b = 0; b <= numBlocks; b++) { // write data block and flush to file std::vector dataShape = {numSamples}; dataset->writeDataBlock(dataShape, BaseDataType::I32, &testData[0]); @@ -396,7 +403,7 @@ TEST_CASE("SWMRmode", "[hdf5io]") SizeArray {1}, dataPathPostRestart); - for (int b = 0; b <= numBlocks; b++) { + for (SizeType b = 0; b <= numBlocks; b++) { // write data block and flush to file std::vector dataShape = {numSamples}; datasetPostRestart->writeDataBlock( From 58b515c7ede30a8b892a2faf1349bc46b2c6330f Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 8 Aug 2024 17:56:36 -0700 Subject: [PATCH 03/29] Add HDF5 example code snippet that is run in the tests --- docs/Doxyfile.in | 1 + docs/pages/userdocs/hdf5io.dox | 7 +++ tests/CMakeLists.txt | 4 +- tests/testHDF5IO_docs_examples.cpp | 70 ++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/testHDF5IO_docs_examples.cpp diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 6c60981c..0873ef12 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -8,6 +8,7 @@ PROJECT_NUMBER = "@PROJECT_VERSION@" # Add sources INPUT = "@PROJECT_SOURCE_DIR@/README.md" "@PROJECT_SOURCE_DIR@/src" "@PROJECT_SOURCE_DIR@/docs/pages" +EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests" IMAGE_PATH = "@PROJECT_SOURCE_DIR@/resources/images" EXTRACT_ALL = YES RECURSIVE = YES diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index 4d13be38..455b21f4 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -2,4 +2,11 @@ * @page hdf5io HDF5 I/O * * Coming soon + * + * \snippet tests/testHDF5IO_docs_examples.cpp example_HDF5_with_SWMR_mode + * + * - Initial size (data is expandable so doesn't matter too much), but if know it then we can set it + * - What chunking to use? + * - When to flush data to disk? + * - using std::make_unique(path) to manage memory */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7cbb5efe..b3f611bf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,7 +17,9 @@ add_executable(aq-nwb_test testFile.cpp testHDF5IO.cpp testNWBFile.cpp - testNWBRecording.cpp) + testNWBRecording.cpp + testHDF5IO_docs_examples.cpp + ) target_link_libraries( aq-nwb_test PRIVATE aq-nwb_lib diff --git a/tests/testHDF5IO_docs_examples.cpp b/tests/testHDF5IO_docs_examples.cpp new file mode 100644 index 00000000..ec6ebc83 --- /dev/null +++ b/tests/testHDF5IO_docs_examples.cpp @@ -0,0 +1,70 @@ +// HDF5 I/O Examples used in the Documentation +#include +#include +#include +#include +#include +#include + +#include + +#include "hdf5/HDF5IO.hpp" +#include "nwb/NWBFile.hpp" +#include "nwb/file/ElectrodeTable.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; +namespace fs = std::filesystem; + +TEST_CASE("SWMRmodeExamples", "[hdf5io]") +{ + SECTION("withSWMRMode") + { + // [example_HDF5_with_SWMR_mode] + // create and open the HDF5 file. SWMR mode is used by default + std::string path = getTestFilePath("testWithSWMRMode.h5"); + std::unique_ptr hdf5io = std::make_unique(path); + hdf5io->open(); + + // add a dataset + std::vector testData(10000); + std::iota(testData.begin(), testData.end(), 1); // Initalize the testData to 0, 1, 2, ... 10000 + std::string dataPath = "/data"; + SizeType numBlocks = 10; // write 10 chunks of + SizeType numSamples = testData.size(); + std::unique_ptr dataset = hdf5io->createArrayDataSet( + BaseDataType::I32, // type + SizeArray {0}, // size. Initial size of the dataset + SizeArray {1000}, // chunking. Size of a data chunk + dataPath); // path. Path to the dataset in the HDF5 file + + // Start recording. Starting the recording places the HDF5 file in SWMR mode + Status status = hdf5io->startRecording(); + REQUIRE(status == Status::Success); + + // Once in SWMR mode we can add data to the file but we can no longer create + // new data objects (Groups, Datasets, Attributes etc.). + REQUIRE(hdf5io->canModifyObjects() == false); + + // write the our testData to the file. + for (SizeType b = 0; b <= numBlocks; b++) { + // write a single 1D block of data and flush to file + std::vector dataShape = {numSamples}; + dataset->writeDataBlock(dataShape, BaseDataType::I32, &testData[0]); + // Optionally we can flush all data to disk + status = hdf5io->flush(); + REQUIRE(status == Status::Success); + } + + // stop recording. In SWMR mode the file is now closed and recording cannot be restarted + status = hdf5io->stopRecording(); + REQUIRE(hdf5io->isOpen() == false); + REQUIRE(hdf5io->startRecording() == Status::Failure); + // [example_HDF5_with_SWMR_mode] + } + + SECTION("disableSWMRMode") + { + // TODO + } +} From cbc063113bf12db325b44179a5e0f99bf23735e5 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 8 Aug 2024 18:02:26 -0700 Subject: [PATCH 04/29] Fix lint issues --- src/hdf5/HDF5IO.cpp | 2 +- src/hdf5/HDF5IO.hpp | 2 +- tests/testHDF5IO_docs_examples.cpp | 17 ++++++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/hdf5/HDF5IO.cpp b/src/hdf5/HDF5IO.cpp index 707cf808..f78b8449 100644 --- a/src/hdf5/HDF5IO.cpp +++ b/src/hdf5/HDF5IO.cpp @@ -86,7 +86,7 @@ Status checkStatus(int status) Status HDF5IO::flush() { - int status = H5Fflush(this->file->getId(), H5F_SCOPE_GLOBAL); + int status = H5Fflush(this->file->getId(), H5F_SCOPE_GLOBAL); return checkStatus(status); } diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 1842714f..9ccb39b6 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -78,7 +78,7 @@ class HDF5IO : public BaseIO * @brief Flush data to disk * @return The status of the flush operation. */ - Status flush() override; + Status flush() override; /** * @brief Creates an attribute at a given location in the file. diff --git a/tests/testHDF5IO_docs_examples.cpp b/tests/testHDF5IO_docs_examples.cpp index ec6ebc83..422f5c4e 100644 --- a/tests/testHDF5IO_docs_examples.cpp +++ b/tests/testHDF5IO_docs_examples.cpp @@ -3,8 +3,8 @@ #include #include #include -#include #include +#include #include @@ -28,15 +28,17 @@ TEST_CASE("SWMRmodeExamples", "[hdf5io]") // add a dataset std::vector testData(10000); - std::iota(testData.begin(), testData.end(), 1); // Initalize the testData to 0, 1, 2, ... 10000 + std::iota(testData.begin(), + testData.end(), + 1); // Initalize the testData to 0, 1, 2, ... 10000 std::string dataPath = "/data"; - SizeType numBlocks = 10; // write 10 chunks of + SizeType numBlocks = 10; // write 10 chunks of SizeType numSamples = testData.size(); std::unique_ptr dataset = hdf5io->createArrayDataSet( BaseDataType::I32, // type - SizeArray {0}, // size. Initial size of the dataset - SizeArray {1000}, // chunking. Size of a data chunk - dataPath); // path. Path to the dataset in the HDF5 file + SizeArray {0}, // size. Initial size of the dataset + SizeArray {1000}, // chunking. Size of a data chunk + dataPath); // path. Path to the dataset in the HDF5 file // Start recording. Starting the recording places the HDF5 file in SWMR mode Status status = hdf5io->startRecording(); @@ -56,7 +58,8 @@ TEST_CASE("SWMRmodeExamples", "[hdf5io]") REQUIRE(status == Status::Success); } - // stop recording. In SWMR mode the file is now closed and recording cannot be restarted + // stop recording. In SWMR mode the file is now closed and recording cannot + // be restarted status = hdf5io->stopRecording(); REQUIRE(hdf5io->isOpen() == false); REQUIRE(hdf5io->startRecording() == Status::Failure); From b175b17bd4c36da95cd7ed4fb5e63304712100bc Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 8 Aug 2024 18:04:55 -0700 Subject: [PATCH 05/29] Fix codespell --- tests/testHDF5IO_docs_examples.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testHDF5IO_docs_examples.cpp b/tests/testHDF5IO_docs_examples.cpp index 422f5c4e..ec462042 100644 --- a/tests/testHDF5IO_docs_examples.cpp +++ b/tests/testHDF5IO_docs_examples.cpp @@ -27,10 +27,10 @@ TEST_CASE("SWMRmodeExamples", "[hdf5io]") hdf5io->open(); // add a dataset - std::vector testData(10000); + std::vector testData(10000); // Initialize the testData to 0, 1, 2, ... 10000 with std::iota std::iota(testData.begin(), testData.end(), - 1); // Initalize the testData to 0, 1, 2, ... 10000 + 1); std::string dataPath = "/data"; SizeType numBlocks = 10; // write 10 chunks of SizeType numSamples = testData.size(); From ba9fcc48adc3c377890ed7c2af80dbf25c7543bb Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 8 Aug 2024 18:09:39 -0700 Subject: [PATCH 06/29] Fix lint --- tests/testHDF5IO_docs_examples.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/testHDF5IO_docs_examples.cpp b/tests/testHDF5IO_docs_examples.cpp index ec462042..4a298764 100644 --- a/tests/testHDF5IO_docs_examples.cpp +++ b/tests/testHDF5IO_docs_examples.cpp @@ -27,10 +27,8 @@ TEST_CASE("SWMRmodeExamples", "[hdf5io]") hdf5io->open(); // add a dataset - std::vector testData(10000); // Initialize the testData to 0, 1, 2, ... 10000 with std::iota - std::iota(testData.begin(), - testData.end(), - 1); + std::vector testData(10000); + std::iota(testData.begin(), testData.end(), 1); // Initialize testData std::string dataPath = "/data"; SizeType numBlocks = 10; // write 10 chunks of SizeType numSamples = testData.size(); From f8577d36117c005828bd2bb9dfe29c8a944eb9a0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 14 Aug 2024 22:29:52 -0700 Subject: [PATCH 07/29] Move examples to tests/examples subfolder --- docs/pages/userdocs/hdf5io.dox | 2 +- tests/CMakeLists.txt | 22 ++++++++++++++++--- .../test_HDF5IO_examples.cpp} | 0 3 files changed, 20 insertions(+), 4 deletions(-) rename tests/{testHDF5IO_docs_examples.cpp => examples/test_HDF5IO_examples.cpp} (100%) diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index 455b21f4..a47807f2 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -3,7 +3,7 @@ * * Coming soon * - * \snippet tests/testHDF5IO_docs_examples.cpp example_HDF5_with_SWMR_mode + * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_with_SWMR_mode * * - Initial size (data is expandable so doesn't matter too much), but if know it then we can set it * - What chunking to use? diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b3f611bf..e0f756b7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -18,13 +18,21 @@ add_executable(aq-nwb_test testHDF5IO.cpp testNWBFile.cpp testNWBRecording.cpp - testHDF5IO_docs_examples.cpp - ) + examples/test_HDF5IO_examples.cpp # Include examples source file +) + +# Ensure the aq-nwb_test target can include headers from the current directory and Catch2 +target_include_directories(aq-nwb_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} # Include current directory + ${CATCH2_INCLUDE_DIR} # Include Catch2 headers, if found by find_package +) + target_link_libraries( aq-nwb_test PRIVATE aq-nwb_lib Catch2::Catch2WithMain ) + target_compile_features(aq-nwb_test PRIVATE cxx_std_17) catch_discover_tests(aq-nwb_test) @@ -32,11 +40,19 @@ catch_discover_tests(aq-nwb_test) # ---- Custom Executable ---- add_executable(reader_executable - reader.cpp) + reader.cpp +) + +# Ensure the reader_executable target can include headers from the current directory +target_include_directories(reader_executable PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} # Include current directory +) + target_link_libraries( reader_executable PRIVATE aq-nwb_lib ) + target_compile_features(reader_executable PRIVATE cxx_std_17) # ---- End-of-file commands ---- diff --git a/tests/testHDF5IO_docs_examples.cpp b/tests/examples/test_HDF5IO_examples.cpp similarity index 100% rename from tests/testHDF5IO_docs_examples.cpp rename to tests/examples/test_HDF5IO_examples.cpp From a0d4f62ed7a3d3b5ecdd68dfb878f8ca6ed9ff9b Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 15 Aug 2024 15:24:55 -0700 Subject: [PATCH 08/29] Update line breaks in LICENSE --- LICENSE | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/LICENSE b/LICENSE index 0dc5d2ab..62249b4e 100644 --- a/LICENSE +++ b/LICENSE @@ -18,9 +18,17 @@ may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. You are under no obligation whatsoever to provide any bug fixes, patches, or upgrades to the features, functionality or performance of the source From 443620a653c2b01b59552e71e3d716d680a28c54 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 15 Aug 2024 15:34:12 -0700 Subject: [PATCH 09/29] Add legal abd build instructions to the docs --- README.md | 170 -------------------- docs/Doxyfile.in | 2 +- docs/pages/0_install.dox | 126 +++++++++++++++ docs/pages/{userdocs.dox => 1_userdocs.dox} | 4 +- docs/pages/2_devdocs.dox | 11 ++ docs/pages/devdocs.dox | 7 - docs/pages/devdocs/code_of_conduct.dox | 7 + docs/pages/devdocs/copyright.dox | 6 + docs/pages/devdocs/license.dox | 6 + 9 files changed, 159 insertions(+), 180 deletions(-) create mode 100644 docs/pages/0_install.dox rename docs/pages/{userdocs.dox => 1_userdocs.dox} (75%) create mode 100644 docs/pages/2_devdocs.dox delete mode 100644 docs/pages/devdocs.dox create mode 100644 docs/pages/devdocs/code_of_conduct.dox create mode 100644 docs/pages/devdocs/copyright.dox create mode 100644 docs/pages/devdocs/license.dox diff --git a/README.md b/README.md index 78f6f6bd..a42c5cd8 100644 --- a/README.md +++ b/README.md @@ -19,176 +19,6 @@ Below is a high-level overview of the project structure and capabilities we are ![Project Overview](resources/images/aqnwb_objective_500px.png) -# Requirements -* A C++17-compliant compiler -* CMake `>= 3.15` -* HDF5 `>= 1.10` -* Boost -* Additional requirements for building the documentation (optional) - * Doxygen - * Graphviz -* Additional requirements for developers (mode `dev`) - * cppcheck - * clang-format -# Building and installing -## Build -Here are the steps for building in release mode with a multi-configuration generator: - -```sh -cmake -S . -B build -cmake --build build --config Release -``` - -Note, if you are using custom installations of HDF5 or BOOST that are not being detected -automatically by cmake, you can specify `HDF5_ROOT` and `BOOST_ROOT` environment variables to -point to install directories of HDF5 and BOOST respectively. - - -## Install - -Here is the command for installing the release mode artifacts with a -multi-configuration generator: - -```sh -cmake --install build --config Release -``` - -## Developing - -Build system targets that are only useful for developers of this project are -hidden if the `aq-nwb_DEVELOPER_MODE` option is disabled. Enabling this -option makes tests and other developer targets and options available. You can enable -the option when configuring the build by adding ``-Daq-nwb_DEVELOPER_MODE=ON``, e.g.,: - -```sh -cmake -S . -B build -Daq-nwb_DEVELOPER_MODE=ON -``` -### Presets - -As a developer, you can create your own dev preset by making a `CMakeUserPresets.json` file at the root of -the project: - -```json -{ - "version": 2, - "cmakeMinimumRequired": { - "major": 3, - "minor": 15, - "patch": 0 - }, - "configurePresets": [ - { - "name": "dev", - "binaryDir": "${sourceDir}/build/dev", - "inherits": ["dev-mode", "ci-"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - } - ], - "buildPresets": [ - { - "name": "dev", - "configurePreset": "dev", - "configuration": "Debug" - } - ], - "testPresets": [ - { - "name": "dev", - "configurePreset": "dev", - "configuration": "Debug", - "output": { - "outputOnFailure": true - } - } - ] -} -``` -Replace `` in the `CmakeUserPresets.json` file with the name of -the operating system you have (`win64`, `linux` or `darwin`). - -### Configure, build and test - -You can configure, build and test the project respectively with the following commands from the project root on -any operating system with any build system: - -```sh -cmake --preset=dev -cmake --build --preset=dev -ctest --preset=dev -``` - -### Developer mode targets - -Additional targets can be invoked when in development mode using the commands below - -```sh -cmake --build --preset=dev --target= -``` - -#### Target options -- `format-check` and `format-fix`: run the clang-format tool on the codebase to check errors and to fix them respectively. -- `spell-check` and `spell-fix`: run the codespell tool on the codebase to check errors and to fix them respectively. -- `docs` : builds to documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target) - -# Code of Conduct - -This project and everyone participating in it is govered by our [code of conduct guidelines](https://github.com/NeurodataWithoutBorders/aqnwb/blob/main/.github/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. - -# LICENSE - -AqNWB Copyright (c) 2024, The Regents of the University of California, -through Lawrence Berkeley National Laboratory (subject to receipt of any -required approvals from the U.S. Dept. of Energy). All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -(1) Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -(2) Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -(3) Neither the name of the University of California, Lawrence Berkeley -National Laboratory, U.S. Dept. of Energy nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. - - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You are under no obligation whatsoever to provide any bug fixes, patches, -or upgrades to the features, functionality or performance of the source -code ("Enhancements") to anyone; however, if you choose to make your -Enhancements available either publicly, or directly to Lawrence Berkeley -National Laboratory, without imposing a separate written license agreement -for such Enhancements, then you hereby grant the following license: a -non-exclusive, royalty-free perpetual license to install, use, modify, -prepare derivative works, incorporate into other computer software, -distribute, and sublicense such enhancements or derivative works thereof, -in binary and source code form. - -# COPYRIGHT - -AqNWB Copyright (c) 2024, The Regents of the University of California, -through Lawrence Berkeley National Laboratory (subject to receipt of any -required approvals from the U.S. Dept. of Energy). All rights reserved. - -If you have questions about your rights to use or distribute this software, -please contact Berkeley Lab's Intellectual Property Office at -IPO@lbl.gov. - -NOTICE. This Software was developed under funding from the U.S. Department -of Energy and the U.S. Government consequently retains certain rights. As -such, the U.S. Government has been granted for itself and others acting on -its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the -Software to reproduce, distribute copies to the public, prepare derivative -works, and perform publicly and display publicly, and to permit others to do so. diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 0873ef12..cb13342e 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -8,7 +8,7 @@ PROJECT_NUMBER = "@PROJECT_VERSION@" # Add sources INPUT = "@PROJECT_SOURCE_DIR@/README.md" "@PROJECT_SOURCE_DIR@/src" "@PROJECT_SOURCE_DIR@/docs/pages" -EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests" +EXAMPLE_PATH = "@PROJECT_SOURCE_DIR@/tests" "@PROJECT_SOURCE_DIR@/.github/CODE_OF_CONDUCT.md" "@PROJECT_SOURCE_DIR@/Legal.txt" "@PROJECT_SOURCE_DIR@/LICENSE" IMAGE_PATH = "@PROJECT_SOURCE_DIR@/resources/images" EXTRACT_ALL = YES RECURSIVE = YES diff --git a/docs/pages/0_install.dox b/docs/pages/0_install.dox new file mode 100644 index 00000000..5f836e3f --- /dev/null +++ b/docs/pages/0_install.dox @@ -0,0 +1,126 @@ +/** + * @page install_page Installing AqNWB + * + * \section requirements Requirements + * - A C++17-compliant compiler + * - CMake >= 3.15 + * - HDF5 >= 1.10 + * - Boost + * - Additional requirements for building the documentation (optional) + * - Doxygen + * - Graphviz + * - Additional requirements for developers (mode `dev`) + * - cppcheck + * - clang-format + * + * \section userbuild_sec User Build + * + * \subsection userbuild_build_subsec Build + * + * Here are the steps for building in release mode with a multi-configuration generator: + * + * \code{.sh} + * cmake -S . -B build + * cmake --build build --config Release + * \endcode + * + * Note, if you are using custom installations of HDF5 or BOOST that are not being detected + * automatically by cmake, you can specify `HDF5_ROOT` and `BOOST_ROOT` environment variables to + * point to install directories of HDF5 and BOOST respectively. + * + * \subsection userbuild_install_subsec Install + * + * Here is the command for installing the release mode artifacts with a + * multi-configuration generator: + * + * \code{.sh} + * cmake --install build --config Release + * \endcode + * + * + * + * \section devbuild_sec Developer Build + * + * Build system targets that are only useful for developers of AqNWB are + * hidden if the `aq-nwb_DEVELOPER_MODE` option is disabled. Enabling this + * option makes tests and other developer targets and options available. You can enable + * the option when configuring the build by adding `-Daq-nwb_DEVELOPER_MODE=ON`, e.g., + * + * \code{.sh} + * cmake -S . -B build -Daq-nwb_DEVELOPER_MODE=ON + * \endcode + * + * \subsection devbuild_presets_subsec Presets + * + * As a developer, you can create your own dev preset by making a `CMakeUserPresets.json` file at the root of + * the project: + * + * \code{.json} + * { + * "version": 2, + * "cmakeMinimumRequired": { + * "major": 3, + * "minor": 15, + * "patch": 0 + * }, + * "configurePresets": [ + * { + * "name": "dev", + * "binaryDir": "${sourceDir}/build/dev", + * "inherits": ["dev-mode", "ci-"], + * "cacheVariables": { + * "CMAKE_BUILD_TYPE": "Debug" + * } + * } + * ], + * "buildPresets": [ + * { + * "name": "dev", + * "configurePreset": "dev", + * "configuration": "Debug" + * } + * ], + * "testPresets": [ + * { + * "name": "dev", + * "configurePreset": "dev", + * "configuration": "Debug", + * "output": { + * "outputOnFailure": true + * } + * } + * ] + * } + * \endcode + * Replace `` in the `CMakeUserPresets.json` file with the name of + * the operating system you have (`win64`, `linux` or `darwin`). + * + * \subsection configure_build_test Configure, build and test + * + * You can configure, build and test the project respectively with the following commands from the project root on + * any operating system with any build system: + * + * \code{.sh} + * cmake --preset=dev + * cmake --build --preset=dev + * ctest --preset=dev + * \endcode + * + * \subsection devbuild_dev_mode_targets_subsec Developer mode targets + * + * Additional targets can be invoked when in development mode using the commands below + * + * \code{.sh} + * cmake --build --preset=dev --target= + * \endcode + * + * \subsubsection devbuild_target_options_subsubsec Target options + * - `format-check` and `format-fix`: run the clang-format tool on the codebase to check errors and to fix them respectively. + * - `spell-check` and `spell-fix`: run the codespell tool on the codebase to check errors and to fix them respectively. + * - `docs` : builds the documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target) + */ + + + + + */ diff --git a/docs/pages/userdocs.dox b/docs/pages/1_userdocs.dox similarity index 75% rename from docs/pages/userdocs.dox rename to docs/pages/1_userdocs.dox index a2078142..6c4c1f1c 100644 --- a/docs/pages/userdocs.dox +++ b/docs/pages/1_userdocs.dox @@ -1,8 +1,8 @@ /** - * @page userdocs User Documentation + * @page userdocs For Users * * This documentation is intended for users of AqNWB, e.g., developers seeking to integrate NWB * with a particular data acquisition software via AqNWB. * - * @subpage hdf5io + * - @subpage hdf5io */ diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox new file mode 100644 index 00000000..c0ab3880 --- /dev/null +++ b/docs/pages/2_devdocs.dox @@ -0,0 +1,11 @@ +/** + * @page devdocs For Developers + * + * This documentation is intended for developers of AqNWB. + * + * - @subpage testing + * - @subpage code_of_conduct_page + * - @subpage license_page + * - @subpage copyright_page + * + */ diff --git a/docs/pages/devdocs.dox b/docs/pages/devdocs.dox deleted file mode 100644 index cde46058..00000000 --- a/docs/pages/devdocs.dox +++ /dev/null @@ -1,7 +0,0 @@ -/** - * @page devdocs For Developers - * - * This documentation is intended for developers of AqNWB. - * - * @subpage testing - */ diff --git a/docs/pages/devdocs/code_of_conduct.dox b/docs/pages/devdocs/code_of_conduct.dox new file mode 100644 index 00000000..24d0364f --- /dev/null +++ b/docs/pages/devdocs/code_of_conduct.dox @@ -0,0 +1,7 @@ +/** + * @page code_of_conduct_page Code of Conduct + * + * This project and everyone participating in it is governed by our code of conduct guidelines. By participating, you are expected to uphold this code. + * \include CODE_OF_CONDUCT.md + */ + diff --git a/docs/pages/devdocs/copyright.dox b/docs/pages/devdocs/copyright.dox new file mode 100644 index 00000000..8f6532be --- /dev/null +++ b/docs/pages/devdocs/copyright.dox @@ -0,0 +1,6 @@ +/** + * @page copyright_page Copyright + * + * \include Legal.txt + */ + diff --git a/docs/pages/devdocs/license.dox b/docs/pages/devdocs/license.dox new file mode 100644 index 00000000..7b57b4a5 --- /dev/null +++ b/docs/pages/devdocs/license.dox @@ -0,0 +1,6 @@ +/** + * @page license_page License + * + * \include LICENSE + */ + From 07d0e6933a0c95e882d242dc3bccb766ab86d851 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Aug 2024 19:08:17 -0700 Subject: [PATCH 10/29] Attempt to fix test failure --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e0f756b7..9d4f8e70 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,7 +24,7 @@ add_executable(aq-nwb_test # Ensure the aq-nwb_test target can include headers from the current directory and Catch2 target_include_directories(aq-nwb_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} # Include current directory - ${CATCH2_INCLUDE_DIR} # Include Catch2 headers, if found by find_package + # ${CATCH2_INCLUDE_DIR} # Include Catch2 headers, if found by find_package ) target_link_libraries( From c97caa7d196723e66b1eaa6aa66339b47e347cdd Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Aug 2024 19:20:14 -0700 Subject: [PATCH 11/29] Add codespell to the requirements --- docs/pages/0_install.dox | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/0_install.dox b/docs/pages/0_install.dox index 5f836e3f..66f3f902 100644 --- a/docs/pages/0_install.dox +++ b/docs/pages/0_install.dox @@ -12,6 +12,7 @@ * - Additional requirements for developers (mode `dev`) * - cppcheck * - clang-format + * - codespell (optional, required for ``target=spell-check``) * * \section userbuild_sec User Build * From d904ca1791623f738f199c04c73fb760171b0f59 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Aug 2024 19:20:40 -0700 Subject: [PATCH 12/29] Add Doxygen commands to the codespell ignore list --- .codespell_ignore | 5 +++++ .codespellrc | 1 + 2 files changed, 6 insertions(+) create mode 100644 .codespell_ignore diff --git a/.codespell_ignore b/.codespell_ignore new file mode 100644 index 00000000..7d2e12d3 --- /dev/null +++ b/.codespell_ignore @@ -0,0 +1,5 @@ +endcode +code +param +brief +retval diff --git a/.codespellrc b/.codespellrc index 9052615f..8951f4e3 100644 --- a/.codespellrc +++ b/.codespellrc @@ -4,3 +4,4 @@ check-filenames = check-hidden = skip = */.git,*/build,*/prefix,*/resources,*/.vscode quiet-level = 2 +ignore-words=./.codespell_ignore From a869fdbafa3fc64b7731c9c997278d733f998bbd Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 20 Aug 2024 20:12:28 -0700 Subject: [PATCH 13/29] Add brief testing docs --- docs/pages/0_install.dox | 3 ++- docs/pages/devdocs/testing.dox | 43 +++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/pages/0_install.dox b/docs/pages/0_install.dox index 66f3f902..a1867f0d 100644 --- a/docs/pages/0_install.dox +++ b/docs/pages/0_install.dox @@ -25,7 +25,8 @@ * cmake --build build --config Release * \endcode * - * Note, if you are using custom installations of HDF5 or BOOST that are not being detected + * \note + * If you are using custom installations of **HDF5** or **BOOST** that are not being detected * automatically by cmake, you can specify `HDF5_ROOT` and `BOOST_ROOT` environment variables to * point to install directories of HDF5 and BOOST respectively. * diff --git a/docs/pages/devdocs/testing.dox b/docs/pages/devdocs/testing.dox index 4925b3e0..ce62fd79 100644 --- a/docs/pages/devdocs/testing.dox +++ b/docs/pages/devdocs/testing.dox @@ -1,5 +1,46 @@ /** * @page testing Testing * - * Coming soon + * \section testing_unit Unit Tests + * + * `AqNWB` uses `ctest` and `CATCH2` for unit testing. + * Relevant files for the unit tests are located at: + * * ``/tests`` : Sources of the unit tests + * * ``/tests/examples`` : Sources of example code used in the docs. The examples are implemented like + * regular unit tests but their purpose is to provide examples for the docs (rather than testing + * specific cases) and as such commonly include Doxygen section markers to allow include of sections + * of code in the Doxygen docs. + * + * + * \subsection testing_unit_run Running Unit Tests + * + * If you built AqNWB with the `dev` mode preset, then simply call: + * + * \code{.sh} + * ctest --preset=dev + * \endcode + * + * Alternatively go to your build directory and run ``ctest`` directly, e.g.: + * + * \code{.sh} + * cd + * ctest + * \endcode + * + * \section testing_spellcheck Spellcheck + * + * AqNWB uses ``codepsell`` to check the code for spelling errors. You can run the spellchecker via: + * + * \code{.sh} + * cmake --build --preset=dev --target=spell-check + * \endcode + * + * \section testing_lint Linting + * + * AqNWB uses ``clang-format`` for linting the code. You can run the linter via: + * + * \code{.sh} + * cmake --build --preset=dev --target=format-check + * \endcode + * */ From a4ed61cb12ecd11e5f08899ceb3f1a0bd8239955 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Aug 2024 00:13:46 -0700 Subject: [PATCH 14/29] Add HDF5 I/O SWMR and Chunking docs --- docs/pages/userdocs/hdf5io.dox | 84 +++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index a47807f2..27afb5cc 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -1,12 +1,84 @@ /** * @page hdf5io HDF5 I/O * - * Coming soon + * \section hdf5io_swmr Single-Writer Multiple-Reader (SWMR) Mode * - * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_with_SWMR_mode + * The \ref AQNWB::HDF5::HDF5IO I/O backend uses by default SWMR mode while recording data. + * The SWMR mode in HDF5 allows one process to write to an HDF5 file while allowing multiple + * other processes to read from the file concurrently. + * + * \subsection hdf5io_swmr_features Why does AqNWB use SMWR mode? + * + * Using SWMR has several key advantages for data acquisition applications: + * + * - \b Concurrent \b Access: Enables one writer process to update the file while + * multiple reader processes read from it without blocking each other. + * - \b Data \b Consistency \b and \b Integrity: Ensures that readers see a consistent view of + * the data, even as it is being written. Readers will only see data that has been completely + * written and flushed to disk. Hence, SWMR mode, maintains the integrity and consistency of + * the data, ensuring that the HDF5 file remains readable even if errors should occur during + * the data acquisition process. + * - \b Real-Time \b Data \b Access: Useful for applications that need to monitor + * and analyze data in real-time as it is being generated. + * - \b Simplified \b Workflow \b for \b Real \b Time \b Analyses: Simplifies the + * architecture of applications that require real-time data consumption during acquisition, + * avoiding the need for intermediate storage solutions and complex inter-process communication + * or file locking mechanisms. + * + * \note + * While SWMR mode ensures data integrity, some data loss may still occur if the application crashes. + * Only data that has been completely written and flushed to disk will be readable. To manually + * flush data to disk use \ref AQNWB::HDF5::HDF5IO::flush . + * + * \subsection hdf5io_swmr_workflow SWMR Workflow + * + * SWMR mode is enabled when calling \ref AQNWB::HDF5::HDF5IO::startRecording . Once SWMR mode is + * enabled, no new data objects (Datasets, Groups, Attributes etc.) can be created, but we can + * only add and set values to existing data objects. Since other processes may read from the + * HDF5 file, it is not possible to intermittently disable SWMR mode to add new objects, i.e., + * once SWMR mode is enabled, the only way to add new objects to the file is to close the + * file and reopen in read/write mode. As such, the typical workflow when using + * SWMR mode during data acquisition is to: + * + * 1. Open the HDF5 file + * 2. Create all elements of the NWB file + * 3. Start the recording process + * 4. Stop recording and close the file + * + * This workflow is applicable to a wide range of data acquisition use-cases. However, + * for use cases that require creation of new Groups and Datasets during acquisition, + * you can disable the use of SWMR mode by setting `disableSWMRMode=true` when + * constructing the \ref AQNWB::HDF5::HDF5IO object. + * + * \warning + * While disabling SWMR mode allows Groups and Datasets to be created during and after + * recording, this comes at the cost of losing the concurrent access and data integrity + * features that SWMR mode provides. + * + * \subsection hdf5io_swmr_example Code Example: SWMR Workflow + * + * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_with_SWMR_mode + * + * \section hdf5io_chunking Chunking + * + * For datasets intended for recording, `AqNWB` using chunking by default. + * Using chunking in HDF5, a dataset is divided into fixed-size blocks (called chunks), + * which are stored separately in the file. This technique is particularly + * beneficial for large datasets and offers several advantages: + * + * - **Extend datasets**: Chunked datasets can be easily extended in any dimension. + * This flexibility is crucial for recording datasets where the size of the dataset + * is not known in advance. + * - **Performance Optimization**: By carefully choosing the chunk size, you can optimize + * performance based on your particular read/write access patterns. When only a portion + * of a chunked dataset is accessed, only the relevant chunks are read or written, + * reducing the amount of I/O operations. + * - **Compression**: Data within each chunk can be compressed independently, which can help + * to significant reduce data size, especially for datasets with redundancy. + * + * \warning + * Choosing a chunking configuration that does not align well with the desired read/write pattern + * may lead to reduced performance due to repeated read, decompression, and update to the same + * chunk or read of extra data as chunks are always read fully. * - * - Initial size (data is expandable so doesn't matter too much), but if know it then we can set it - * - What chunking to use? - * - When to flush data to disk? - * - using std::make_unique(path) to manage memory */ From 5ffdea68eec6dc0739782b941f7cec49c40bbc3b Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Aug 2024 00:14:07 -0700 Subject: [PATCH 15/29] Add docstrings for namespaces --- src/BaseIO.hpp | 4 ++++ src/hdf5/HDF5IO.hpp | 4 ++++ src/nwb/NWBFile.hpp | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/BaseIO.hpp b/src/BaseIO.hpp index 83d1c9f2..3dd375db 100644 --- a/src/BaseIO.hpp +++ b/src/BaseIO.hpp @@ -15,6 +15,10 @@ using Status = AQNWB::Types::Status; using SizeArray = AQNWB::Types::SizeArray; using SizeType = AQNWB::Types::SizeType; +/*! + * \namespace AQNWB + * \brief The main namespace for AqNWB + */ namespace AQNWB { diff --git a/src/hdf5/HDF5IO.hpp b/src/hdf5/HDF5IO.hpp index 9ccb39b6..2a94aef2 100644 --- a/src/hdf5/HDF5IO.hpp +++ b/src/hdf5/HDF5IO.hpp @@ -17,6 +17,10 @@ class DataType; class Exception; } // namespace H5 +/*! + * \namespace AQNWB::HDF5 + * \brief Namespace for all components of the HDF5 I/O backend + */ namespace AQNWB::HDF5 { class HDF5RecordingData; // declare here because gets used in HDF5IO class diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index f4e58bf4..04a0b357 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -8,6 +8,10 @@ #include "Types.hpp" #include "nwb/base/TimeSeries.hpp" +/*! + * \namespace AQNWB::NWB + * \brief Namespace for all classes related to the NWB data standard + */ namespace AQNWB::NWB { From 19b2660c164c9f330c30de36f1b4eb59bda4db59 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Aug 2024 00:47:51 -0700 Subject: [PATCH 16/29] Move install docs and diverse fixes --- README.md | 7 +++---- docs/pages/1_userdocs.dox | 1 + docs/pages/2_devdocs.dox | 1 + docs/pages/userdocs/hdf5io.dox | 18 +++++++++--------- .../{0_install.dox => userdocs/install.dox} | 0 5 files changed, 14 insertions(+), 13 deletions(-) rename docs/pages/{0_install.dox => userdocs/install.dox} (100%) diff --git a/README.md b/README.md index a42c5cd8..57a7ab3c 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,16 @@ [![Docs](https://img.shields.io/badge/AqNWB-Docs-8A2BE2?style=flat)](https://neurodatawithoutborders.github.io/aqnwb/) [![Code Stats](https://img.shields.io/badge/AqNWB-Code%20Statistics-8A2BE2?style=flat)](https://nwb-overview.readthedocs.io/en/latest/nwb-project-analytics/docs/source/code_stat_pages/AqNWB_stats.html) +**AqNWB is currently under active development and should not yet be used in practice.** AqNWB is a C++ API for acquiring neurophysiological data directly into the NWB (Neurodata Without Borders) format. Our goal is to provide a lightweight API to integrate with existing acquisition systems. -Please note, AqNWB is currently under development and should not yet be used in practice. +Please see the [Documentation](https://neurodatawithoutborders.github.io/aqnwb/) for +installation instructions and to learn more about AqNWB. Below is a high-level overview of the project structure and capabilities we are targeting: ![Project Overview](resources/images/aqnwb_objective_500px.png) - - - diff --git a/docs/pages/1_userdocs.dox b/docs/pages/1_userdocs.dox index 6c4c1f1c..c6a90b61 100644 --- a/docs/pages/1_userdocs.dox +++ b/docs/pages/1_userdocs.dox @@ -4,5 +4,6 @@ * This documentation is intended for users of AqNWB, e.g., developers seeking to integrate NWB * with a particular data acquisition software via AqNWB. * + * - @subpage install_page * - @subpage hdf5io */ diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox index c0ab3880..30ed6283 100644 --- a/docs/pages/2_devdocs.dox +++ b/docs/pages/2_devdocs.dox @@ -3,6 +3,7 @@ * * This documentation is intended for developers of AqNWB. * + * - \ref install_page (part of the \ref userdocs docs) * - @subpage testing * - @subpage code_of_conduct_page * - @subpage license_page diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index 27afb5cc..1841e848 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -3,7 +3,7 @@ * * \section hdf5io_swmr Single-Writer Multiple-Reader (SWMR) Mode * - * The \ref AQNWB::HDF5::HDF5IO I/O backend uses by default SWMR mode while recording data. + * The \ref AQNWB::HDF5::HDF5IO "HDF5IO" I/O backend uses by default SWMR mode while recording data. * The SWMR mode in HDF5 allows one process to write to an HDF5 file while allowing multiple * other processes to read from the file concurrently. * @@ -28,17 +28,17 @@ * \note * While SWMR mode ensures data integrity, some data loss may still occur if the application crashes. * Only data that has been completely written and flushed to disk will be readable. To manually - * flush data to disk use \ref AQNWB::HDF5::HDF5IO::flush . + * flush data to disk use \ref AQNWB::HDF5::HDF5IO::flush "HDF5IO::flush". * * \subsection hdf5io_swmr_workflow SWMR Workflow * - * SWMR mode is enabled when calling \ref AQNWB::HDF5::HDF5IO::startRecording . Once SWMR mode is - * enabled, no new data objects (Datasets, Groups, Attributes etc.) can be created, but we can - * only add and set values to existing data objects. Since other processes may read from the - * HDF5 file, it is not possible to intermittently disable SWMR mode to add new objects, i.e., - * once SWMR mode is enabled, the only way to add new objects to the file is to close the - * file and reopen in read/write mode. As such, the typical workflow when using - * SWMR mode during data acquisition is to: + * SWMR mode is enabled when calling \ref AQNWB::HDF5::HDF5IO::startRecording "HDF5IO::startRecording". + * Once SWMR mode is enabled, no new data objects (Datasets, Groups, Attributes etc.) + * can be created, but we can only add and set values to existing data objects. Since other + * processes may read from the HDF5 file, it is not possible to intermittently disable + * SWMR mode to add new objects, i.e., once SWMR mode is enabled, the only way to add + * new objects to the file is to close the file and reopen in read/write mode. As such, + * the typical workflow when using SWMR mode during data acquisition is to: * * 1. Open the HDF5 file * 2. Create all elements of the NWB file diff --git a/docs/pages/0_install.dox b/docs/pages/userdocs/install.dox similarity index 100% rename from docs/pages/0_install.dox rename to docs/pages/userdocs/install.dox From 711b6e1c2cac519e4e64ae9f5ca3fd0adab28eb1 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 21 Aug 2024 01:35:43 -0700 Subject: [PATCH 17/29] Add example with SMWR disabled --- docs/pages/userdocs/hdf5io.dox | 52 +++++++++++++--------- tests/examples/test_HDF5IO_examples.cpp | 59 ++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 21 deletions(-) diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index 1841e848..521ef0a6 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -1,6 +1,29 @@ /** * @page hdf5io HDF5 I/O * + * \section hdf5io_chunking Chunking + * + * For datasets intended for recording, `AqNWB` using chunking by default. + * Using chunking in HDF5, a dataset is divided into fixed-size blocks (called chunks), + * which are stored separately in the file. This technique is particularly + * beneficial for large datasets and offers several advantages: + * + * - **Extend datasets**: Chunked datasets can be easily extended in any dimension. + * This flexibility is crucial for recording datasets where the size of the dataset + * is not known in advance. + * - **Performance Optimization**: By carefully choosing the chunk size, you can optimize + * performance based on your particular read/write access patterns. When only a portion + * of a chunked dataset is accessed, only the relevant chunks are read or written, + * reducing the amount of I/O operations. + * - **Compression**: Data within each chunk can be compressed independently, which can help + * to significant reduce data size, especially for datasets with redundancy. + * + * \warning + * Choosing a chunking configuration that does not align well with the desired read/write pattern + * may lead to reduced performance due to repeated read, decompression, and update to the same + * chunk or read of extra data as chunks are always read fully. + * + * * \section hdf5io_swmr Single-Writer Multiple-Reader (SWMR) Mode * * The \ref AQNWB::HDF5::HDF5IO "HDF5IO" I/O backend uses by default SWMR mode while recording data. @@ -55,30 +78,19 @@ * recording, this comes at the cost of losing the concurrent access and data integrity * features that SWMR mode provides. * - * \subsection hdf5io_swmr_example Code Example: SWMR Workflow + * \subsection hdf5io_swmr_examples Code Examples * - * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_with_SWMR_mode + * This code snippet shows all the includes that are being used by the code examples + * shown in this section: * - * \section hdf5io_chunking Chunking + * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_includes * - * For datasets intended for recording, `AqNWB` using chunking by default. - * Using chunking in HDF5, a dataset is divided into fixed-size blocks (called chunks), - * which are stored separately in the file. This technique is particularly - * beneficial for large datasets and offers several advantages: + * \subsubsection hdf5io_swmr_examples_with_swmr Workflow with SWMR * - * - **Extend datasets**: Chunked datasets can be easily extended in any dimension. - * This flexibility is crucial for recording datasets where the size of the dataset - * is not known in advance. - * - **Performance Optimization**: By carefully choosing the chunk size, you can optimize - * performance based on your particular read/write access patterns. When only a portion - * of a chunked dataset is accessed, only the relevant chunks are read or written, - * reducing the amount of I/O operations. - * - **Compression**: Data within each chunk can be compressed independently, which can help - * to significant reduce data size, especially for datasets with redundancy. + * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_with_SWMR_mode * - * \warning - * Choosing a chunking configuration that does not align well with the desired read/write pattern - * may lead to reduced performance due to repeated read, decompression, and update to the same - * chunk or read of extra data as chunks are always read fully. + * \subsubsection hdf5io_noswmr_examples_without_swmr Workflow with SWMR disabled + * + * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_without_SWMR_mode * */ diff --git a/tests/examples/test_HDF5IO_examples.cpp b/tests/examples/test_HDF5IO_examples.cpp index 4a298764..3a978f60 100644 --- a/tests/examples/test_HDF5IO_examples.cpp +++ b/tests/examples/test_HDF5IO_examples.cpp @@ -1,4 +1,5 @@ // HDF5 I/O Examples used in the Documentation +// [example_HDF5_includes] #include #include #include @@ -15,6 +16,7 @@ using namespace AQNWB; namespace fs = std::filesystem; +// [example_HDF5_includes] TEST_CASE("SWMRmodeExamples", "[hdf5io]") { @@ -66,6 +68,61 @@ TEST_CASE("SWMRmodeExamples", "[hdf5io]") SECTION("disableSWMRMode") { - // TODO + // [example_HDF5_without_SWMR_mode] + // create and open the HDF5 file. With SWMR mode explicitly disabled + std::string path = getTestFilePath("testWithoutSWMRMode.h5"); + std::unique_ptr hdf5io = + std::make_unique(path, + true // Disable SWMR mode + ); + hdf5io->open(); + + // add a dataset + std::vector testData(10000); + std::iota(testData.begin(), testData.end(), 1); // Initialize testData + std::string dataPath = "/data"; + SizeType numBlocks = 10; // write 10 chunks of + SizeType numSamples = testData.size(); + std::unique_ptr dataset = hdf5io->createArrayDataSet( + BaseDataType::I32, // type + SizeArray {0}, // size. Initial size of the dataset + SizeArray {1000}, // chunking. Size of a data chunk + dataPath); // path. Path to the dataset in the HDF5 file + + // Start recording. Starting the recording places the HDF5 file in SWMR mode + Status status = hdf5io->startRecording(); + REQUIRE(status == Status::Success); + + // With SWMR mode disabled we are still allowed to create new data objects + // (Groups, Datasets, Attributes etc.) during the recording. However, with + // SWMR mode disabled, we lose the data consistency and concurrent read + // features that SWMR mode provides. + REQUIRE(hdf5io->canModifyObjects() == true); + + // write the our testData to the file. + for (SizeType b = 0; b <= numBlocks; b++) { + // write a single 1D block of data and flush to file + std::vector dataShape = {numSamples}; + dataset->writeDataBlock(dataShape, BaseDataType::I32, &testData[0]); + // Optionally we can flush all data to disk + status = hdf5io->flush(); + REQUIRE(status == Status::Success); + } + + // stop recording. + status = hdf5io->stopRecording(); + + // Since SWMR mode is disabled, stopping the recording won't close the file + // so that we can restart the recording if we want to + REQUIRE(hdf5io->isOpen() == true); + + // Restart the recording + REQUIRE(hdf5io->startRecording() == Status::Success); + + // Stop the recording and close the file + hdf5io->stopRecording(); + hdf5io->close(); + REQUIRE(hdf5io->isOpen() == false); + // [example_HDF5_without_SWMR_mode] } } From 44dc911b101b1f7fd50fc308114a419fce1158cd Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 22 Aug 2024 11:13:08 -0700 Subject: [PATCH 18/29] Add dot figures for the HDF5IO docs --- docs/pages/userdocs/hdf5io.dox | 114 +++++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 4 deletions(-) diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index 521ef0a6..26bc3aad 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -3,7 +3,8 @@ * * \section hdf5io_chunking Chunking * - * For datasets intended for recording, `AqNWB` using chunking by default. + * For datasets intended for recording, `AqNWB` uses chunking to ensure the dataset + * can be extended as new data arrives during the recording process. * Using chunking in HDF5, a dataset is divided into fixed-size blocks (called chunks), * which are stored separately in the file. This technique is particularly * beneficial for large datasets and offers several advantages: @@ -24,11 +25,105 @@ * chunk or read of extra data as chunks are always read fully. * * + * + * \dot + * digraph Chunking { + * node [shape=plaintext]; + * + * // Define the HDF5 file node with a distinct shape + * subgraph cluster_0 { + * label="HDF5 File"; + * style=filled; + * shape=folder; + * color=white; + * penwidth=1; + * pencolor=black; + * + * // Define the dataset node + * dataset [label=< + * + * + * + * + * + * + * + * + * + * + * + * + *
HDF5 Dataset
+ * + * + * + *
+ * 		+ * + * + * + *
+ *
+ * + * + * + *
+ * 		+ * + * + * + *
+ *
+ * >]; + * + * // Define the chunks + * chunk1 [label="Chunk 1", shape=box, fillcolor=lightgreen, style=filled]; + * chunk2 [label="Chunk 2", shape=box, fillcolor=lightgreen, style=filled]; + * chunk4 [label="Chunk 4", shape=box, fillcolor=lightgreen, style=filled]; + * chunk3 [label="Chunk 3", shape=box, fillcolor=lightgreen, style=filled]; + * + * // Define the edges + * dataset:chunk1 -> chunk1 [label=""]; + * dataset:chunk2 -> chunk2 [label=""]; + * dataset:chunk3 -> chunk3 [label=""]; + * dataset:chunk4 -> chunk4 [label=""]; + * } + * } + * \enddot + * + * * \section hdf5io_swmr Single-Writer Multiple-Reader (SWMR) Mode * * The \ref AQNWB::HDF5::HDF5IO "HDF5IO" I/O backend uses by default SWMR mode while recording data. - * The SWMR mode in HDF5 allows one process to write to an HDF5 file while allowing multiple - * other processes to read from the file concurrently. + * Using SWMR, one process can write to the HDF5 file and multiple other processes can read + * from the file concurrently while ensuring that the readers see a consistent view of the data. + * + * \dot + * digraph SWMR { + * size="4,3"; // Set the size of the diagram to 4 inches wide by 3 inches tall + * node [shape=box, style=filled]; + * + * // Define the writer node with light blue fill color + * writer [label="AqNWB HDF5IO\n(SWMR Write)", fillcolor=lightblue, URL="\ref AQNWB::HDF5::HDF5IO"]; + * // Define the HDF5 file node with a distinct shape + * hdf5_file [label="\n.nwb\n(HDF5 File)\n ", shape=folder, fillcolor=lightgray, URL="\ref AQNWB::NWB::NWBFile"]; + * // Define reader nodes with light green fill color + * reader1 [label="Reader 1\n(SWMR Read)", fillcolor=lightgreen, URL="\ref hdf5io_swmr_read"]; + * reader2 [label="Reader 2\n(SWMR Read)", fillcolor=lightgreen, URL="\ref hdf5io_swmr_read"]; + * reader3 [label="Reader 3\n(SWMR Read)", fillcolor=lightgreen, URL="\ref hdf5io_swmr_read"]; + * + * // Set the rank for the nodes + * { rank=source; writer; } + * { rank=same; hdf5_file; } + * { rank=sink; reader1; reader2; reader3; } + * + * // Define the edges + * writer -> hdf5_file [label="Write"]; + * reader1 -> hdf5_file [label="Read"]; + * reader2 -> hdf5_file [label="Read"]; + * reader3 -> hdf5_file [label="Read"]; + * } + * \enddot * * \subsection hdf5io_swmr_features Why does AqNWB use SMWR mode? * @@ -53,7 +148,7 @@ * Only data that has been completely written and flushed to disk will be readable. To manually * flush data to disk use \ref AQNWB::HDF5::HDF5IO::flush "HDF5IO::flush". * - * \subsection hdf5io_swmr_workflow SWMR Workflow + * \subsection hdf5io_swmr_workflow Writing an NWB file with SWMR mode * * SWMR mode is enabled when calling \ref AQNWB::HDF5::HDF5IO::startRecording "HDF5IO::startRecording". * Once SWMR mode is enabled, no new data objects (Datasets, Groups, Attributes etc.) @@ -93,4 +188,15 @@ * * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_without_SWMR_mode * + * + * \subsection hdf5io_swmr_read Reading with SWMR mode + * + * While the file is being written to in SWMR mode, readers must open the file with + * the ``H5F_ACC_RDONLY`` flag and then enable SWMR read mode using the ``H5Fstart_swmr_read`` + * function, e.g.: + * + * \code{.c} + * hid_t file_id = H5Fopen("example.h5", H5F_ACC_RDONLY, H5P_DEFAULT); + * H5Fstart_swmr_read(file_id); + * \endcode */ From 66cb457e11d64c6d626dc63c91da42f7bd04cca0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 22 Aug 2024 11:18:45 -0700 Subject: [PATCH 19/29] Fix section levels --- docs/pages/userdocs/hdf5io.dox | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pages/userdocs/hdf5io.dox b/docs/pages/userdocs/hdf5io.dox index 26bc3aad..acd7b715 100644 --- a/docs/pages/userdocs/hdf5io.dox +++ b/docs/pages/userdocs/hdf5io.dox @@ -173,18 +173,18 @@ * recording, this comes at the cost of losing the concurrent access and data integrity * features that SWMR mode provides. * - * \subsection hdf5io_swmr_examples Code Examples + * \subsubsection hdf5io_swmr_examples Code Examples * * This code snippet shows all the includes that are being used by the code examples * shown in this section: * * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_includes * - * \subsubsection hdf5io_swmr_examples_with_swmr Workflow with SWMR + * \paragraph hdf5io_swmr_examples_with_swmr Workflow with SWMR * * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_with_SWMR_mode * - * \subsubsection hdf5io_noswmr_examples_without_swmr Workflow with SWMR disabled + * \paragraph hdf5io_noswmr_examples_without_swmr Workflow with SWMR disabled * * \snippet tests/examples/test_HDF5IO_examples.cpp example_HDF5_without_SWMR_mode * From fbac49d7b7322a46db8bb6d42b6873abc9f28042 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Thu, 22 Aug 2024 15:23:22 -0700 Subject: [PATCH 20/29] Add format-fix and spell-fix targets in the docs --- .codespellrc | 2 +- docs/pages/devdocs/testing.dox | 16 ++++++++++++++++ docs/pages/userdocs/install.dox | 6 ++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.codespellrc b/.codespellrc index 8951f4e3..d1d483a5 100644 --- a/.codespellrc +++ b/.codespellrc @@ -2,6 +2,6 @@ builtin = clear,rare,en-GB_to_en-US,names,informal,code check-filenames = check-hidden = -skip = */.git,*/build,*/prefix,*/resources,*/.vscode +skip = */.git,*/build,*/prefix,*/resources,*/.vscode,*/libs quiet-level = 2 ignore-words=./.codespell_ignore diff --git a/docs/pages/devdocs/testing.dox b/docs/pages/devdocs/testing.dox index ce62fd79..2e3c0a04 100644 --- a/docs/pages/devdocs/testing.dox +++ b/docs/pages/devdocs/testing.dox @@ -35,6 +35,17 @@ * cmake --build --preset=dev --target=spell-check * \endcode * + * To automatically fix spelling errors you can run codespell via the `spell-fix` target instead: + * + * \code{.sh} + * cmake --build --preset=dev --target=spell-fix + * \endcode + * + * \note + * ``codespell`` uses a relatively small dictionary of common misspellings and their corrections. For + * more comprehensive spellchecking (in particular of the documentation) additional spellchecking via more + * detailed spellcheckers (e.g., ``aspell``) is recommended. + * * \section testing_lint Linting * * AqNWB uses ``clang-format`` for linting the code. You can run the linter via: @@ -43,4 +54,9 @@ * cmake --build --preset=dev --target=format-check * \endcode * + * To automatically fix formatting errors you can run the linter via the `format-fix` target instead: + * \code{.sh} + * cmake --build --preset=dev --target=format-fix + * \endcode + * */ diff --git a/docs/pages/userdocs/install.dox b/docs/pages/userdocs/install.dox index a1867f0d..655b6fa5 100644 --- a/docs/pages/userdocs/install.dox +++ b/docs/pages/userdocs/install.dox @@ -117,8 +117,10 @@ * \endcode * * \subsubsection devbuild_target_options_subsubsec Target options - * - `format-check` and `format-fix`: run the clang-format tool on the codebase to check errors and to fix them respectively. - * - `spell-check` and `spell-fix`: run the codespell tool on the codebase to check errors and to fix them respectively. + * - `format-check`: run the `clang-format` tool on the codebase to check for formatting errors + * - `format-fix` : run the `clang-format` tool on the codebase with `FIX=YES` to both check and automatically fix for formatting errors + * - `spell-check`: run the `codespell` tool on the codebase to check for common spelling errors + * - `spell-fix` : run the `codespell` tool on the codebase with `FIX=YES` to both check and automatically fix common spelling errors * - `docs` : builds the documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target) */ From da190c27229cd35224f7615b6264b2ff055f7873 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 23 Aug 2024 17:08:29 -0700 Subject: [PATCH 21/29] Apply suggestions from code review Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --- docs/pages/devdocs/testing.dox | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages/devdocs/testing.dox b/docs/pages/devdocs/testing.dox index 2e3c0a04..28244619 100644 --- a/docs/pages/devdocs/testing.dox +++ b/docs/pages/devdocs/testing.dox @@ -8,7 +8,7 @@ * * ``/tests`` : Sources of the unit tests * * ``/tests/examples`` : Sources of example code used in the docs. The examples are implemented like * regular unit tests but their purpose is to provide examples for the docs (rather than testing - * specific cases) and as such commonly include Doxygen section markers to allow include of sections + * specific cases) and as such commonly include Doxygen section markers to allow inclusion of sections * of code in the Doxygen docs. * * @@ -29,7 +29,7 @@ * * \section testing_spellcheck Spellcheck * - * AqNWB uses ``codepsell`` to check the code for spelling errors. You can run the spellchecker via: + * AqNWB uses ``codespell`` to check the code for spelling errors. You can run the spellchecker via: * * \code{.sh} * cmake --build --preset=dev --target=spell-check From c05a2e3a86195b2b400eb10570a0b9cd346bdd7d Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 24 Aug 2024 17:28:54 -0700 Subject: [PATCH 22/29] Separate User and Developer installation docs --- README.md | 15 +++++ docs/Doxyfile.in | 3 + docs/pages/2_devdocs.dox | 2 +- docs/pages/devdocs/install.dox | 97 ++++++++++++++++++++++++++++++ docs/pages/userdocs/install.dox | 103 ++------------------------------ 5 files changed, 121 insertions(+), 99 deletions(-) create mode 100644 docs/pages/devdocs/install.dox diff --git a/README.md b/README.md index 57a7ab3c..e3f37117 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,18 @@ Below is a high-level overview of the project structure and capabilities we are ![Project Overview](resources/images/aqnwb_objective_500px.png) + +## Requirements + +- **Minimum Requirements:** + - A C++17-compliant compiler + - [CMake >= 3.15](https://cmake.org) + - [HDF5 >= 1.10](https://github.com/HDFGroup/hdf5) + - [Boost](https://www.boost.org/) +- **Documentation:** Additional requirements for building the documentation (optional) + - [Doxygen](https://www.doxygen.nl/) + - [Graphviz](https://graphviz.org/) +- **For Developers:** Additional requirements for developers (mode `dev`) + - cppcheck + - clang-format (optional, required for ``target=format-check``, ``target=format-fix``) + - codespell (optional, required for ``target=spell-check``, ``target=spell-fix``) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index cb13342e..23395980 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -14,6 +14,9 @@ EXTRACT_ALL = YES RECURSIVE = YES OUTPUT_DIRECTORY = "@DOXYGEN_OUTPUT_DIRECTORY@" +# Enable Markdown support +MARKDOWN_SUPPORT = YES + # Use the README as a main page USE_MDFILE_AS_MAINPAGE = "@PROJECT_SOURCE_DIR@/README.md" diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox index 30ed6283..357392f2 100644 --- a/docs/pages/2_devdocs.dox +++ b/docs/pages/2_devdocs.dox @@ -3,7 +3,7 @@ * * This documentation is intended for developers of AqNWB. * - * - \ref install_page (part of the \ref userdocs docs) + * - @subpage dev_install_page * - @subpage testing * - @subpage code_of_conduct_page * - @subpage license_page diff --git a/docs/pages/devdocs/install.dox b/docs/pages/devdocs/install.dox new file mode 100644 index 00000000..ac96f398 --- /dev/null +++ b/docs/pages/devdocs/install.dox @@ -0,0 +1,97 @@ +/** + * @page dev_install_page Installation & Setup + * + * \section dev_requirements_sec Requirements + * + * Please ensure that the required libraries described in the + * \ref requirements "Requirements" section are installed. For developers we recommend + * to also install optional dependencies to ease development. + * + * \section devbuild_sec Developer Build + * + * Build system targets that are only useful for developers of AqNWB are + * hidden if the `aq-nwb_DEVELOPER_MODE` option is disabled. Enabling this + * option makes \ref testing "tests" and other developer targets and options available. You can enable + * the option when configuring the build by adding `-Daq-nwb_DEVELOPER_MODE=ON`, e.g., + * + * \code{.sh} + * cmake -S . -B build -Daq-nwb_DEVELOPER_MODE=ON + * \endcode + * + * \note + * If you are using custom installations of **HDF5** or **BOOST** that are not being detected + * automatically by cmake, you can specify `HDF5_ROOT` and `BOOST_ROOT` environment variables to + * point to install directories of HDF5 and BOOST respectively. + * + * \section devbuild_presets_subsec Developer Presets + * + * As a developer, you can create your own dev preset by making a `CMakeUserPresets.json` file at the root of + * the project: + * + * \code{.json} + * { + * "version": 2, + * "cmakeMinimumRequired": { + * "major": 3, + * "minor": 15, + * "patch": 0 + * }, + * "configurePresets": [ + * { + * "name": "dev", + * "binaryDir": "${sourceDir}/build/dev", + * "inherits": ["dev-mode", "ci-"], + * "cacheVariables": { + * "CMAKE_BUILD_TYPE": "Debug" + * } + * } + * ], + * "buildPresets": [ + * { + * "name": "dev", + * "configurePreset": "dev", + * "configuration": "Debug" + * } + * ], + * "testPresets": [ + * { + * "name": "dev", + * "configurePreset": "dev", + * "configuration": "Debug", + * "output": { + * "outputOnFailure": true + * } + * } + * ] + * } + * \endcode + * Replace `` in the `CMakeUserPresets.json` file with the name of + * the operating system you have (`win64`, `linux` or `darwin`). + * + * \subsection configure_build_test Configure, Build and Test + * + * You can configure, build and test the project respectively with the following commands from the project root on + * any operating system with any build system: + * + * \code{.sh} + * cmake --preset=dev + * cmake --build --preset=dev + * ctest --preset=dev + * \endcode + * + * \section devbuild_dev_mode_targets_subsec Developer Mode Targets + * + * Additional targets can be invoked when in development mode using the commands below + * + * \code{.sh} + * cmake --build --preset=dev --target= + * \endcode + * + * \subsubsection devbuild_target_options_subsubsec Target options + * - `format-check`: run the `clang-format` tool on the codebase to check for formatting errors + * - `format-fix` : run the `clang-format` tool on the codebase with `FIX=YES` to both check and automatically fix for formatting errors + * - `spell-check`: run the `codespell` tool on the codebase to check for common spelling errors + * - `spell-fix` : run the `codespell` tool on the codebase with `FIX=YES` to both check and automatically fix common spelling errors + * - `docs` : builds the documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target) + */ + diff --git a/docs/pages/userdocs/install.dox b/docs/pages/userdocs/install.dox index 655b6fa5..8fc41c83 100644 --- a/docs/pages/userdocs/install.dox +++ b/docs/pages/userdocs/install.dox @@ -1,22 +1,12 @@ /** * @page install_page Installing AqNWB * - * \section requirements Requirements - * - A C++17-compliant compiler - * - CMake >= 3.15 - * - HDF5 >= 1.10 - * - Boost - * - Additional requirements for building the documentation (optional) - * - Doxygen - * - Graphviz - * - Additional requirements for developers (mode `dev`) - * - cppcheck - * - clang-format - * - codespell (optional, required for ``target=spell-check``) + * \section user_requirements_sec Requirements * - * \section userbuild_sec User Build + * Please ensure that the required libraries described in the + * \ref requirements "Requirements" section are installed. * - * \subsection userbuild_build_subsec Build + * \section userbuild_build_sec Build * * Here are the steps for building in release mode with a multi-configuration generator: * @@ -30,7 +20,7 @@ * automatically by cmake, you can specify `HDF5_ROOT` and `BOOST_ROOT` environment variables to * point to install directories of HDF5 and BOOST respectively. * - * \subsection userbuild_install_subsec Install + * \section userbuild_install_sec Install * * Here is the command for installing the release mode artifacts with a * multi-configuration generator: @@ -39,89 +29,6 @@ * cmake --install build --config Release * \endcode * - * - * - * \section devbuild_sec Developer Build - * - * Build system targets that are only useful for developers of AqNWB are - * hidden if the `aq-nwb_DEVELOPER_MODE` option is disabled. Enabling this - * option makes tests and other developer targets and options available. You can enable - * the option when configuring the build by adding `-Daq-nwb_DEVELOPER_MODE=ON`, e.g., - * - * \code{.sh} - * cmake -S . -B build -Daq-nwb_DEVELOPER_MODE=ON - * \endcode - * - * \subsection devbuild_presets_subsec Presets - * - * As a developer, you can create your own dev preset by making a `CMakeUserPresets.json` file at the root of - * the project: - * - * \code{.json} - * { - * "version": 2, - * "cmakeMinimumRequired": { - * "major": 3, - * "minor": 15, - * "patch": 0 - * }, - * "configurePresets": [ - * { - * "name": "dev", - * "binaryDir": "${sourceDir}/build/dev", - * "inherits": ["dev-mode", "ci-"], - * "cacheVariables": { - * "CMAKE_BUILD_TYPE": "Debug" - * } - * } - * ], - * "buildPresets": [ - * { - * "name": "dev", - * "configurePreset": "dev", - * "configuration": "Debug" - * } - * ], - * "testPresets": [ - * { - * "name": "dev", - * "configurePreset": "dev", - * "configuration": "Debug", - * "output": { - * "outputOnFailure": true - * } - * } - * ] - * } - * \endcode - * Replace `` in the `CMakeUserPresets.json` file with the name of - * the operating system you have (`win64`, `linux` or `darwin`). - * - * \subsection configure_build_test Configure, build and test - * - * You can configure, build and test the project respectively with the following commands from the project root on - * any operating system with any build system: - * - * \code{.sh} - * cmake --preset=dev - * cmake --build --preset=dev - * ctest --preset=dev - * \endcode - * - * \subsection devbuild_dev_mode_targets_subsec Developer mode targets - * - * Additional targets can be invoked when in development mode using the commands below - * - * \code{.sh} - * cmake --build --preset=dev --target= - * \endcode - * - * \subsubsection devbuild_target_options_subsubsec Target options - * - `format-check`: run the `clang-format` tool on the codebase to check for formatting errors - * - `format-fix` : run the `clang-format` tool on the codebase with `FIX=YES` to both check and automatically fix for formatting errors - * - `spell-check`: run the `codespell` tool on the codebase to check for common spelling errors - * - `spell-fix` : run the `codespell` tool on the codebase with `FIX=YES` to both check and automatically fix common spelling errors - * - `docs` : builds the documentation using Doxygen. (Note: run `cmake --preset=dev -DBUILD_DOCS=ON` before building to add docs target) */ From 686a1a3fada6c257a1a781782a5ff2e23fcc9677 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 24 Aug 2024 17:32:25 -0700 Subject: [PATCH 23/29] Apply suggestions from code review Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --- README.md | 2 +- tests/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index e3f37117..5603a557 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ AqNWB is a C++ API for acquiring neurophysiological data directly into the NWB (Neurodata Without Borders) format. Our goal is to provide a lightweight API to integrate with existing acquisition systems. -Please see the [Documentation](https://neurodatawithoutborders.github.io/aqnwb/) for +Please see the [AqNWB Documentation](https://neurodatawithoutborders.github.io/aqnwb/) for installation instructions and to learn more about AqNWB. Below is a high-level overview of the project structure and capabilities we are targeting: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9d4f8e70..08c30bf2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,7 +24,6 @@ add_executable(aq-nwb_test # Ensure the aq-nwb_test target can include headers from the current directory and Catch2 target_include_directories(aq-nwb_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} # Include current directory - # ${CATCH2_INCLUDE_DIR} # Include Catch2 headers, if found by find_package ) target_link_libraries( From e8cf56d09775d81b7fcc8cac794a7181fe8b5e24 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 24 Aug 2024 17:33:24 -0700 Subject: [PATCH 24/29] Update docs/pages/2_devdocs.dox Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --- docs/pages/2_devdocs.dox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox index 357392f2..415903d3 100644 --- a/docs/pages/2_devdocs.dox +++ b/docs/pages/2_devdocs.dox @@ -1,7 +1,7 @@ /** * @page devdocs For Developers * - * This documentation is intended for developers of AqNWB. + * This documentation is intended for developers of AqNWB. * * - @subpage dev_install_page * - @subpage testing From ae3dd83ad4863c0d87477e64d2536fe30d277be6 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 24 Aug 2024 17:44:06 -0700 Subject: [PATCH 25/29] Add links to the installation docs on the README.md page --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 5603a557..e29a5f1f 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,9 @@ Below is a high-level overview of the project structure and capabilities we are - cppcheck - clang-format (optional, required for ``target=format-check``, ``target=format-fix``) - codespell (optional, required for ``target=spell-check``, ``target=spell-fix``) + +## Installation + +See the [AqNWB Documentation](https://neurodatawithoutborders.github.io/aqnwb/) for installation and integration instructions. +* [User Installation](https://neurodatawithoutborders.github.io/aqnwb/install_page.html) +* [Developer Installation](https://neurodatawithoutborders.github.io/aqnwb/dev_install_page.html) From d660b60cf4e829147663bc48f70f856c5d111c14 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 24 Aug 2024 19:47:39 -0700 Subject: [PATCH 26/29] Add documentation on how to write Documentation in Doxygen --- docs/pages/2_devdocs.dox | 1 + docs/pages/devdocs/documentation.dox | 117 +++++++++++++++++++++++++++ tests/CMakeLists.txt | 3 +- tests/examples/test_example.cpp | 18 +++++ 4 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 docs/pages/devdocs/documentation.dox create mode 100644 tests/examples/test_example.cpp diff --git a/docs/pages/2_devdocs.dox b/docs/pages/2_devdocs.dox index 415903d3..7b868d38 100644 --- a/docs/pages/2_devdocs.dox +++ b/docs/pages/2_devdocs.dox @@ -5,6 +5,7 @@ * * - @subpage dev_install_page * - @subpage testing + * - @subpage dev_docs_page * - @subpage code_of_conduct_page * - @subpage license_page * - @subpage copyright_page diff --git a/docs/pages/devdocs/documentation.dox b/docs/pages/devdocs/documentation.dox new file mode 100644 index 00000000..a86453d8 --- /dev/null +++ b/docs/pages/devdocs/documentation.dox @@ -0,0 +1,117 @@ +/** + * @page dev_docs_page Documentation + * + * AqNWB uses [Doxygen](https://www.doxygen.nl/) with [Graphviz](https://graphviz.org/) for + * documentation. The ``cmake`` related instructions on this page follow the + * \ref devbuild_presets_subsec "Developer Presets" installation instructions using the ``dev`` preset. + * + * \section dev_docs_requirements_sec Building the Docs + * + * 1. Ensure that [Doxygen](https://www.doxygen.nl/) and [Graphviz](https://graphviz.org/) are installed + * + * 2. Enable building the docs by setting the cmake variable ``BUILD_DOCS=ON``, e.g., + * by calling ``cmake --preset=dev -DBUILD_DOCS=ON`` + * + * 3. Build the documentation by running: ``cmake --build --preset=dev --target=docs`` + * + * \note + * To edit settings in cmake (e.g., to enable ``BUILD_DOCS=ON``) after the initial configuration, + * you can also use the ``ccmake`` command line UI by navigating to your build directory (e.g., + * ``cd build/dev``) and running ``ccmake ../../`` (with the path pointing to the aqnwb root directory). + * + * \section dev_docs_creating Creating New Documentation Pages + * + * To create a new page as part of the \ref userdocs docs, simply create a new ``.dox`` file in the + * ``docs/pages/userdocs`` folder, e.g.: + * + * \par **docs/pages/userdocs/mypage.dox** + * \code{.sh} + * /** + * * @page mypage My New Page + * * + * */ + * \endcode + * + * and add a corresponding ``- @subpage mypage`` entry to the page listing of the ``docs/pages/1_userdocs.dox`` + * file. Note, the ``subpage`` command uses the label of the page (here ``mypage``) and not the name of the file. + * + * Similarly, we can add pages to the \ref devdocs docs by adding a new ``.dox`` file to the ``docs/pages/devdocs`` + * and including it as as subpage in ``docs/pages/2_devdocs.dox``. + * + * \section dev_docs_codeexamples_sec Creating Code Examples + * + * Code examples are defined as part of the \ref testing "unit tests" to ensure code examples used in the + * documentation are tested and work as expected. + * + * \subsection dev_docs_codeexamples_def_sec Creating the Example Code + * + * First we need to define our code example by creating a new ``cpp`` file + * in the ``tests/examples`` folder. Here we use the file ``test_examples.cpp`` shown below. + * The file looks like a regular unit test with the addition of Doxygen comments of the form + * `// [example_hdf5io_code_snippet]`` to mark regions in the code that we want to include + * in the docs as code sinppets. + * + * \par test_example.cpp + * \snippet tests/examples/test_example.cpp example_all + * + * \subsection dev_docs_codeexamples_run_sec Testing the Example Code + * + * To make sure our example code builds and runs correctly, we add it to the unit test suite. + * 1. Add the new ``text_examples.cpp`` file to the ``add_executable`` section of the ``tests/CMakeLists.txt`` file + * 2. Rebuild the code as usual with ``BUILD_TESTING=ON`` set, e.g., via ``cmake --build --preset=dev`` + * 3. Run the test suite to ensure our example is working, e.g, via ``ctest --preset=dev`` + * + * See the \ref testing section for more details about how to build and run the test suite. + * + * \subsection dev_docs_codeexamples_incl_sec Including Code Examples in Doxygen + * + * To display a code snippet from our example in the documentation we can use the + * ``\snippet