From 7975c0487998b77d23ab9618c2c24dabcdfb1282 Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso <41900536+jorblancoa@users.noreply.github.com> Date: Fri, 28 Feb 2020 16:12:52 +0100 Subject: [PATCH] Adapt to SONATA specification * Change report data datatype from double to float * Add time dataset (tstart, tstop, dt) * Add missing attributes (units) --- src/reports/data/node.cpp | 6 ++- src/reports/data/node.h | 2 +- src/reports/data/sonata_data.cpp | 9 +++- src/reports/data/sonata_data.h | 5 +- src/reports/io/hdf5_writer.cpp | 44 ++++++++++++++++-- src/reports/io/hdf5_writer.h | 8 +++- .../integration/compartment_report.ref | Bin 16936 -> 21056 bytes .../integration/compartment_report_serial.ref | Bin 7360 -> 11808 bytes tests/reports/integration/soma_report.ref | Bin 7360 -> 10784 bytes .../integration/soma_report_serial.ref | Bin 7360 -> 10400 bytes tests/reports/integration/spikes.ref | Bin 8624 -> 12720 bytes tests/reports/integration/spikes_serial.ref | Bin 5896 -> 11440 bytes tests/reports/unit/test_node.cpp | 10 ++-- tests/reports/unit/test_sonatadata.cpp | 6 +-- 14 files changed, 70 insertions(+), 20 deletions(-) diff --git a/src/reports/data/node.cpp b/src/reports/data/node.cpp index d7636b6..69cc7ed 100644 --- a/src/reports/data/node.cpp +++ b/src/reports/data/node.cpp @@ -13,8 +13,10 @@ void Node::add_element(double* element_value, uint32_t element_id) { element_ids_.push_back(element_id); } -void Node::fill_data(std::vector::iterator it) { - std::transform(elements_.begin(), elements_.end(), it, [](auto elem) { return *elem; }); +void Node::fill_data(std::vector::iterator it) { + std::transform(elements_.begin(), elements_.end(), it, [](auto elem) { + return static_cast(*elem); + }); } void Node::refresh_pointers(std::function refresh_function) { diff --git a/src/reports/data/node.h b/src/reports/data/node.h index b3b6808..9d1d317 100644 --- a/src/reports/data/node.h +++ b/src/reports/data/node.h @@ -13,7 +13,7 @@ class Node public: explicit Node(uint64_t node_id); virtual ~Node() = default; - void fill_data(std::vector::iterator it); + void fill_data(std::vector::iterator it); void refresh_pointers(std::function refresh_function); virtual void add_element(double* element_value, uint32_t element_id); diff --git a/src/reports/data/sonata_data.cpp b/src/reports/data/sonata_data.cpp index c216040..28ff247 100644 --- a/src/reports/data/sonata_data.cpp +++ b/src/reports/data/sonata_data.cpp @@ -24,6 +24,7 @@ SonataData::SonataData(const std::string& report_name, , nodes_(nodes) { prepare_buffer(max_buffer_size); index_pointers_.resize(nodes->size()); + time_ = {tstart, tend, dt}; reporting_period_ = static_cast(dt / SonataReport::atomic_step_); last_step_recorded_ = tstart / SonataReport::atomic_step_; @@ -54,7 +55,7 @@ void SonataData::prepare_buffer(size_t max_buffer_size) { // Calculate the timesteps that fit given the buffer size { - uint32_t max_steps_to_write = max_buffer_size / (sizeof(double) * total_elements_); + uint32_t max_steps_to_write = max_buffer_size / (sizeof(float) * total_elements_); uint32_t common_max_steps_to_write = Implementation::get_max_steps_to_write(report_name_, max_steps_to_write); if (common_max_steps_to_write < num_steps_) { // More steps asked that buffer can contain @@ -231,10 +232,13 @@ void SonataData::write_report_header() { hdf5_writer_->configure_dataset(reports_population_group + "/data", num_steps_, total_elements_); + hdf5_writer_->configure_attribute(reports_population_group + "/data", "units", "mV"); hdf5_writer_->write(reports_population_group + "/mapping/node_ids", node_ids_); hdf5_writer_->write(reports_population_group + "/mapping/index_pointers", index_pointers_); hdf5_writer_->write(reports_population_group + "/mapping/element_ids", element_ids_); + hdf5_writer_->write_time(reports_population_group + "/mapping/time", time_); + hdf5_writer_->configure_attribute(reports_population_group + "/mapping/time", "units", "ms"); } void SonataData::write_spikes_header(const std::string& order_by) { @@ -246,12 +250,13 @@ void SonataData::write_spikes_header(const std::string& order_by) { const std::string spikes_population_group = "/spikes/" + population_name_; hdf5_writer_->configure_group("/spikes"); hdf5_writer_->configure_group(spikes_population_group); - hdf5_writer_->configure_attribute(spikes_population_group, "sorting", order_by); + hdf5_writer_->configure_enum_attribute(spikes_population_group, "sorting", order_by); hsize_t timestamps_size = Implementation::get_global_dims(report_name_, spike_timestamps_.size()); if (timestamps_size > 0) { Implementation::sort_spikes(spike_timestamps_, spike_node_ids_, order_by); hdf5_writer_->write(spikes_population_group + "/timestamps", spike_timestamps_); + hdf5_writer_->configure_attribute(spikes_population_group + "/timestamps", "units", "ms"); hdf5_writer_->write(spikes_population_group + "/node_ids", spike_node_ids_); } } diff --git a/src/reports/data/sonata_data.h b/src/reports/data/sonata_data.h index 0fe07c2..7e2552a 100644 --- a/src/reports/data/sonata_data.h +++ b/src/reports/data/sonata_data.h @@ -35,7 +35,7 @@ class SonataData void record_data(double step); void check_and_write(double timestep); - const std::vector& get_report_buffer() const noexcept { + const std::vector& get_report_buffer() const noexcept { return report_buffer_; } @@ -58,7 +58,7 @@ class SonataData private: std::string report_name_; std::string population_name_; - std::vector report_buffer_; + std::vector report_buffer_; uint32_t total_elements_ = 0; uint32_t num_steps_ = 0; uint32_t steps_to_write_ = 0; @@ -73,6 +73,7 @@ class SonataData std::vector node_ids_; std::vector index_pointers_; std::vector element_ids_; + std::array time_; std::vector spike_timestamps_; std::vector spike_node_ids_; diff --git a/src/reports/io/hdf5_writer.cpp b/src/reports/io/hdf5_writer.cpp index bd28b6f..3d00fe7 100644 --- a/src/reports/io/hdf5_writer.cpp +++ b/src/reports/io/hdf5_writer.cpp @@ -48,9 +48,31 @@ void HDF5Writer::configure_group(const std::string& group_name) { H5Gclose(group); } -void HDF5Writer::configure_attribute(const std::string& group_name, +void HDF5Writer::configure_attribute(const std::string& dataset_name, const std::string& attribute_name, const std::string& attribute_value) { + logger->trace("Configuring attribute '{}' for group name '{}' with value {}", + attribute_name, + dataset_name, + attribute_value); + hid_t dataset_id = H5Dopen(file_, dataset_name.data(), H5P_DEFAULT); + hid_t attr_space = H5Screate(H5S_SCALAR); + hid_t type = H5Tcopy(H5T_C_S1); + H5Tset_size(type, H5T_VARIABLE); + H5Tset_cset(type, H5T_CSET_UTF8); + const char* value[] = {attribute_value.data()}; + hid_t attr_id = + H5Acreate2(dataset_id, attribute_name.data(), type, attr_space, H5P_DEFAULT, H5P_DEFAULT); + H5Awrite(attr_id, type, value); + + H5Aclose(attr_id); + H5Sclose(attr_space); + H5Dclose(dataset_id); +} + +void HDF5Writer::configure_enum_attribute(const std::string& group_name, + const std::string& attribute_name, + const std::string& attribute_value) { logger->trace("Configuring attribute '{}' for group name '{}'", attribute_name, group_name); hid_t group_id = H5Gopen(file_, group_name.data(), H5P_DEFAULT); hid_t attr_space = H5Screate(H5S_SCALAR); @@ -86,7 +108,7 @@ void HDF5Writer::configure_dataset(const std::string& dataset_name, H5Sclose(data_space); } -void HDF5Writer::write_2D(const std::vector& buffer, +void HDF5Writer::write_2D(const std::vector& buffer, uint32_t steps_to_write, uint32_t total_elements) { std::array count; @@ -103,7 +125,7 @@ void HDF5Writer::write_2D(const std::vector& buffer, H5Sselect_hyperslab(space, H5S_SELECT_OR, &offset_[i], NULL, &count[i], NULL); }*/ - H5Dwrite(dataset_, H5T_NATIVE_DOUBLE, memspace, filespace, collective_list_, buffer.data()); + H5Dwrite(dataset_, H5T_NATIVE_FLOAT, memspace, filespace, collective_list_, buffer.data()); offset_[0] += steps_to_write; H5Sclose(filespace); @@ -134,6 +156,22 @@ void HDF5Writer::write(const std::string& dataset_name, const std::vector& bu H5Sclose(data_space); } +void HDF5Writer::write_time(const std::string& dataset_name, const std::array& buffer) { + hsize_t dims = buffer.size(); + hid_t data_space = H5Screate_simple(1, &dims, nullptr); + hid_t data_set = H5Dcreate(file_, + dataset_name.c_str(), + H5T_NATIVE_DOUBLE, + data_space, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT); + + H5Dwrite(data_set, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer.data()); + H5Dclose(data_set); + H5Sclose(data_space); +} + void HDF5Writer::close() { // We close the dataset "/data", the spikes enum type and the hdf5 file if (dataset_) { diff --git a/src/reports/io/hdf5_writer.h b/src/reports/io/hdf5_writer.h index 5bcd44f..40375db 100644 --- a/src/reports/io/hdf5_writer.h +++ b/src/reports/io/hdf5_writer.h @@ -14,17 +14,21 @@ class HDF5Writer public: explicit HDF5Writer(const std::string& report_name); void configure_group(const std::string& group_name); - void configure_attribute(const std::string& group_name, + void configure_attribute(const std::string& dataset_name, const std::string& attribute_name, const std::string& attribute_value); + void configure_enum_attribute(const std::string& group_name, + const std::string& attribute_name, + const std::string& attribute_value); void configure_dataset(const std::string& dataset_name, uint32_t total_steps, uint32_t total_elements); - void write_2D(const std::vector& buffer, + void write_2D(const std::vector& buffer, uint32_t steps_to_write, uint32_t total_elements); template void write(const std::string& name, const std::vector& buffer); + void write_time(const std::string& dataset_name, const std::array& buffer); void close(); private: diff --git a/tests/reports/integration/compartment_report.ref b/tests/reports/integration/compartment_report.ref index 8786ce9fd164ab8b7e8b47fbf8a1c774ea9d4b97..8479439d4e081b055e76fda8f5884d9fec09fcbd 100644 GIT binary patch delta 552 zcmZ3{!gyc_;{**RhoFsG{@ff63}9ffaU(kuqr>D*UU$Kg%-mFvAR|PIL1Obm-bhA9 z_Q}@#GyNk}Bjb1&9KceH3~US{3>*xld6^}}KoLnsMg{>O4Z;x398h&&ih&78ap*(X z5P5}-8~YhIZxRfX^>lak_W=unbOQm*c913z$PI(gFg}dV1=)-Q4oqg`liw_0{DT>V z*uXqF!7z!5!+mm&z5+AQ7nA$-`*~OxG@x1woF;CR&tU#1VJ=NFo2AsTBt(;&0sU5@mi+M9Oeq#Fb^{rgTZC{Su;Uk Gr#%4t%Tzu9 delta 177 zcmX@GgmFa+;{**R4X2G-{@jcP8&~l$FQCJG45%}i5m{sBF`_$8o5c5W`v?_l1{;jX|90DsXk ATmS$7 diff --git a/tests/reports/integration/compartment_report_serial.ref b/tests/reports/integration/compartment_report_serial.ref index c805979dd14b710da9d62eccf2734591ff1c5c5f..ed4fc4c779b8499120195e5e3c6df42a4031c7f1 100644 GIT binary patch delta 565 zcmX?LxgchO29tu`MlC;X&IzgvU|_LvBRdnL!{kn0cfpd(+*FVtBSeZpV)H}ZNJd8X z$=3Wc{hzBu#_=#XfTb82*ce0@I2cOvGE0hqB9e@Z3<5wJgdv(apz6RB0~3&H5P`5E z@(LR__A_qYBzTU=)7{zM2P_EE4FoXTL7G4yHw;3<_%J#bWHS;tFqx50ezSs@1`~c_ z;(>@6Xy%?vdGK}&!=EvR4* z3{wzrU@+lkF!|5SFu7Yoo{K{dqCtWIWE|BZ0uqh}n+3(pnI;}6+uR^Bfl zJ$hW@7#K1(ZoJO8d6VEdrp*mH6Bsw&7O`NO+@O-oC^FebU4a>>XmY%I=i~>hY&^^i u8VnGyL3-jwxyc7qIhYnmfmjNB984Y(6E{8s>P%vqfUNZm&>$ZTxIqB0!!s}d diff --git a/tests/reports/integration/soma_report.ref b/tests/reports/integration/soma_report.ref index 5b605d83f95cf4853c06dd0bea91893976770709..1368becaed7da7a1a3ca74d8e4c710c8814d9917 100644 GIT binary patch delta 538 zcmX?LxgcbM29tu;MlC;XP6j0gFtFITk)4UrVR9$0yI@IXZYoHS5hBGPvH2lyBqJmH zWNZGJ{t>E?aXbtTU@1lhHU<#}4u;aa%#vcDh$JH;g8+~QVTfi9s5&skzyzc|fFywc zBCjxUqyFR|fq96;bMgl1WDZS;8jH!h`6ZzibTTtCuut}ukkw{}8i1fBzze2GaqVjaq)(Oc9bBSMf11T1@Wbb!TQ|V3;f`AiMbkuNUKFFJ*zrUi@9` z)!nXf3=A0)H diff --git a/tests/reports/integration/soma_report_serial.ref b/tests/reports/integration/soma_report_serial.ref index 3e42558ed1139370a35eba23d95c8d334cc83ce3..8e3cf083e46f501ff9443de2c74979f2a802dd04 100644 GIT binary patch delta 535 zcmX?LxgcUKn5qyxLAdm}k0TMVcnUPO^^8$et z>|kQz0guTGq>`8tG$yZ+P+$gnWAb^4ejXME4XBDP%}LVDm(QwM>%}1Pla>!Qis}teGIN G(;fiV*jJSR delta 173 zcmZ1wc))Uk2GaqVjaq)(Oc9bBSMf11T1@Wbb!TQ|V3;f`AiMbkuNULwT}lFzz4*J> zJ$hW@7#K1pZY<|(uxDTZVFm{fo}8dzuz9z@4))0j3dxKDlVy|@n1PBX+befYe!$Ac y!_1%oR0aYYq$h5ao4i1ggK2>jh^4^C!Q>$^apNbCZpm;tqShRzeEitU*e8pui4-i$G}iAabxY~4Lqwk8D%E(@F*|?MK{-r L8L} res; + std::vector res; node.fill_data(res.begin()); REQUIRE(res.empty()); @@ -41,14 +41,14 @@ SCENARIO("Test Node class", "[Node]") { REQUIRE(node.get_element_ids() == compare); } THEN("fill_data will return something correct") { - std::vector result(5, -1.); + std::vector result(5, -1.); node.fill_data(result.begin()); - REQUIRE(result == elements); + REQUIRE(result == std::vector(elements.begin(), elements.end())); } THEN("refresh_pointers will be call on all elements") { node.refresh_pointers(&square); - std::vector compare{100, 121, 144, 169, 196}; - std::vector result(5, -1); + std::vector compare{100, 121, 144, 169, 196}; + std::vector result(5, -1); node.fill_data(result.begin()); REQUIRE(result == compare); } diff --git a/tests/reports/unit/test_sonatadata.cpp b/tests/reports/unit/test_sonatadata.cpp index 52a6c4a..0bc5e97 100644 --- a/tests/reports/unit/test_sonatadata.cpp +++ b/tests/reports/unit/test_sonatadata.cpp @@ -61,7 +61,7 @@ SCENARIO("Test SonataData class", "[SonataData][IOWriter]") { THEN( "The buffer size is the total number of steps times the total number of elements") { - // 1024 / sizeof(double) / 11 = 11.6 > 3 (total number of steps) + // 1024 / (sizeof(float) * 11) = 23.27 > 3 (total number of steps) // buffer_size = 11 * 3 REQUIRE(sonata->get_report_buffer().size() == 33); } @@ -88,7 +88,7 @@ SCENARIO("Test SonataData class", "[SonataData][IOWriter]") { std::shared_ptr nodes = std::make_shared( std::initializer_list{{1, node}, {2, node2}, {42, node42}}); int num_steps = 3; - size_t max_buffer_size = 256; + size_t max_buffer_size = 128; std::unique_ptr sonata2 = std::make_unique("test_sonatadata2", population_name, max_buffer_size, @@ -108,7 +108,7 @@ SCENARIO("Test SonataData class", "[SonataData][IOWriter]") { THEN( "The buffer size is the number of steps to write that fit on the buffer times the " "total elements") { - // 256 / sizeof(double) / 11 = 2 + // 128 / (sizeof(float) * 11) = 2 < 3 (total number of steps) // buffer_size = 11 * 2 REQUIRE(sonata2->get_report_buffer().size() == 22); }