Skip to content

Commit

Permalink
Adapt to SONATA specification
Browse files Browse the repository at this point in the history
 * Change report data datatype from double to float
 * Add time dataset (tstart, tstop, dt)
 * Add missing attributes (units)
  • Loading branch information
jorblancoa authored and Nicolas Cornu committed Feb 1, 2021
1 parent 0558bff commit 7975c04
Show file tree
Hide file tree
Showing 14 changed files with 70 additions and 20 deletions.
6 changes: 4 additions & 2 deletions src/reports/data/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ void Node::add_element(double* element_value, uint32_t element_id) {
element_ids_.push_back(element_id);
}

void Node::fill_data(std::vector<double>::iterator it) {
std::transform(elements_.begin(), elements_.end(), it, [](auto elem) { return *elem; });
void Node::fill_data(std::vector<float>::iterator it) {
std::transform(elements_.begin(), elements_.end(), it, [](auto elem) {
return static_cast<float>(*elem);
});
}

void Node::refresh_pointers(std::function<double*(double*)> refresh_function) {
Expand Down
2 changes: 1 addition & 1 deletion src/reports/data/node.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class Node
public:
explicit Node(uint64_t node_id);
virtual ~Node() = default;
void fill_data(std::vector<double>::iterator it);
void fill_data(std::vector<float>::iterator it);
void refresh_pointers(std::function<double*(double*)> refresh_function);
virtual void add_element(double* element_value, uint32_t element_id);

Expand Down
9 changes: 7 additions & 2 deletions src/reports/data/sonata_data.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ SonataData::SonataData(const std::string& report_name,
, nodes_(nodes) {
prepare_buffer(max_buffer_size);
index_pointers_.resize(nodes->size());
time_ = {tstart, tend, dt};

reporting_period_ = static_cast<int>(dt / SonataReport::atomic_step_);
last_step_recorded_ = tstart / SonataReport::atomic_step_;
Expand Down Expand Up @@ -54,7 +55,7 @@ void SonataData::prepare_buffer(size_t max_buffer_size) {

// Calculate the timesteps that fit given the buffer size
{
uint32_t max_steps_to_write = max_buffer_size / (sizeof(double) * total_elements_);
uint32_t max_steps_to_write = max_buffer_size / (sizeof(float) * total_elements_);
uint32_t common_max_steps_to_write =
Implementation::get_max_steps_to_write(report_name_, max_steps_to_write);
if (common_max_steps_to_write < num_steps_) { // More steps asked that buffer can contain
Expand Down Expand Up @@ -231,10 +232,13 @@ void SonataData::write_report_header() {
hdf5_writer_->configure_dataset(reports_population_group + "/data",
num_steps_,
total_elements_);
hdf5_writer_->configure_attribute(reports_population_group + "/data", "units", "mV");

hdf5_writer_->write(reports_population_group + "/mapping/node_ids", node_ids_);
hdf5_writer_->write(reports_population_group + "/mapping/index_pointers", index_pointers_);
hdf5_writer_->write(reports_population_group + "/mapping/element_ids", element_ids_);
hdf5_writer_->write_time(reports_population_group + "/mapping/time", time_);
hdf5_writer_->configure_attribute(reports_population_group + "/mapping/time", "units", "ms");
}

void SonataData::write_spikes_header(const std::string& order_by) {
Expand All @@ -246,12 +250,13 @@ void SonataData::write_spikes_header(const std::string& order_by) {
const std::string spikes_population_group = "/spikes/" + population_name_;
hdf5_writer_->configure_group("/spikes");
hdf5_writer_->configure_group(spikes_population_group);
hdf5_writer_->configure_attribute(spikes_population_group, "sorting", order_by);
hdf5_writer_->configure_enum_attribute(spikes_population_group, "sorting", order_by);
hsize_t timestamps_size = Implementation::get_global_dims(report_name_,
spike_timestamps_.size());
if (timestamps_size > 0) {
Implementation::sort_spikes(spike_timestamps_, spike_node_ids_, order_by);
hdf5_writer_->write(spikes_population_group + "/timestamps", spike_timestamps_);
hdf5_writer_->configure_attribute(spikes_population_group + "/timestamps", "units", "ms");
hdf5_writer_->write(spikes_population_group + "/node_ids", spike_node_ids_);
}
}
Expand Down
5 changes: 3 additions & 2 deletions src/reports/data/sonata_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class SonataData
void record_data(double step);
void check_and_write(double timestep);

const std::vector<double>& get_report_buffer() const noexcept {
const std::vector<float>& get_report_buffer() const noexcept {
return report_buffer_;
}

Expand All @@ -58,7 +58,7 @@ class SonataData
private:
std::string report_name_;
std::string population_name_;
std::vector<double> report_buffer_;
std::vector<float> report_buffer_;
uint32_t total_elements_ = 0;
uint32_t num_steps_ = 0;
uint32_t steps_to_write_ = 0;
Expand All @@ -73,6 +73,7 @@ class SonataData
std::vector<uint64_t> node_ids_;
std::vector<uint64_t> index_pointers_;
std::vector<uint32_t> element_ids_;
std::array<double, 3> time_;

std::vector<double> spike_timestamps_;
std::vector<uint64_t> spike_node_ids_;
Expand Down
44 changes: 41 additions & 3 deletions src/reports/io/hdf5_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,31 @@ void HDF5Writer::configure_group(const std::string& group_name) {
H5Gclose(group);
}

void HDF5Writer::configure_attribute(const std::string& group_name,
void HDF5Writer::configure_attribute(const std::string& dataset_name,
const std::string& attribute_name,
const std::string& attribute_value) {
logger->trace("Configuring attribute '{}' for group name '{}' with value {}",
attribute_name,
dataset_name,
attribute_value);
hid_t dataset_id = H5Dopen(file_, dataset_name.data(), H5P_DEFAULT);
hid_t attr_space = H5Screate(H5S_SCALAR);
hid_t type = H5Tcopy(H5T_C_S1);
H5Tset_size(type, H5T_VARIABLE);
H5Tset_cset(type, H5T_CSET_UTF8);
const char* value[] = {attribute_value.data()};
hid_t attr_id =
H5Acreate2(dataset_id, attribute_name.data(), type, attr_space, H5P_DEFAULT, H5P_DEFAULT);
H5Awrite(attr_id, type, value);

H5Aclose(attr_id);
H5Sclose(attr_space);
H5Dclose(dataset_id);
}

void HDF5Writer::configure_enum_attribute(const std::string& group_name,
const std::string& attribute_name,
const std::string& attribute_value) {
logger->trace("Configuring attribute '{}' for group name '{}'", attribute_name, group_name);
hid_t group_id = H5Gopen(file_, group_name.data(), H5P_DEFAULT);
hid_t attr_space = H5Screate(H5S_SCALAR);
Expand Down Expand Up @@ -86,7 +108,7 @@ void HDF5Writer::configure_dataset(const std::string& dataset_name,
H5Sclose(data_space);
}

void HDF5Writer::write_2D(const std::vector<double>& buffer,
void HDF5Writer::write_2D(const std::vector<float>& buffer,
uint32_t steps_to_write,
uint32_t total_elements) {
std::array<hsize_t, 2> count;
Expand All @@ -103,7 +125,7 @@ void HDF5Writer::write_2D(const std::vector<double>& buffer,
H5Sselect_hyperslab(space, H5S_SELECT_OR, &offset_[i], NULL, &count[i], NULL);
}*/

H5Dwrite(dataset_, H5T_NATIVE_DOUBLE, memspace, filespace, collective_list_, buffer.data());
H5Dwrite(dataset_, H5T_NATIVE_FLOAT, memspace, filespace, collective_list_, buffer.data());
offset_[0] += steps_to_write;

H5Sclose(filespace);
Expand Down Expand Up @@ -134,6 +156,22 @@ void HDF5Writer::write(const std::string& dataset_name, const std::vector<T>& bu
H5Sclose(data_space);
}

void HDF5Writer::write_time(const std::string& dataset_name, const std::array<double, 3>& buffer) {
hsize_t dims = buffer.size();
hid_t data_space = H5Screate_simple(1, &dims, nullptr);
hid_t data_set = H5Dcreate(file_,
dataset_name.c_str(),
H5T_NATIVE_DOUBLE,
data_space,
H5P_DEFAULT,
H5P_DEFAULT,
H5P_DEFAULT);

H5Dwrite(data_set, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, buffer.data());
H5Dclose(data_set);
H5Sclose(data_space);
}

void HDF5Writer::close() {
// We close the dataset "/data", the spikes enum type and the hdf5 file
if (dataset_) {
Expand Down
8 changes: 6 additions & 2 deletions src/reports/io/hdf5_writer.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,21 @@ class HDF5Writer
public:
explicit HDF5Writer(const std::string& report_name);
void configure_group(const std::string& group_name);
void configure_attribute(const std::string& group_name,
void configure_attribute(const std::string& dataset_name,
const std::string& attribute_name,
const std::string& attribute_value);
void configure_enum_attribute(const std::string& group_name,
const std::string& attribute_name,
const std::string& attribute_value);
void configure_dataset(const std::string& dataset_name,
uint32_t total_steps,
uint32_t total_elements);
void write_2D(const std::vector<double>& buffer,
void write_2D(const std::vector<float>& buffer,
uint32_t steps_to_write,
uint32_t total_elements);
template <typename T>
void write(const std::string& name, const std::vector<T>& buffer);
void write_time(const std::string& dataset_name, const std::array<double, 3>& buffer);
void close();

private:
Expand Down
Binary file modified tests/reports/integration/compartment_report.ref
Binary file not shown.
Binary file modified tests/reports/integration/compartment_report_serial.ref
Binary file not shown.
Binary file modified tests/reports/integration/soma_report.ref
Binary file not shown.
Binary file modified tests/reports/integration/soma_report_serial.ref
Binary file not shown.
Binary file modified tests/reports/integration/spikes.ref
Binary file not shown.
Binary file modified tests/reports/integration/spikes_serial.ref
Binary file not shown.
10 changes: 5 additions & 5 deletions tests/reports/unit/test_node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ SCENARIO("Test Node class", "[Node]") {
REQUIRE(node.get_num_elements() == 0);
REQUIRE(node.get_element_ids().empty());

std::vector<double> res;
std::vector<float> res;
node.fill_data(res.begin());
REQUIRE(res.empty());

Expand All @@ -41,14 +41,14 @@ SCENARIO("Test Node class", "[Node]") {
REQUIRE(node.get_element_ids() == compare);
}
THEN("fill_data will return something correct") {
std::vector<double> result(5, -1.);
std::vector<float> result(5, -1.);
node.fill_data(result.begin());
REQUIRE(result == elements);
REQUIRE(result == std::vector<float>(elements.begin(), elements.end()));
}
THEN("refresh_pointers will be call on all elements") {
node.refresh_pointers(&square);
std::vector<double> compare{100, 121, 144, 169, 196};
std::vector<double> result(5, -1);
std::vector<float> compare{100, 121, 144, 169, 196};
std::vector<float> result(5, -1);
node.fill_data(result.begin());
REQUIRE(result == compare);
}
Expand Down
6 changes: 3 additions & 3 deletions tests/reports/unit/test_sonatadata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ SCENARIO("Test SonataData class", "[SonataData][IOWriter]") {

THEN(
"The buffer size is the total number of steps times the total number of elements") {
// 1024 / sizeof(double) / 11 = 11.6 > 3 (total number of steps)
// 1024 / (sizeof(float) * 11) = 23.27 > 3 (total number of steps)
// buffer_size = 11 * 3
REQUIRE(sonata->get_report_buffer().size() == 33);
}
Expand All @@ -88,7 +88,7 @@ SCENARIO("Test SonataData class", "[SonataData][IOWriter]") {
std::shared_ptr<nodes_t> nodes = std::make_shared<nodes_t>(
std::initializer_list<nodes_t::value_type>{{1, node}, {2, node2}, {42, node42}});
int num_steps = 3;
size_t max_buffer_size = 256;
size_t max_buffer_size = 128;
std::unique_ptr<SonataData> sonata2 = std::make_unique<SonataData>("test_sonatadata2",
population_name,
max_buffer_size,
Expand All @@ -108,7 +108,7 @@ SCENARIO("Test SonataData class", "[SonataData][IOWriter]") {
THEN(
"The buffer size is the number of steps to write that fit on the buffer times the "
"total elements") {
// 256 / sizeof(double) / 11 = 2
// 128 / (sizeof(float) * 11) = 2 < 3 (total number of steps)
// buffer_size = 11 * 2
REQUIRE(sonata2->get_report_buffer().size() == 22);
}
Expand Down

0 comments on commit 7975c04

Please sign in to comment.