From 6c9a328053911d35fef326bb15dfd0643e82a652 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Thu, 2 Jan 2025 20:01:42 +0100 Subject: [PATCH 1/6] Allow to row::get blob into suitable containers "Suitable" means that the container uses contiguous storage in memory and holds entries of type T where sizeof(T) == 1. If this is given, then the blob's data can be copied into a new instance of such a container, which is then returned in the row::get<>() call. This restores the previous possibility of reading blobs into objects of type std::string but also generalizes the concept to any suitable container (such as std::vector). Note: The check for whether the given container object uses contiguous object storage is based on its iterators being classified as random-access iterators. This is not a perfect check as in principle there could be random-access iterators that don't iterate a contiguous memory region. However, for typical containers this is very unexpected as this would typically lead to rather inefficient access patterns when making use of the random-access capability. A container that is wrongly determined to be contiguous will likely result in a program crash due to a segfault. Fixes #1173 --- include/soci/type-holder.h | 91 +++++++++++++++++++++++++++++++++++++- tests/common/test-lob.cpp | 25 +++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/include/soci/type-holder.h b/include/soci/type-holder.h index 375a5fd39..6dc514d0c 100644 --- a/include/soci/type-holder.h +++ b/include/soci/type-holder.h @@ -19,6 +19,7 @@ #include #include #include +#include namespace soci { @@ -128,6 +129,93 @@ struct soci_cast< } }; +template +struct is_contiguous_byte_container +{ + static constexpr bool value = false; +}; + +// What we are checking here is not really a guarantee that the container will +// use contiguous memory for storing its elements. However, usually the ability +// of random-access is only supplied for containers that are in fact contiguous +// in memory. +// Once switched to C++20, the check should be for contiguous_iterator_tag instead +template +struct is_contiguous_byte_container< + T, + typename std::enable_if< + std::is_same< + typename std::iterator_traits::iterator_category, + std::random_access_iterator_tag + >::value + && sizeof(typename T::value_type) == sizeof(char) + >::type +> +{ + static constexpr bool value = true; +}; + +static_assert(is_contiguous_byte_container::value, "std::string should pass constraint"); +static_assert(is_contiguous_byte_container>::value, "std::vector should pass constraint"); + +// Specialization for the blob type +// It throws unless it is requested to be cast to a container +// of type T where sizeof(T) == 1 and which is using +// contiguous storage in memory. +template +struct soci_cast::value>::type> +{ + static inline T cast(blob &) + { + // Blobs are non-copyable and therefore, we have to throw + // here even if T is soci::blob as this function is always + // used in context where copying of the blob would happen. + throw std::bad_cast(); + } +}; + +template +struct soci_cast::value>::type> +{ + static inline T cast(blob &blob) + { + T container; + container.resize(blob.get_len()); + + if (container.size() > 0) + { + // Note: if the container isn't actually contiguous in memory, this + // will likely lead to a segmentation fault! + blob.read_from_start(&container[0], container.size(), 0); + } + + return container; + } +}; + +template<> +struct soci_cast +{ + static inline std::string cast(blob &blob) + { + std::string data; + data.resize(blob.get_len()); + + if (data.empty()) + { + return data; + } + + // std::string::data() doesn't return non-const pointer + // before C++17. Hence, we have to use the uglier version. + blob.read_from_start(&data[0], data.size(), 0); + + return data; + } +}; + union type_holder { std::string* s; @@ -340,8 +428,7 @@ class holder case db_date: return soci_cast::cast(*val_.t); case db_blob: - // blob is not copyable - break; + return soci_cast::cast(*val_.b); case db_xml: case db_string: return soci_cast::cast(*val_.s); diff --git a/tests/common/test-lob.cpp b/tests/common/test-lob.cpp index b696a9d71..3f44426f3 100644 --- a/tests/common/test-lob.cpp +++ b/tests/common/test-lob.cpp @@ -653,6 +653,31 @@ TEST_CASE_METHOD(common_tests, "BLOB", "[core][blob]") } CHECK(containedData); } + SECTION("get into container") + { + soci::rowset< soci::row > rowSet = (sql.prepare << "select b from soci_test where id=:id", soci::use(id1)); + bool containedData = false; + for (auto it = rowSet.begin(); it != rowSet.end(); ++it) + { + containedData = true; + const soci::row ¤tRow = *it; + + std::string strData = currentRow.get(0); + std::vector vecData = currentRow.get>(0); + + // Container is required to hold a type that has a size of 1 byte + CHECK_THROWS(currentRow.get>(0)); + + CHECK(strData.size() == 10); + CHECK(vecData.size() == 10); + for (std::size_t i = 0; i < 10; ++i) + { + CHECK(strData[i] == dummy_data[i]); + CHECK(vecData[i] == dummy_data[i]); + } + } + CHECK(containedData); + } SECTION("reusing bound blob") { int secondID = id2 + 1; From b53facfea2decd51c3e55db1fd77ae5d46ea84ea Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Thu, 2 Jan 2025 20:10:08 +0100 Subject: [PATCH 2/6] Remove unneeded template specialization --- include/soci/type-holder.h | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/include/soci/type-holder.h b/include/soci/type-holder.h index 6dc514d0c..ae677f857 100644 --- a/include/soci/type-holder.h +++ b/include/soci/type-holder.h @@ -195,27 +195,6 @@ struct soci_cast -struct soci_cast -{ - static inline std::string cast(blob &blob) - { - std::string data; - data.resize(blob.get_len()); - - if (data.empty()) - { - return data; - } - - // std::string::data() doesn't return non-const pointer - // before C++17. Hence, we have to use the uglier version. - blob.read_from_start(&data[0], data.size(), 0); - - return data; - } -}; - union type_holder { std::string* s; From 7db4f5e52c4dc78c1276c4bbae00193da079e683 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Thu, 2 Jan 2025 20:14:00 +0100 Subject: [PATCH 3/6] Test for exceptions with non-contiguous containers --- tests/common/test-lob.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/common/test-lob.cpp b/tests/common/test-lob.cpp index 3f44426f3..b4d1e1b11 100644 --- a/tests/common/test-lob.cpp +++ b/tests/common/test-lob.cpp @@ -11,6 +11,10 @@ #include "test-context.h" +#include +#include +#include + namespace soci { @@ -667,6 +671,9 @@ TEST_CASE_METHOD(common_tests, "BLOB", "[core][blob]") // Container is required to hold a type that has a size of 1 byte CHECK_THROWS(currentRow.get>(0)); + // Container has to use contiguous storage + CHECK_THROWS(currentRow.get>(0)); + CHECK_THROWS(currentRow.get>(0)); CHECK(strData.size() == 10); CHECK(vecData.size() == 10); From 8acc0ee06164c2db1891d1b8b58e21c7aab9d002 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Fri, 3 Jan 2025 13:08:40 +0100 Subject: [PATCH 4/6] Silence unreachable code warning of old MSVC versions --- include/soci/row.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/soci/row.h b/include/soci/row.h index 43e77cb99..dba0172ff 100644 --- a/include/soci/row.h +++ b/include/soci/row.h @@ -72,6 +72,12 @@ class SOCI_DECL row column_properties const& get_properties(std::size_t pos) const; column_properties const& get_properties(std::string const& name) const; +#ifdef _MSC_VER +// MSVC complains about "unreachable code" in case get can +// be determined at compile time to throw. +#pragma warning(push) +#pragma warning(disable:4702) +#endif template T get(std::size_t pos) const { @@ -104,6 +110,9 @@ class SOCI_DECL row return ret; } +#ifdef _MSC_VER +#pragma warning(pop) +#endif template T get(std::size_t pos, T const &nullValue) const From 4c8cefb757d47782113a79a9c722e5f9b14b7881 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Thu, 9 Jan 2025 16:55:00 +0100 Subject: [PATCH 5/6] Allow to get blobs only into pre-selected containers --- include/soci/type-holder.h | 78 ++++++++++++++++++++------------------ tests/common/test-lob.cpp | 3 +- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/include/soci/type-holder.h b/include/soci/type-holder.h index ae677f857..867d9c4f8 100644 --- a/include/soci/type-holder.h +++ b/include/soci/type-holder.h @@ -19,11 +19,36 @@ #include #include #include -#include +#include +#include namespace soci { +// Trait to determine whether a given T is a container that stores its elements +// in a continuous region in memory and which can be resized. +template +struct is_contiguous_resizable_container : std::false_type {}; + +template +constexpr bool is_contiguous_resizable_container_v = is_contiguous_resizable_container::value; + +// An accessor template for contiguous, resizable containers that can be +// specialized for types for which this default implementation doesn't work. +template>> +struct contiguous_resizable_container_accessor +{ + static void *data(T &container) + { + static_assert(sizeof(decltype(container[0])) == sizeof(char), "Expected value-type of container to be byte-sized"); + return &container[0]; + } + + static std::size_t size(const T &container) { return container.size(); } + + static void resize(T &container, std::size_t size) { container.resize(size); } +}; + namespace details { @@ -129,42 +154,13 @@ struct soci_cast< } }; -template -struct is_contiguous_byte_container -{ - static constexpr bool value = false; -}; - -// What we are checking here is not really a guarantee that the container will -// use contiguous memory for storing its elements. However, usually the ability -// of random-access is only supplied for containers that are in fact contiguous -// in memory. -// Once switched to C++20, the check should be for contiguous_iterator_tag instead -template -struct is_contiguous_byte_container< - T, - typename std::enable_if< - std::is_same< - typename std::iterator_traits::iterator_category, - std::random_access_iterator_tag - >::value - && sizeof(typename T::value_type) == sizeof(char) - >::type -> -{ - static constexpr bool value = true; -}; - -static_assert(is_contiguous_byte_container::value, "std::string should pass constraint"); -static_assert(is_contiguous_byte_container>::value, "std::vector should pass constraint"); - // Specialization for the blob type // It throws unless it is requested to be cast to a container // of type T where sizeof(T) == 1 and which is using // contiguous storage in memory. template struct soci_cast::value>::type> + typename std::enable_if>::type> { static inline T cast(blob &) { @@ -177,18 +173,20 @@ struct soci_cast struct soci_cast::value>::type> + typename std::enable_if>::type> { static inline T cast(blob &blob) { T container; - container.resize(blob.get_len()); + using Accessor = contiguous_resizable_container_accessor; - if (container.size() > 0) + const std::size_t size = blob.get_len(); + + if (size > 0) { - // Note: if the container isn't actually contiguous in memory, this - // will likely lead to a segmentation fault! - blob.read_from_start(&container[0], container.size(), 0); + Accessor::resize(container, size); + + blob.read_from_start(Accessor::data(container), size); } return container; @@ -510,6 +508,12 @@ class holder } // namespace details +template<> +struct is_contiguous_resizable_container : std::true_type {}; + +template +struct is_contiguous_resizable_container, typename std::enable_if::type> : std::true_type {}; + } // namespace soci #endif // SOCI_TYPE_HOLDER_H_INCLUDED diff --git a/tests/common/test-lob.cpp b/tests/common/test-lob.cpp index b4d1e1b11..989443f88 100644 --- a/tests/common/test-lob.cpp +++ b/tests/common/test-lob.cpp @@ -671,7 +671,8 @@ TEST_CASE_METHOD(common_tests, "BLOB", "[core][blob]") // Container is required to hold a type that has a size of 1 byte CHECK_THROWS(currentRow.get>(0)); - // Container has to use contiguous storage + // Using containers for which the soci::is_contiguous_resizable_container trait is not + // defined yield a std::bad_cast exception. CHECK_THROWS(currentRow.get>(0)); CHECK_THROWS(currentRow.get>(0)); From f2e60f376ca54408a7b363ce4e8866757aa212e6 Mon Sep 17 00:00:00 2001 From: Robert Adam Date: Thu, 9 Jan 2025 17:18:41 +0100 Subject: [PATCH 6/6] Add documentation for row's blob support --- docs/types.md | 49 ++++++++++++++++++++++++++++++++++++++ include/soci/type-holder.h | 3 +++ 2 files changed, 52 insertions(+) diff --git a/docs/types.md b/docs/types.md index e71962611..dedccd33d 100644 --- a/docs/types.md +++ b/docs/types.md @@ -138,6 +138,53 @@ Users are encouraged to use the latter as it supports a wider range of numerical The mapping of underlying database column types to SOCI datatypes is database specific. See the [backend documentation](backends/index.md) for details. +While `row::get()` (semantically) copies the internally stored data (that was fetched from the database), there is also `row::move_as`, which +instead makes use of C++11's move semantics by moving the data out of the `row` instance. In case the used `T` does not support move semantics, +`move_as` is functionally equivalent to `get`. + +### Dealing with Blobs + +If the fetched data is of type `db_blob` it is strongly recommended to use `row::move_as()` in order to obtain the data as a `blob` object. This +allows for the most flexible and efficient access of the underlying data. + +Note that it is not possible to `row::get()` as `blob` objects are not copyable. However, you can instead get the blob's data into a +`std::string` or a `std::vector` as long as `T` has a size of exactly one byte (i.e. is a byte-like type such as `char` or `std::byte`). Depending +on the used backend, this will either directly fetch the data from the database directly into the provided container or copy the already fetched data +into the provided container. + +If you want to use a different container than the ones mentioned above, you need to specialize the `soci::is_contiguous_resizable_container` trait. +E.g. +```cpp +template<> +struct ::soci::is_contiguous_resizable_container : std::true_type {}; + +template +struct ::soci::is_contiguous_resizable_container, std::enable_if_t> : std::true_type {}; +``` +Potentially, you may also need to specialize the `soci::contiguous_resizable_container_accessor` struct, if the default implementation doesn't work +for your custom container type: +```cpp +template>> +struct contiguous_resizable_container_accessor +{ + // Gets the pointer to the beginning of the data store + static void *data(T &container) + { + static_assert(sizeof(decltype(container[0])) == sizeof(char), "Expected value-type of container to be byte-sized"); + return &container[0]; + } + + // Gets the size **in bytes** of this container + static std::size_t size(const T &container) { return container.size(); } + + // Resizes the container to the given size **in bytes** + static void resize(T &container, std::size_t size) { container.resize(size); } +}; +``` + + +### Indicators + The `row` also provides access to indicators for each column: ```cpp @@ -149,6 +196,8 @@ if (r.get_indicator(0) != soci::i_null) } ``` +### Stream API + It is also possible to extract data from the `row` object using its stream-like interface, where each extracted variable should have matching type respective to its position in the chain: ```cpp diff --git a/include/soci/type-holder.h b/include/soci/type-holder.h index 867d9c4f8..98d80529f 100644 --- a/include/soci/type-holder.h +++ b/include/soci/type-holder.h @@ -38,14 +38,17 @@ constexpr bool is_contiguous_resizable_container_v = is_contiguous_resizable_con template>> struct contiguous_resizable_container_accessor { + // Gets the pointer to the beginning of the data store static void *data(T &container) { static_assert(sizeof(decltype(container[0])) == sizeof(char), "Expected value-type of container to be byte-sized"); return &container[0]; } + // Gets the size **in bytes** of this container static std::size_t size(const T &container) { return container.size(); } + // Resizes the container to the given size **in bytes** static void resize(T &container, std::size_t size) { container.resize(size); } };