-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement simple key ranges for broadcast
Problem: broadcasting a range of keys currently requires application
to populate the full range into a container and pass it to ttg::broadcast.
These keys then have to be processed and send to peers, leading to
large messages (in addition to the actual data transfer).
Many (dense) applications send a range of keys that can be expressed
as a linear function in an N-dimensional space. With key ranges,
application can express the start and end of a range as well as
a step size for iterating through the range (a vector in the N dimensional
space). Some applications may require splitting the range, i.e.,
providing a list or ranges (e.g., the Floyd-Warshall example).
These ranges can be serialized and send to peers instead of the fully
populated list of keys. Such compact representation of the key range
is cheaper to transfer and store.
An example for using key ranges:
```
auto range1_low = ttg::make_keyrange(Key(I, 0, K), /* start */
Key(I, J, K), /* end, i.e., last + 1 */
Key(0, 1, 0)); /* step vector */
```
It is possible to define a different type for the step vector by defining
`Key::difference_type` or by specializing the `ttg::key_diff_type` trait.
Signed-off-by: Joseph Schuchart <[email protected]>- Loading branch information
Showing
9 changed files
with
964 additions
and
218 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,209 @@ | ||
| #ifndef TTG_KEYTERATOR_H | ||
| #define TTG_KEYTERATOR_H | ||
|
|
||
| #include <type_traits> | ||
| #include <iterator> | ||
| #include <exception> | ||
|
|
||
| #include "ttg/util/meta.h" | ||
| #include "ttg/serialization.h" | ||
|
|
||
| namespace ttg { | ||
|
|
||
| /* Trait providing the diff type of a given key | ||
| * Defaults to the key itself. | ||
| * May be provided as \c diff_type member of the key or by | ||
| * overloading this trait. | ||
| */ | ||
| template<typename Key, typename = void> | ||
| struct key_diff_type { | ||
| using type = Key; | ||
| }; | ||
|
|
||
| /* Overload for Key::diff_type */ | ||
| template<typename Key> | ||
| struct key_diff_type<Key, std::void_t<typename Key::diff_type>>{ | ||
| using type = typename Key::diff_type; | ||
| }; | ||
|
|
||
| /* Convenience type */ | ||
| template<typename Key> | ||
| using key_diff_type_t = typename key_diff_type<Key>::type; | ||
|
|
||
| namespace detail { | ||
| /** | ||
| * Trait checking whether a key is compatible with the LinearKeyRange. | ||
| * Keys must at least support comparison as well as addition or compound addition. | ||
| */ | ||
| template<typename Key> | ||
| struct is_range_compatible { | ||
| using key_type = std::decay_t<Key>; | ||
| using difference_type = key_diff_type_t<key_type>; | ||
| constexpr static bool value = ttg::meta::is_comparable_v<key_type> | ||
| && (ttg::meta::has_addition_v<key_type, difference_type> | ||
| || ttg::meta::has_compound_addition_v<key_type, difference_type>) | ||
| && (std::is_trivially_copyable_v<Key> || is_user_buffer_serializable_v<Key>); | ||
| }; | ||
|
|
||
| template<typename Key> | ||
| constexpr bool is_range_compatible_v = is_range_compatible<Key>::value; | ||
|
|
||
| /** | ||
| * Represents a range of keys that can be represented as a linear iteration | ||
| * space, i.e., using a start and end (one past the last element) as well as | ||
| * a step increment. An iterator is provided to iterate over the range of keys. | ||
| * | ||
| * The step increment is of type \sa ttg::key_diff_type, which defaults | ||
| * to the key type but can be overridden by either defining a \c diff_type | ||
| * member type or by specializing ttg::key_diff_type. | ||
| */ | ||
| template<typename Key> | ||
| struct LinearKeyRange { | ||
| using key_type = std::decay_t<Key>; | ||
| using diff_type = key_diff_type_t<key_type>; | ||
|
|
||
| /* Forward Iterator for the linear key range */ | ||
| struct iterator | ||
| { | ||
| /* types for std::iterator_trait */ | ||
| using value_type = key_type; | ||
| using difference_type = diff_type; | ||
| using pointer = const value_type*; | ||
| using reference = const value_type&; | ||
| using iterator_category = std::forward_iterator_tag; | ||
|
|
||
| iterator(const key_type& pos, const diff_type& inc) | ||
| : m_pos(pos), m_inc(inc) | ||
| { } | ||
|
|
||
| iterator& operator++() { | ||
| if constexpr (meta::has_compound_addition_v<value_type, difference_type>) { | ||
| m_pos += m_inc; | ||
| } else if constexpr (meta::has_addition_v<value_type, difference_type>) { | ||
| m_pos = m_pos + m_inc; | ||
| } else { | ||
| throw std::logic_error("Key type does not support addition its with difference type"); | ||
| } | ||
| return *this; | ||
| } | ||
|
|
||
| iterator operator++(int) { | ||
| iterator retval = *this; | ||
| ++(*this); | ||
| return retval; | ||
| } | ||
|
|
||
| bool operator==(const iterator& other) const { | ||
| if constexpr(meta::is_comparable_v<value_type>) { | ||
| return m_pos == other.m_pos; | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| bool operator!=(const iterator& other) const { | ||
| return !(*this == other); | ||
| } | ||
| const key_type& operator*() const { | ||
| return m_pos; | ||
| } | ||
| const key_type* operator->() const { | ||
| return &m_pos; | ||
| } | ||
|
|
||
| private: | ||
| key_type m_pos; | ||
| const diff_type& m_inc; | ||
| }; | ||
|
|
||
| LinearKeyRange() | ||
| { } | ||
|
|
||
| LinearKeyRange(const Key& begin, const Key& end, const diff_type& inc) | ||
| : m_begin(begin) | ||
| , m_end(end) | ||
| , m_inc(inc) | ||
| { } | ||
|
|
||
| iterator begin() const { | ||
| return iterator(m_begin, m_inc); | ||
| } | ||
|
|
||
| iterator end() const { | ||
| return iterator(m_end, m_inc); | ||
| } | ||
|
|
||
| #ifdef TTG_SERIALIZATION_SUPPORTS_MADNESS | ||
| /* Make the LinearKeyRange madness serializable */ | ||
| template<typename Archive, typename Key_ = key_type, typename Diff_ = diff_type, | ||
| typename = std::enable_if_t<is_madness_buffer_serializable_v<Key_> | ||
| && is_madness_buffer_serializable_v<Diff_>>> | ||
| void serialize(Archive& ar) { | ||
| ar & m_begin; | ||
| ar & m_end; | ||
| ar & m_inc; | ||
| } | ||
| #endif | ||
|
|
||
| #ifdef TTG_SERIALIZATION_SUPPORTS_BOOST | ||
| /* Make the LinearKeyRange boost serializable */ | ||
| template<typename Archive, typename Key_ = key_type, typename Diff_ = diff_type, | ||
| typename = std::enable_if_t<is_boost_buffer_serializable_v<Key_> | ||
| && is_boost_buffer_serializable_v<Diff_>>> | ||
| void serialize(Archive& ar, const unsigned int version) { | ||
| ar & m_begin; | ||
| ar & m_end; | ||
| ar & m_inc; | ||
| } | ||
| #endif | ||
|
|
||
| friend std::ostream& operator<<(std::ostream& out, LinearKeyRange const& k) { | ||
| out << "LinearKeyRange[" << k.m_begin << "," << k.m_end << "):"<<k.m_inc; | ||
| return out; | ||
| } | ||
|
|
||
| private: | ||
| key_type m_begin, m_end; | ||
| diff_type m_inc; | ||
| }; | ||
|
|
||
| } // namespace detail | ||
|
|
||
| /** | ||
| * Create a key range [begin, end) with step size \c inc. | ||
| * \return A representation of the range that can be passed to send/broadcast. | ||
| */ | ||
| template<typename Key> | ||
| inline auto make_keyrange(const Key& begin, | ||
| const Key& end, | ||
| const key_diff_type_t<Key>& inc) { | ||
| static_assert(detail::is_range_compatible_v<Key>, | ||
| "Key type does not support all required operations: operator==, " | ||
| "operator+ or operator+=, and serialization (trivially_copyable, madness, or boost)"); | ||
| return detail::LinearKeyRange(begin, end, inc); | ||
| } | ||
|
|
||
| /** | ||
| * Create a key range [begin, end) with unit stride. | ||
| * | ||
| * Requires \c operator++ and \c operator- to be defined on Key. | ||
| * If a difference type (\c Key::diff_type) is defined, \c operator- should return | ||
| * the difference type. | ||
| * | ||
| * \return A representation of the range that can be passed to send/broadcast. | ||
| */ | ||
| template<typename Key> | ||
| inline auto make_keyrange(const Key& begin, const Key& end) { | ||
| static_assert(detail::is_range_compatible_v<Key>, | ||
| "Key type does not support all required operations: operator==, " | ||
| "operator+ or operator+=, and serialization (trivially_copyable, madness, or boost)"); | ||
| static_assert(meta::has_increment_v<Key> && meta::has_difference_v<Key>, | ||
| "Unit stride key range requires operator++ and operator- on Key"); | ||
| if constexpr (meta::has_pre_increment_v<Key>) { | ||
| return detail::LinearKeyRange(begin, end, ++begin - begin); | ||
| } else { | ||
| return detail::LinearKeyRange(begin, end, begin++ - begin); | ||
| } | ||
| } | ||
|
|
||
| } // namespace ttg | ||
| #endif // TTG_KEYTERATOR_H |
Oops, something went wrong.