diff --git a/include/ankerl/unordered_dense.h b/include/ankerl/unordered_dense.h index 2aaacd6..5b7d3ca 100644 --- a/include/ankerl/unordered_dense.h +++ b/include/ankerl/unordered_dense.h @@ -70,6 +70,17 @@ # define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline)) #endif +#if !defined(ANKERL_UNORDERED_DENSE_FORCEINLINE) +# if defined(_MSC_VER) +# define ANKERL_UNORDERED_DENSE_FORCEINLINE __forceinline +# elif defined(__GNUC__) && __GNUC__ > 3 + // Clang also defines __GNUC__ (as 4) +# define ANKERL_UNORDERED_DENSE_FORCEINLINE inline __attribute__ ((__always_inline__)) +# else +# define ANKERL_UNORDERED_DENSE_FORCEINLINE inline +# endif +#endif + // defined in unordered_dense.cpp #if !defined(ANKERL_UNORDERED_DENSE_EXPORT) # define ANKERL_UNORDERED_DENSE_EXPORT @@ -142,13 +153,13 @@ namespace detail { # else -[[noreturn]] inline void on_error_key_not_found() { +[[noreturn]] ANKERL_UNORDERED_DENSE_FORCEINLINE void on_error_key_not_found() { abort(); } -[[noreturn]] inline void on_error_bucket_overflow() { +[[noreturn]] ANKERL_UNORDERED_DENSE_FORCEINLINE void on_error_bucket_overflow() { abort(); } -[[noreturn]] inline void on_error_too_many_elements() { +[[noreturn]] ANKERL_UNORDERED_DENSE_FORCEINLINE void on_error_too_many_elements() { abort(); } @@ -163,7 +174,7 @@ namespace detail { // hardcodes seed and the secret, reformats the code, and clang-tidy fixes. namespace detail::wyhash { -inline void mum(uint64_t* a, uint64_t* b) { +ANKERL_UNORDERED_DENSE_FORCEINLINE void mum(uint64_t* a, uint64_t* b) { # if defined(__SIZEOF_INT128__) __uint128_t r = *a; r *= *b; @@ -193,30 +204,30 @@ inline void mum(uint64_t* a, uint64_t* b) { } // multiply and xor mix function, aka MUM -[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t { +[[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto mix(uint64_t a, uint64_t b) -> uint64_t { mum(&a, &b); return a ^ b; } // read functions. WARNING: we don't care about endianness, so results are different on big endian! -[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t { +[[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto r8(const uint8_t* p) -> uint64_t { uint64_t v{}; std::memcpy(&v, p, 8U); return v; } -[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t { +[[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto r4(const uint8_t* p) -> uint64_t { uint32_t v{}; std::memcpy(&v, p, 4); return v; } // reads 1, 2, or 3 bytes -[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t { +[[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto r3(const uint8_t* p, size_t k) -> uint64_t { return (static_cast(p[0]) << 16U) | (static_cast(p[k >> 1U]) << 8U) | p[k - 1]; } -[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t { +[[maybe_unused]] [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto hash(void const* key, size_t len) -> uint64_t { static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f), UINT64_C(0xe7037ed1a0b428db), UINT64_C(0x8ebc6af09c88c6e3), @@ -263,7 +274,7 @@ inline void mum(uint64_t* a, uint64_t* b) { return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); } -[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t { +[[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto hash(uint64_t x) -> uint64_t { return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15)); } @@ -848,29 +859,29 @@ class table : public std::conditional_t, base_table_type_map, bas KeyEqual m_equal{}; uint8_t m_shifts = initial_shifts; - [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto next(value_idx_type bucket_idx) const -> value_idx_type { return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets) ? 0 : static_cast(bucket_idx + 1U); } // Helper to access bucket through pointer types - [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& { return *(bucket_ptr + static_cast::difference_type>(offset)); } // use the dist_inc and dist_dec functions so that uint16_t types work without warning - [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type { return static_cast(x + Bucket::dist_inc); } - [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type { return static_cast(x - Bucket::dist_inc); } // The goal of mixed_hash is to always produce a high quality 64bit hash. template - [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE constexpr auto mixed_hash(K const& key) const -> uint64_t { if constexpr (is_detected_v) { // we know that the hash is good because is_avalanching. if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) { @@ -886,11 +897,11 @@ class table : public std::conditional_t, base_table_type_map, bas } } - [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type { return Bucket::dist_inc | (static_cast(hash) & Bucket::fingerprint_mask); } - [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type { return static_cast(hash >> m_shifts); } @@ -903,7 +914,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - [[nodiscard]] auto next_while_less(K const& key) const -> Bucket { + [[nodiscard]] ANKERL_UNORDERED_DENSE_FORCEINLINE auto next_while_less(K const& key) const -> Bucket { auto hash = mixed_hash(key); auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); auto bucket_idx = bucket_idx_from_hash(hash); @@ -915,7 +926,7 @@ class table : public std::conditional_t, base_table_type_map, bas return {dist_and_fingerprint, bucket_idx}; } - void place_and_shift_up(Bucket bucket, value_idx_type place) { + ANKERL_UNORDERED_DENSE_FORCEINLINE void place_and_shift_up(Bucket bucket, value_idx_type place) { while (0 != at(m_buckets, place).m_dist_and_fingerprint) { bucket = std::exchange(at(m_buckets, place), bucket); bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint); @@ -1010,7 +1021,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - void do_erase(value_idx_type bucket_idx, Op handle_erased_value) { + ANKERL_UNORDERED_DENSE_FORCEINLINE void do_erase(value_idx_type bucket_idx, Op handle_erased_value) { auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx; // shift down until either empty or an element with correct spot is found @@ -1043,7 +1054,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto do_erase_key(K&& key, Op handle_erased_value) -> size_t { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_erase_key(K&& key, Op handle_erased_value) -> size_t { if (empty()) { return 0; } @@ -1073,7 +1084,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args) + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args) -> std::pair { // emplace the new value. If that throws an exception, no harm done; index is still in a valid state @@ -1091,7 +1102,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto do_try_emplace(K&& key, Args&&... args) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_try_emplace(K&& key, Args&&... args) -> std::pair { auto hash = mixed_hash(key); auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); auto bucket_idx = bucket_idx_from_hash(hash); @@ -1115,7 +1126,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto do_find(K const& key) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_find(K const& key) -> iterator { if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) { return end(); } @@ -1155,12 +1166,12 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto do_find(K const& key) const -> const_iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_find(K const& key) const -> const_iterator { return const_cast(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast) } template , bool> = true> - auto do_at(K const& key) -> Q& { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_at(K const& key) -> Q& { if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) { return it->second; } @@ -1168,7 +1179,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template , bool> = true> - auto do_at(K const& key) const -> Q const& { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto do_at(K const& key) const -> Q const& { return const_cast(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast) } @@ -1370,41 +1381,41 @@ class table : public std::conditional_t, base_table_type_map, bas clear_buckets(); } - auto insert(value_type const& value) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto insert(value_type const& value) -> std::pair { return emplace(value); } - auto insert(value_type&& value) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto insert(value_type&& value) -> std::pair { return emplace(std::move(value)); } template , bool> = true> - auto insert(P&& value) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto insert(P&& value) -> std::pair { return emplace(std::forward

(value)); } - auto insert(const_iterator /*hint*/, value_type const& value) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto insert(const_iterator /*hint*/, value_type const& value) -> iterator { return insert(value).first; } - auto insert(const_iterator /*hint*/, value_type&& value) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto insert(const_iterator /*hint*/, value_type&& value) -> iterator { return insert(std::move(value)).first; } template , bool> = true> - auto insert(const_iterator /*hint*/, P&& value) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto insert(const_iterator /*hint*/, P&& value) -> iterator { return insert(std::forward

(value)).first; } template - void insert(InputIt first, InputIt last) { + ANKERL_UNORDERED_DENSE_FORCEINLINE void insert(InputIt first, InputIt last) { while (first != last) { insert(*first); ++first; } } - void insert(std::initializer_list ilist) { + ANKERL_UNORDERED_DENSE_FORCEINLINE void insert(std::initializer_list ilist) { insert(ilist.begin(), ilist.end()); } @@ -1514,7 +1525,7 @@ class table : public std::conditional_t, base_table_type_map, bas typename H = Hash, typename KE = KeyEqual, std::enable_if_t && is_transparent_v, bool> = true> - auto emplace(K&& key) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto emplace(K&& key) -> std::pair { auto hash = mixed_hash(key); auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); auto bucket_idx = bucket_idx_from_hash(hash); @@ -1534,7 +1545,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto emplace(Args&&... args) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto emplace(Args&&... args) -> std::pair { // we have to instantiate the value_type to be able to access the key. // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop. auto& key = get_key(m_values.emplace_back(std::forward(args)...)); @@ -1565,27 +1576,27 @@ class table : public std::conditional_t, base_table_type_map, bas } template - auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator { return emplace(std::forward(args)...).first; } template , bool> = true> - auto try_emplace(Key const& key, Args&&... args) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto try_emplace(Key const& key, Args&&... args) -> std::pair { return do_try_emplace(key, std::forward(args)...); } template , bool> = true> - auto try_emplace(Key&& key, Args&&... args) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto try_emplace(Key&& key, Args&&... args) -> std::pair { return do_try_emplace(std::move(key), std::forward(args)...); } template , bool> = true> - auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator { return do_try_emplace(key, std::forward(args)...).first; } template , bool> = true> - auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator { return do_try_emplace(std::move(key), std::forward(args)...).first; } @@ -1597,7 +1608,7 @@ class table : public std::conditional_t, base_table_type_map, bas typename KE = KeyEqual, std::enable_if_t && is_transparent_v && is_neither_convertible_v, bool> = true> - auto try_emplace(K&& key, Args&&... args) -> std::pair { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto try_emplace(K&& key, Args&&... args) -> std::pair { return do_try_emplace(std::forward(key), std::forward(args)...); } @@ -1609,11 +1620,11 @@ class table : public std::conditional_t, base_table_type_map, bas typename KE = KeyEqual, std::enable_if_t && is_transparent_v && is_neither_convertible_v, bool> = true> - auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator { return do_try_emplace(std::forward(key), std::forward(args)...).first; } - auto erase(iterator it) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto erase(iterator it) -> iterator { auto hash = mixed_hash(get_key(*it)); auto bucket_idx = bucket_idx_from_hash(hash); @@ -1644,7 +1655,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template , bool> = true> - auto erase(const_iterator it) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto erase(const_iterator it) -> iterator { return erase(begin() + (it - cbegin())); } @@ -1677,7 +1688,7 @@ class table : public std::conditional_t, base_table_type_map, bas return begin() + idx_first; } - auto erase(Key const& key) -> size_t { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto erase(Key const& key) -> size_t { return do_erase_key(key, [](value_type&& /*unused*/) { }); } @@ -1691,7 +1702,7 @@ class table : public std::conditional_t, base_table_type_map, bas } template , bool> = true> - auto erase(K&& key) -> size_t { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto erase(K&& key) -> size_t { return do_erase_key(std::forward(key), [](value_type&& /*unused*/) { }); } @@ -1742,12 +1753,12 @@ class table : public std::conditional_t, base_table_type_map, bas } template , bool> = true> - auto operator[](Key const& key) -> Q& { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto operator[](Key const& key) -> Q& { return try_emplace(key).first->second; } template , bool> = true> - auto operator[](Key&& key) -> Q& { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto operator[](Key&& key) -> Q& { return try_emplace(std::move(key)).first->second; } @@ -1760,39 +1771,39 @@ class table : public std::conditional_t, base_table_type_map, bas return try_emplace(std::forward(key)).first->second; } - auto count(Key const& key) const -> size_t { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto count(Key const& key) const -> size_t { return find(key) == end() ? 0 : 1; } template , bool> = true> - auto count(K const& key) const -> size_t { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto count(K const& key) const -> size_t { return find(key) == end() ? 0 : 1; } - auto find(Key const& key) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto find(Key const& key) -> iterator { return do_find(key); } - auto find(Key const& key) const -> const_iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto find(Key const& key) const -> const_iterator { return do_find(key); } template , bool> = true> - auto find(K const& key) -> iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto find(K const& key) -> iterator { return do_find(key); } template , bool> = true> - auto find(K const& key) const -> const_iterator { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto find(K const& key) const -> const_iterator { return do_find(key); } - auto contains(Key const& key) const -> bool { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto contains(Key const& key) const -> bool { return find(key) != end(); } template , bool> = true> - auto contains(K const& key) const -> bool { + ANKERL_UNORDERED_DENSE_FORCEINLINE auto contains(K const& key) const -> bool { return find(key) != end(); }