diff --git a/debian/copyright b/debian/copyright index 6db681295..08af5120c 100644 --- a/debian/copyright +++ b/debian/copyright @@ -201,7 +201,7 @@ License: Expat IN THE SOFTWARE. Files: spot/priv/robin_hood.hh -Copyright: 2018-2019 Martin Ankerl +Copyright: 2018-2021 Martin Ankerl License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/spot/priv/Makefile.am b/spot/priv/Makefile.am index b3d3c542f..d4e9cc77c 100644 --- a/spot/priv/Makefile.am +++ b/spot/priv/Makefile.am @@ -1,6 +1,6 @@ ## -*- coding: utf-8 -*- -## Copyright (C) 2013-2019 Laboratoire de Recherche et Développement -## de l'Epita (LRDE). +## Copyright (C) 2013-2019, 2021 Laboratoire de Recherche et +## Développement de l'Epita (LRDE). ## ## This file is part of Spot, a model checking library. ## @@ -41,5 +41,6 @@ GH = https://raw.githubusercontent.com/martinus RH = $(GH)/robin-hood-hashing/master/src/include/robin_hood.h .PHONY: update update: - wget $(RH) -O $(srcdir)/robin_hood.hh || \ - curl $(RH) -o $(srcdir)/robin_hood.hh + wget $(RH) -O robin_hood.tmp || curl $(RH) -o robin_hood.tmp + sed 's/std::malloc/malloc/' robin_hood.tmp > $(srcdir)/robin_hood.hh + rm -f robin_hood.tmp diff --git a/spot/priv/robin_hood.hh b/spot/priv/robin_hood.hh index 49ef01045..8c151d517 100644 --- a/spot/priv/robin_hood.hh +++ b/spot/priv/robin_hood.hh @@ -6,12 +6,11 @@ // _/_____/ // // Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20 -// version 3.4.1 // https://github.com/martinus/robin-hood-hashing // // Licensed under the MIT License . // SPDX-License-Identifier: MIT -// Copyright (c) 2018-2019 Martin Ankerl +// Copyright (c) 2018-2021 Martin Ankerl // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,23 +34,29 @@ #define ROBIN_HOOD_H_INCLUDED // see https://semver.org/ -#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes -#define ROBIN_HOOD_VERSION_MINOR 4 // for adding functionality in a backwards-compatible manner -#define ROBIN_HOOD_VERSION_PATCH 1 // for backwards-compatible bug fixes +#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes +#define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner +#define ROBIN_HOOD_VERSION_PATCH 3 // for backwards-compatible bug fixes #include #include #include #include +#include +#include // only to support hash of smart pointers #include #include #include #include +#if __cplusplus >= 201703L +# include +#endif // #define ROBIN_HOOD_LOG_ENABLED #ifdef ROBIN_HOOD_LOG_ENABLED # include -# define ROBIN_HOOD_LOG(x) std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << x << std::endl +# define ROBIN_HOOD_LOG(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; #else # define ROBIN_HOOD_LOG(x) #endif @@ -59,12 +64,34 @@ // #define ROBIN_HOOD_TRACE_ENABLED #ifdef ROBIN_HOOD_TRACE_ENABLED # include -# define ROBIN_HOOD_TRACE(x) \ - std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << x << std::endl +# define ROBIN_HOOD_TRACE(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; #else # define ROBIN_HOOD_TRACE(x) #endif +// #define ROBIN_HOOD_COUNT_ENABLED +#ifdef ROBIN_HOOD_COUNT_ENABLED +# include +# define ROBIN_HOOD_COUNT(x) ++counts().x; +namespace robin_hood { +struct Counts { + uint64_t shiftUp{}; + uint64_t shiftDown{}; +}; +inline std::ostream& operator<<(std::ostream& os, Counts const& c) { + return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; +} + +static Counts& counts() { + static Counts counts{}; + return counts; +} +} // namespace robin_hood +#else +# define ROBIN_HOOD_COUNT(x) +#endif + // all non-argument macros should use this facility. See // https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/ #define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x() @@ -106,31 +133,32 @@ #endif // count leading/trailing bits -#ifdef _MSC_VER -# if ROBIN_HOOD(BITNESS) == 32 -# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward +#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS) +# ifdef _MSC_VER +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 +# endif +# include +# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ + [](size_t mask) noexcept -> int { \ + unsigned long index; \ + return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) \ + : ROBIN_HOOD(BITNESS); \ + }(x) # else -# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll +# endif +# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) # endif -# include -# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) -# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ - [](size_t mask) noexcept->int { \ - unsigned long index; \ - return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) \ - : ROBIN_HOOD(BITNESS); \ - } \ - (x) -#else -# if ROBIN_HOOD(BITNESS) == 32 -# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl -# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl -# else -# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll -# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll -# endif -# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) -# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) #endif // fallthrough @@ -154,6 +182,28 @@ # define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) #endif +// detect if native wchar_t type is availiable in MSVC +#ifdef _MSC_VER +# ifdef _NATIVE_WCHAR_T_DEFINED +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0 +# endif +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +#endif + +// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr +#ifdef _MSC_VER +# if _MSC_VER <= 1900 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1 +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 +# endif +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 +#endif + // workaround missing "is_trivially_copyable" in g++ < 5.0 // See https://stackoverflow.com/a/31798726/48181 #if defined(__GNUC__) && __GNUC__ < 5 @@ -253,39 +303,18 @@ using index_sequence_for = make_index_sequence; namespace detail { -// umul -#if defined(__SIZEOF_INT128__) -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 1 -# if defined(__GNUC__) || defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wpedantic" -using uint128_t = unsigned __int128; -# pragma GCC diagnostic pop -# endif -inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) noexcept { - auto result = static_cast(a) * static_cast(b); - *high = static_cast(result >> 64U); - return static_cast(result); -} -#elif (defined(_MSC_VER) && ROBIN_HOOD(BITNESS) == 64) -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 1 -# include // for __umulh -# pragma intrinsic(__umulh) -# ifndef _M_ARM64 -# pragma intrinsic(_umul128) -# endif -inline uint64_t umul128(uint64_t a, uint64_t b, uint64_t* high) noexcept { -# ifdef _M_ARM64 - *high = __umulh(a, b); - return ((uint64_t)(a)) * (b); -# else - return _umul128(a, b, high); -# endif -} +// make sure we static_cast to the correct type for hash_int +#if ROBIN_HOOD(BITNESS) == 64 +using SizeT = uint64_t; #else -# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_UMUL128() 0 +using SizeT = uint32_t; #endif +template +T rotr(T x, unsigned k) { + return (x >> k) | (x << (8U * sizeof(T) - k)); +} + // This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to // 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with // care! @@ -302,14 +331,14 @@ inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept { // make sure this is not inlined as it is slow and dramatically enlarges code, thus making other // inlinings more difficult. Throws are also generally the slow path. template -ROBIN_HOOD(NOINLINE) +[[noreturn]] ROBIN_HOOD(NOINLINE) #if ROBIN_HOOD(HAS_EXCEPTIONS) -void doThrow(Args&&... args) { + void doThrow(Args&&... args) { // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) throw E(std::forward(args)...); } #else -void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) { + void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) { abort(); } #endif @@ -361,6 +390,7 @@ public: } BulkPoolAllocator& + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept { // does not do anything return *this; @@ -374,7 +404,8 @@ public: void reset() noexcept { while (mListForFree) { T* tmp = *mListForFree; - free(mListForFree); + ROBIN_HOOD_LOG("std::free") + std::free(mListForFree); mListForFree = reinterpret_cast_no_cast_align_warning(tmp); } mHead = nullptr; @@ -409,8 +440,10 @@ public: // calculate number of available elements in ptr if (numBytes < ALIGNMENT + ALIGNED_SIZE) { // not enough data for at least one element. Free and return. - free(ptr); + ROBIN_HOOD_LOG("std::free") + std::free(ptr); } else { + ROBIN_HOOD_LOG("add to buffer") add(ptr, numBytes); } } @@ -451,10 +484,10 @@ private: mListForFree = data; // create linked list for newly allocated data - auto const headT = + auto* const headT = reinterpret_cast_no_cast_align_warning(reinterpret_cast(ptr) + ALIGNMENT); - auto const head = reinterpret_cast(headT); + auto* const head = reinterpret_cast(headT); // Visual Studio compiler automatically unrolls this loop, which is pretty cool for (size_t i = 0; i < numElements; ++i) { @@ -474,8 +507,9 @@ private: size_t const numElementsToAlloc = calcNumElementsToAlloc(); // alloc new memory: [prev |T, T, ... T] - // std::cout << (sizeof(T*) + ALIGNED_SIZE * numElementsToAlloc) << " bytes" << std::endl; size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; + ROBIN_HOOD_LOG("malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE + << " * " << numElementsToAlloc) add(assertNotNull(malloc(bytes)), bytes); return mHead; } @@ -503,7 +537,7 @@ private: T** mListForFree{nullptr}; }; -template +template struct NodeAllocator; // dummy allocator that does nothing @@ -512,30 +546,29 @@ struct NodeAllocator { // we are not using the data, so just free it. void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept { - free(ptr); + ROBIN_HOOD_LOG("std::free") + std::free(ptr); } }; template struct NodeAllocator : public BulkPoolAllocator {}; -// dummy hash, unsed as mixer when robin_hood::hash is already used -template -struct identity_hash { - constexpr size_t operator()(T const& obj) const noexcept { - return static_cast(obj); - } -}; - // c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making // my own here. namespace swappable { +#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17) using std::swap; template struct nothrow { static const bool value = noexcept(swap(std::declval(), std::declval())); }; - +#else +template +struct nothrow { + static const bool value = std::is_nothrow_swappable::value; +}; +#endif } // namespace swappable } // namespace detail @@ -564,44 +597,46 @@ struct pair { , second(o.second) {} // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. - explicit constexpr pair(std::pair&& o) noexcept( - noexcept(T1(std::move(std::declval()))) && - noexcept(T2(std::move(std::declval())))) + explicit constexpr pair(std::pair&& o) noexcept(noexcept( + T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) : first(std::move(o.first)) , second(std::move(o.second)) {} - constexpr pair(T1&& a, T2&& b) noexcept(noexcept(T1(std::move(std::declval()))) && - noexcept(T2(std::move(std::declval())))) + constexpr pair(T1&& a, T2&& b) noexcept(noexcept( + T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) : first(std::move(a)) , second(std::move(b)) {} template - constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward(std::declval()))) && - noexcept(T2(std::forward(std::declval())))) + constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward( + std::declval()))) && noexcept(T2(std::forward(std::declval())))) : first(std::forward(a)) , second(std::forward(b)) {} template - constexpr pair( - std::piecewise_construct_t /*unused*/, std::tuple a, - std::tuple b) noexcept(noexcept(pair(std::declval&>(), - std::declval&>(), - ROBIN_HOOD_STD::index_sequence_for(), - ROBIN_HOOD_STD::index_sequence_for()))) + // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members" + // if this constructor is constexpr +#if !ROBIN_HOOD(BROKEN_CONSTEXPR) + constexpr +#endif + pair(std::piecewise_construct_t /*unused*/, std::tuple a, + std::tuple + b) noexcept(noexcept(pair(std::declval&>(), + std::declval&>(), + ROBIN_HOOD_STD::index_sequence_for(), + ROBIN_HOOD_STD::index_sequence_for()))) : pair(a, b, ROBIN_HOOD_STD::index_sequence_for(), - ROBIN_HOOD_STD::index_sequence_for()) {} + ROBIN_HOOD_STD::index_sequence_for()) { + } // constructor called from the std::piecewise_construct_t ctor template - pair(std::tuple& a, std::tuple& b, - ROBIN_HOOD_STD::index_sequence /*unused*/, - ROBIN_HOOD_STD::index_sequence< - I2...> /*unused*/) noexcept(noexcept(T1(std:: - forward(std::get( - std::declval< - std::tuple&>()))...)) && - noexcept(T2(std::forward( - std::get(std::declval&>()))...))) + pair(std::tuple& a, std::tuple& b, ROBIN_HOOD_STD::index_sequence /*unused*/, ROBIN_HOOD_STD::index_sequence /*unused*/) noexcept( + noexcept(T1(std::forward(std::get( + std::declval&>()))...)) && noexcept(T2(std:: + forward(std::get( + std::declval&>()))...))) : first(std::forward(std::get(a))...) , second(std::forward(std::get(b))...) { // make visual studio compiler happy about warning about unused a & b. @@ -610,19 +645,6 @@ struct pair { (void)b; } - ROBIN_HOOD(NODISCARD) first_type& getFirst() noexcept { - return first; - } - ROBIN_HOOD(NODISCARD) first_type const& getFirst() const noexcept { - return first; - } - ROBIN_HOOD(NODISCARD) second_type& getSecond() noexcept { - return second; - } - ROBIN_HOOD(NODISCARD) second_type const& getSecond() const noexcept { - return second; - } - void swap(pair& o) noexcept((detail::swappable::nothrow::value) && (detail::swappable::nothrow::value)) { using std::swap; @@ -635,19 +657,44 @@ struct pair { }; template -void swap(pair& a, pair& b) noexcept( +inline void swap(pair& a, pair& b) noexcept( noexcept(std::declval&>().swap(std::declval&>()))) { a.swap(b); } -// Hash an arbitrary amount of bytes. This is basically Murmur2 hash without caring about big -// endianness. TODO(martinus) add a fallback for very large strings? -static size_t hash_bytes(void const* ptr, size_t const len) noexcept { +template +inline constexpr bool operator==(pair const& x, pair const& y) { + return (x.first == y.first) && (x.second == y.second); +} +template +inline constexpr bool operator!=(pair const& x, pair const& y) { + return !(x == y); +} +template +inline constexpr bool operator<(pair const& x, pair const& y) noexcept(noexcept( + std::declval() < std::declval()) && noexcept(std::declval() < + std::declval())) { + return x.first < y.first || (!(y.first < x.first) && x.second < y.second); +} +template +inline constexpr bool operator>(pair const& x, pair const& y) { + return y < x; +} +template +inline constexpr bool operator<=(pair const& x, pair const& y) { + return !(x > y); +} +template +inline constexpr bool operator>=(pair const& x, pair const& y) { + return !(x < y); +} + +inline size_t hash_bytes(void const* ptr, size_t len) noexcept { static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); static constexpr uint64_t seed = UINT64_C(0xe17a1465); static constexpr unsigned int r = 47; - auto const data64 = static_cast(ptr); + auto const* const data64 = static_cast(ptr); uint64_t h = seed ^ (len * m); size_t const n_blocks = len / 8; @@ -662,7 +709,7 @@ static size_t hash_bytes(void const* ptr, size_t const len) noexcept { h *= m; } - auto const data8 = reinterpret_cast(data64 + n_blocks); + auto const* const data8 = reinterpret_cast(data64 + n_blocks); switch (len & 7U) { case 7: h ^= static_cast(data8[6]) << 48U; @@ -691,65 +738,87 @@ static size_t hash_bytes(void const* ptr, size_t const len) noexcept { } h ^= h >> r; - h *= m; - h ^= h >> r; + + // not doing the final step here, because this will be done by keyToIdx anyways + // h *= m; + // h ^= h >> r; return static_cast(h); } -inline size_t hash_int(uint64_t obj) noexcept { -#if ROBIN_HOOD(HAS_UMUL128) - // 167079903232 masksum, 120428523 ops best: 0xde5fb9d2630458e9 - static constexpr uint64_t k = UINT64_C(0xde5fb9d2630458e9); - uint64_t h; - uint64_t l = detail::umul128(obj, k, &h); - return h + l; -#elif ROBIN_HOOD(BITNESS) == 32 - uint64_t const r = obj * UINT64_C(0xca4bcaa75ec3f625); - auto h = static_cast(r >> 32U); - auto l = static_cast(r); - return h + l; -#else - // murmurhash 3 finalizer - uint64_t h = obj; - h ^= h >> 33; - h *= 0xff51afd7ed558ccd; - h ^= h >> 33; - h *= 0xc4ceb9fe1a85ec53; - h ^= h >> 33; - return static_cast(h); -#endif +inline size_t hash_int(uint64_t x) noexcept { + // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested, + // and doesn't need any special 128bit operations. + x ^= x >> 33U; + x *= UINT64_C(0xff51afd7ed558ccd); + x ^= x >> 33U; + + // not doing the final step here, because this will be done by keyToIdx anyways + // x *= UINT64_C(0xc4ceb9fe1a85ec53); + // x ^= x >> 33U; + return static_cast(x); } // A thin wrapper around std::hash, performing an additional simple mixing step of the result. -template +template struct hash : public std::hash { size_t operator()(T const& obj) const noexcept(noexcept(std::declval>().operator()(std::declval()))) { // call base hash auto result = std::hash::operator()(obj); // return mixed of that, to be save against identity has - return hash_int(static_cast(result)); + return hash_int(static_cast(result)); } }; -template <> -struct hash { - size_t operator()(std::string const& str) const noexcept { - return hash_bytes(str.data(), str.size()); +template +struct hash> { + size_t operator()(std::basic_string const& str) const noexcept { + return hash_bytes(str.data(), sizeof(CharT) * str.size()); } }; +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) +template +struct hash> { + size_t operator()(std::basic_string_view const& sv) const noexcept { + return hash_bytes(sv.data(), sizeof(CharT) * sv.size()); + } +}; +#endif + template struct hash { size_t operator()(T* ptr) const noexcept { - return hash_int(reinterpret_cast(ptr)); + return hash_int(reinterpret_cast(ptr)); + } +}; + +template +struct hash> { + size_t operator()(std::unique_ptr const& ptr) const noexcept { + return hash_int(reinterpret_cast(ptr.get())); + } +}; + +template +struct hash> { + size_t operator()(std::shared_ptr const& ptr) const noexcept { + return hash_int(reinterpret_cast(ptr.get())); + } +}; + +template +struct hash::value>::type> { + size_t operator()(Enum e) const noexcept { + using Underlying = typename std::underlying_type::type; + return hash{}(static_cast(e)); } }; #define ROBIN_HOOD_HASH_INT(T) \ template <> \ struct hash { \ - size_t operator()(T obj) const noexcept { \ + size_t operator()(T const& obj) const noexcept { \ return hash_int(static_cast(obj)); \ } \ } @@ -765,7 +834,9 @@ ROBIN_HOOD_HASH_INT(signed char); ROBIN_HOOD_HASH_INT(unsigned char); ROBIN_HOOD_HASH_INT(char16_t); ROBIN_HOOD_HASH_INT(char32_t); +#if ROBIN_HOOD(HAS_NATIVE_WCHART) ROBIN_HOOD_HASH_INT(wchar_t); +#endif ROBIN_HOOD_HASH_INT(short); ROBIN_HOOD_HASH_INT(unsigned short); ROBIN_HOOD_HASH_INT(int); @@ -779,8 +850,20 @@ ROBIN_HOOD_HASH_INT(unsigned long long); #endif namespace detail { -// using wrapper classes for hash and key_equal prevents the diamond problem when the same type is -// used. see https://stackoverflow.com/a/28771920/48181 +template +struct void_type { + using type = void; +}; + +template +struct has_is_transparent : public std::false_type {}; + +template +struct has_is_transparent::type> + : public std::true_type {}; + +// using wrapper classes for hash and key_equal prevents the diamond problem when the same type +// is used. see https://stackoverflow.com/a/28771920/48181 template struct WrapHash : public T { WrapHash() = default; @@ -797,8 +880,8 @@ struct WrapKeyEqual : public T { // A highly optimized hashmap implementation, using the Robin Hood algorithm. // -// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but be -// about 2x faster in most cases and require much less allocations. +// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but +// be about 2x faster in most cases and require much less allocations. // // This implementation uses the following memory layout: // @@ -806,8 +889,8 @@ struct WrapKeyEqual : public T { // // * Node: either a DataNode that directly has the std::pair as member, // or a DataNode with a pointer to std::pair. Which DataNode representation to use -// depends on how fast the swap() operation is. Heuristically, this is automatically choosen based -// on sizeof(). there are always 2^n Nodes. +// depends on how fast the swap() operation is. Heuristically, this is automatically choosen +// based on sizeof(). there are always 2^n Nodes. // // * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes. // Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the @@ -815,32 +898,38 @@ struct WrapKeyEqual : public T { // actually belongs to the previous position and was pushed out because that place is already // taken. // -// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the need -// for a idx -// variable. +// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the +// need for a idx variable. // -// According to STL, order of templates has effect on throughput. That's why I've moved the boolean -// to the front. +// According to STL, order of templates has effect on throughput. That's why I've moved the +// boolean to the front. // https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ -template -class unordered_map +class Table : public WrapHash, public WrapKeyEqual, detail::NodeAllocator< - robin_hood::pair::type, T>, 4, 16384, - IsFlatMap> { + typename std::conditional< + std::is_void::value, Key, + robin_hood::pair::type, T>>::type, + 4, 16384, IsFlat> { public: + static constexpr bool is_flat = IsFlat; + static constexpr bool is_map = !std::is_void::value; + static constexpr bool is_set = !is_map; + static constexpr bool is_transparent = + has_is_transparent::value && has_is_transparent::value; + using key_type = Key; using mapped_type = T; - using value_type = - robin_hood::pair::type, T>; + using value_type = typename std::conditional< + is_set, Key, + robin_hood::pair::type, T>>::type; using size_type = size_t; using hasher = Hash; using key_equal = KeyEqual; - using Self = - unordered_map; - static constexpr bool is_flat_map = IsFlatMap; + using Self = Table; private: static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, @@ -855,8 +944,9 @@ private: static constexpr size_t InitialNumElements = sizeof(uint64_t); static constexpr uint32_t InitialInfoNumBits = 5; static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; - static constexpr uint8_t InitialInfoHashShift = sizeof(size_t) * 8 - InitialInfoNumBits; - using DataPool = detail::NodeAllocator; + static constexpr size_t InfoMask = InitialInfoInc - 1U; + static constexpr uint8_t InitialInfoHashShift = 0; + using DataPool = detail::NodeAllocator; // type needs to be wider than uint8_t. using InfoType = uint32_t; @@ -864,8 +954,8 @@ private: // DataNode //////////////////////////////////////////////////////// // Primary template for the data node. We have special implementations for small and big - // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these on - // the heap so swap merely swaps a pointer. + // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these + // on the heap so swap merely swaps a pointer. template class DataNode {}; @@ -901,19 +991,38 @@ private: return mData; } - ROBIN_HOOD(NODISCARD) typename value_type::first_type& getFirst() noexcept { + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { return mData.first; } - - ROBIN_HOOD(NODISCARD) typename value_type::first_type const& getFirst() const noexcept { - return mData.first; + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData; } - ROBIN_HOOD(NODISCARD) typename value_type::second_type& getSecond() noexcept { + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type + getFirst() const noexcept { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept { return mData.second; } - ROBIN_HOOD(NODISCARD) typename value_type::second_type const& getSecond() const noexcept { + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept { return mData.second; } @@ -965,19 +1074,38 @@ private: return *mData; } - ROBIN_HOOD(NODISCARD) typename value_type::first_type& getFirst() { + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { return mData->first; } - - ROBIN_HOOD(NODISCARD) typename value_type::first_type const& getFirst() const { - return mData->first; + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return *mData; } - ROBIN_HOOD(NODISCARD) typename value_type::second_type& getSecond() { + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type + getFirst() const noexcept { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept { return mData->second; } - ROBIN_HOOD(NODISCARD) typename value_type::second_type const& getSecond() const { + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept { return mData->second; } @@ -990,7 +1118,26 @@ private: value_type* mData; }; - using Node = DataNode; + using Node = DataNode; + + // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required) + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept { + return n.getFirst(); + } + + // in case we have void mapped_type, we are not using a pair, thus we just route k through. + // No need to disable this because it's just not used if not applicable. + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept { + return k; + } + + // in case we have non-void mapped_type, we have a standard robin_hood::pair + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, key_type const&>::type + getFirstConst(value_type const& vt) const noexcept { + return vt.first; + } // Cloner ////////////////////////////////////////////////////////// @@ -1001,20 +1148,20 @@ private: template struct Cloner { void operator()(M const& source, M& target) const { - // std::memcpy(target.mKeyVals, source.mKeyVals, - // target.calcNumBytesTotal(target.mMask + 1)); - auto src = reinterpret_cast(source.mKeyVals); - auto tgt = reinterpret_cast(target.mKeyVals); - std::copy(src, src + target.calcNumBytesTotal(target.mMask + 1), tgt); + auto const* const src = reinterpret_cast(source.mKeyVals); + auto* tgt = reinterpret_cast(target.mKeyVals); + auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); + std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); } }; template struct Cloner { void operator()(M const& s, M& t) const { - std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(t.mMask + 1), t.mInfo); + auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); + std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); - for (size_t i = 0; i < t.mMask + 1; ++i) { + for (size_t i = 0; i < numElementsWithBuffer; ++i) { if (t.mInfo[i]) { ::new (static_cast(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); } @@ -1024,7 +1171,7 @@ private: // Destroyer /////////////////////////////////////////////////////// - template + template struct Destroyer {}; template @@ -1043,7 +1190,9 @@ private: void nodes(M& m) const noexcept { m.mNumElements = 0; // clear also resets mInfo to 0, that's sometimes not necessary. - for (size_t idx = 0; idx <= m.mMask; ++idx) { + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { if (0 != m.mInfo[idx]) { Node& n = m.mKeyVals[idx]; n.destroy(m); @@ -1055,7 +1204,8 @@ private: void nodesDoNotDeallocate(M& m) const noexcept { m.mNumElements = 0; // clear also resets mInfo to 0, that's sometimes not necessary. - for (size_t idx = 0; idx <= m.mMask; ++idx) { + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { if (0 != m.mInfo[idx]) { Node& n = m.mKeyVals[idx]; n.destroyDoNotDeallocate(); @@ -1087,8 +1237,8 @@ private: // compared to end(). Iter() = default; - // Rule of zero: nothing specified. The conversion constructor is only enabled for iterator - // to const_iterator, so it doesn't accidentally work as a copy ctor. + // Rule of zero: nothing specified. The conversion constructor is only enabled for + // iterator to const_iterator, so it doesn't accidentally work as a copy ctor. // Conversion constructor from iterator to const_iterator. template (mInfo); -#if ROBIN_HOOD(LITTLE_ENDIAN) - inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; + size_t n = 0; + while (0U == (n = detail::unaligned_load(mInfo))) { + mInfo += sizeof(size_t); + mKeyVals += sizeof(size_t); + } +#if defined(ROBIN_HOOD_DISABLE_INTRINSICS) + // we know for certain that within the next 8 bytes we'll find a non-zero one. + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { + mInfo += 4; + mKeyVals += 4; + } + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { + mInfo += 2; + mKeyVals += 2; + } + if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) { + mInfo += 1; + mKeyVals += 1; + } #else - inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; +# if ROBIN_HOOD(LITTLE_ENDIAN) + auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; +# else + auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; +# endif + mInfo += inc; + mKeyVals += inc; #endif - mInfo += inc; - mKeyVals += inc; - } while (inc == static_cast(sizeof(size_t))); } - friend class unordered_map; + friend class Table; NodePtr mKeyVals{nullptr}; uint8_t const* mInfo{nullptr}; }; @@ -1172,22 +1346,22 @@ private: // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. template void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const { - // for a user-specified hash that is *not* robin_hood::hash, apply robin_hood::hash as an - // additional mixing step. This serves as a bad hash prevention, if the given data is badly - // mixed. - using Mix = - typename std::conditional, hasher>::value, - ::robin_hood::detail::identity_hash, - ::robin_hood::hash>::type; - *idx = Mix{}(WHash::operator()(key)); + // In addition to whatever hash is used, add another mul & shift so we get better hashing. + // This serves as a bad hash prevention, if the given data is + // badly mixed. + auto h = static_cast(WHash::operator()(key)); - *info = mInfoInc + static_cast(*idx >> mInfoHashShift); - *idx &= mMask; + h *= mHashMultiplier; + h ^= h >> 33U; + + // the lower InitialInfoNumBits are reserved for info. + *info = mInfoInc + static_cast((h & InfoMask) >> mInfoHashShift); + *idx = (static_cast(h) >> InitialInfoNumBits) & mMask; } // forwards the index by one, wrapping around at the end void next(InfoType* info, size_t* idx) const noexcept { - *idx = (*idx + 1) & mMask; + *idx = *idx + 1; *info += mInfoInc; } @@ -1199,38 +1373,38 @@ private: } // Shift everything up by one element. Tries to move stuff around. - // True if some shifting has occured (entry under idx is a constructed object) - // Fals if no shift has occured (entry under idx is unconstructed memory) void - shiftUp(size_t idx, + shiftUp(size_t startIdx, size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable::value) { + auto idx = startIdx; + ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); + while (--idx != insertion_idx) { + mKeyVals[idx] = std::move(mKeyVals[idx - 1]); + } + + idx = startIdx; while (idx != insertion_idx) { - size_t prev_idx = (idx - 1) & mMask; - if (mInfo[idx]) { - mKeyVals[idx] = std::move(mKeyVals[prev_idx]); - } else { - ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[prev_idx])); - } - mInfo[idx] = static_cast(mInfo[prev_idx] + mInfoInc); + ROBIN_HOOD_COUNT(shiftUp) + mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { mMaxNumElementsAllowed = 0; } - idx = prev_idx; + --idx; } } void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable::value) { // until we find one that is either empty or has zero offset. - // TODO(martinus) we don't need to move everything, just the last one for the same bucket. + // TODO(martinus) we don't need to move everything, just the last one for the same + // bucket. mKeyVals[idx].destroy(*this); // until we find one that is either empty or has zero offset. - size_t nextIdx = (idx + 1) & mMask; - while (mInfo[nextIdx] >= 2 * mInfoInc) { - mInfo[idx] = static_cast(mInfo[nextIdx] - mInfoInc); - mKeyVals[idx] = std::move(mKeyVals[nextIdx]); - idx = nextIdx; - nextIdx = (idx + 1) & mMask; + while (mInfo[idx + 1] >= 2 * mInfoInc) { + ROBIN_HOOD_COUNT(shiftDown) + mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); + mKeyVals[idx] = std::move(mKeyVals[idx + 1]); + ++idx; } mInfo[idx] = 0; @@ -1243,46 +1417,50 @@ private: template ROBIN_HOOD(NODISCARD) size_t findIdx(Other const& key) const { - size_t idx; - InfoType info; + size_t idx{}; + InfoType info{}; keyToIdx(key, &idx, &info); do { // unrolling this twice gives a bit of a speedup. More unrolling did not help. - if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { return idx; } next(&info, &idx); - if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { return idx; } next(&info, &idx); } while (info <= mInfo[idx]); // nothing found! - return mMask == 0 ? 0 : mMask + 1; + return mMask == 0 ? 0 + : static_cast(std::distance( + mKeyVals, reinterpret_cast_no_cast_align_warning(mInfo))); } - void cloneData(const unordered_map& o) { - Cloner()(o, *this); + void cloneData(const Table& o) { + Cloner()(o, *this); } // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. - // @return index where the element was created - size_t insert_move(Node&& keyval) { + // @return True on success, false if something went wrong + void insert_move(Node&& keyval) { // we don't retry, fail if overflowing // don't need to check max num elements if (0 == mMaxNumElementsAllowed && !try_increase_info()) { throwOverflowError(); } - size_t idx; - InfoType info; + size_t idx{}; + InfoType info{}; keyToIdx(keyval.getFirst(), &idx, &info); // skip forward. Use <= because we are certain that the element is not there. while (info <= mInfo[idx]) { - idx = (idx + 1) & mMask; + idx = idx + 1; info += mInfoInc; } @@ -1310,51 +1488,56 @@ private: mInfo[insertion_idx] = insertion_info; ++mNumElements; - return insertion_idx; } public: using iterator = Iter; using const_iterator = Iter; - // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. This - // tremendously speeds up ctor & dtor of a map that never receives an element. The penalty is - // payed at the first insert, and not before. Lookup of this empty map works because everybody - // points to DummyInfoByte::b. parameter bucket_count is dictated by the standard, but we can - // ignore it. - explicit unordered_map(size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, - const Hash& h = Hash{}, - const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && - noexcept(KeyEqual(equal))) + Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual())) + : WHash() + , WKeyEqual() { + ROBIN_HOOD_TRACE(this) + } + + // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. + // This tremendously speeds up ctor & dtor of a map that never receives an element. The + // penalty is payed at the first insert, and not before. Lookup of this empty map works + // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the + // standard, but we can ignore it. + explicit Table( + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) : WHash(h) , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) } template - unordered_map(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, - const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) + Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, + const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) : WHash(h) , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) insert(first, last); } - unordered_map(std::initializer_list initlist, - size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, - const KeyEqual& equal = KeyEqual{}) + Table(std::initializer_list initlist, + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) : WHash(h) , WKeyEqual(equal) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) insert(initlist.begin(), initlist.end()); } - unordered_map(unordered_map&& o) noexcept + Table(Table&& o) noexcept : WHash(std::move(static_cast(o))) , WKeyEqual(std::move(static_cast(o))) , DataPool(std::move(static_cast(o))) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (o.mMask) { + mHashMultiplier = std::move(o.mHashMultiplier); mKeyVals = std::move(o.mKeyVals); mInfo = std::move(o.mInfo); mNumElements = std::move(o.mNumElements); @@ -1367,12 +1550,13 @@ public: } } - unordered_map& operator=(unordered_map&& o) noexcept { - ROBIN_HOOD_TRACE(this); + Table& operator=(Table&& o) noexcept { + ROBIN_HOOD_TRACE(this) if (&o != this) { if (o.mMask) { // only move stuff if the other map actually has some data destroy(); + mHashMultiplier = std::move(o.mHashMultiplier); mKeyVals = std::move(o.mKeyVals); mInfo = std::move(o.mInfo); mNumElements = std::move(o.mNumElements); @@ -1394,19 +1578,25 @@ public: return *this; } - unordered_map(const unordered_map& o) + Table(const Table& o) : WHash(static_cast(o)) , WKeyEqual(static_cast(o)) , DataPool(static_cast(o)) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (!o.empty()) { // not empty: create an exact copy. it is also possible to just iterate through all // elements and insert them, but copying is probably faster. + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + + ROBIN_HOOD_LOG("malloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mHashMultiplier = o.mHashMultiplier; mKeyVals = static_cast( - detail::assertNotNull(malloc(calcNumBytesTotal(o.mMask + 1)))); + detail::assertNotNull(malloc(numBytesTotal))); // no need for calloc because clonData does memcpy - mInfo = reinterpret_cast(mKeyVals + o.mMask + 1); + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); mNumElements = o.mNumElements; mMask = o.mMask; mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; @@ -1417,15 +1607,17 @@ public: } // Creates a copy of the given map. Copy constructor of each entry is used. - unordered_map& operator=(unordered_map const& o) { - ROBIN_HOOD_TRACE(this); + // Not sure why clang-tidy thinks this doesn't handle self assignment, it does + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + Table& operator=(Table const& o) { + ROBIN_HOOD_TRACE(this) if (&o == this) { // prevent assigning of itself return *this; } - // we keep using the old allocator and not assign the new one, because we want to keep the - // memory available. when it is the same size. + // we keep using the old allocator and not assign the new one, because we want to keep + // the memory available. when it is the same size. if (o.empty()) { if (0 == mMask) { // nothing to do, we are empty too @@ -1444,25 +1636,31 @@ public: } // clean up old stuff - Destroyer::value>{}.nodes(*this); + Destroyer::value>{}.nodes(*this); if (mMask != o.mMask) { // no luck: we don't have the same array size allocated, so we need to realloc. if (0 != mMask) { // only deallocate if we actually have data! - free(mKeyVals); + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); } + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("malloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") mKeyVals = static_cast( - detail::assertNotNull(malloc(calcNumBytesTotal(o.mMask + 1)))); + detail::assertNotNull(malloc(numBytesTotal))); // no need for calloc here because cloneData performs a memcpy. - mInfo = reinterpret_cast(mKeyVals + o.mMask + 1); + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); // sentinel is set in cloneData } WHash::operator=(static_cast(o)); WKeyEqual::operator=(static_cast(o)); DataPool::operator=(static_cast(o)); + mHashMultiplier = o.mHashMultiplier; mNumElements = o.mNumElements; mMask = o.mMask; mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; @@ -1474,47 +1672,47 @@ public: } // Swaps everything between the two maps. - void swap(unordered_map& o) { - ROBIN_HOOD_TRACE(this); + void swap(Table& o) { + ROBIN_HOOD_TRACE(this) using std::swap; swap(o, *this); } // Clears all data, without resizing. void clear() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (empty()) { - // don't do anything! also important because we don't want to write to DummyInfoByte::b, - // even though we would just write 0 to it. + // don't do anything! also important because we don't want to write to + // DummyInfoByte::b, even though we would just write 0 to it. return; } - Destroyer::value>{}.nodes(*this); + Destroyer::value>{}.nodes(*this); - // clear everything except the sentinel - // std::memset(mInfo, 0, sizeof(uint8_t) * (mMask + 1)); + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + // clear everything, then set the sentinel again uint8_t const z = 0; - std::fill(mInfo, mInfo + (sizeof(uint8_t) * (mMask + 1)), z); + std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); + mInfo[numElementsWithBuffer] = 1; mInfoInc = InitialInfoInc; mInfoHashShift = InitialInfoHashShift; } // Destroys the map and all it's contents. - ~unordered_map() { - ROBIN_HOOD_TRACE(this); + ~Table() { + ROBIN_HOOD_TRACE(this) destroy(); } - // Checks if both maps contain the same entries. Order is irrelevant. - bool operator==(const unordered_map& other) const { - ROBIN_HOOD_TRACE(this); + // Checks if both tables contain the same entries. Order is irrelevant. + bool operator==(const Table& other) const { + ROBIN_HOOD_TRACE(this) if (other.size() != size()) { return false; } for (auto const& otherEntry : other) { - auto const myIt = find(otherEntry.first); - if (myIt == end() || !(myIt->second == otherEntry.second)) { + if (!has(otherEntry)) { return false; } } @@ -1522,19 +1720,62 @@ public: return true; } - bool operator!=(const unordered_map& other) const { - ROBIN_HOOD_TRACE(this); + bool operator!=(const Table& other) const { + ROBIN_HOOD_TRACE(this) return !operator==(other); } - mapped_type& operator[](const key_type& key) { - ROBIN_HOOD_TRACE(this); - return doCreateByKey(key); + template + typename std::enable_if::value, Q&>::type operator[](const key_type& key) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) + Node(*this, std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple()); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + } + + return mKeyVals[idxAndState.first].getSecond(); } - mapped_type& operator[](key_type&& key) { - ROBIN_HOOD_TRACE(this); - return doCreateByKey(std::move(key)); + template + typename std::enable_if::value, Q&>::type operator[](key_type&& key) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), + std::forward_as_tuple()); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), + std::forward_as_tuple()); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + } + + return mKeyVals[idxAndState.first].getSecond(); } template @@ -1545,31 +1786,98 @@ public: } } + void insert(std::initializer_list ilist) { + for (auto&& vt : ilist) { + insert(std::move(vt)); + } + } + template std::pair emplace(Args&&... args) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) Node n{*this, std::forward(args)...}; - auto r = doInsert(std::move(n)); - if (!r.second) { - // insertion not possible: destroy node - // NOLINTNEXTLINE(bugprone-use-after-move) + auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n)); + switch (idxAndState.second) { + case InsertionState::key_found: n.destroy(*this); + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) Node(*this, std::move(n)); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = std::move(n); + break; + + case InsertionState::overflow_error: + n.destroy(*this); + throwOverflowError(); + break; } - return r; + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + template + std::pair try_emplace(const key_type& key, Args&&... args) { + return try_emplace_impl(key, std::forward(args)...); + } + + template + std::pair try_emplace(key_type&& key, Args&&... args) { + return try_emplace_impl(std::move(key), std::forward(args)...); + } + + template + std::pair try_emplace(const_iterator hint, const key_type& key, + Args&&... args) { + (void)hint; + return try_emplace_impl(key, std::forward(args)...); + } + + template + std::pair try_emplace(const_iterator hint, key_type&& key, Args&&... args) { + (void)hint; + return try_emplace_impl(std::move(key), std::forward(args)...); + } + + template + std::pair insert_or_assign(const key_type& key, Mapped&& obj) { + return insertOrAssignImpl(key, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type&& key, Mapped&& obj) { + return insertOrAssignImpl(std::move(key), std::forward(obj)); + } + + template + std::pair insert_or_assign(const_iterator hint, const key_type& key, + Mapped&& obj) { + (void)hint; + return insertOrAssignImpl(key, std::forward(obj)); + } + + template + std::pair insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) { + (void)hint; + return insertOrAssignImpl(std::move(key), std::forward(obj)); } std::pair insert(const value_type& keyval) { - ROBIN_HOOD_TRACE(this); - return doInsert(keyval); + ROBIN_HOOD_TRACE(this) + return emplace(keyval); } std::pair insert(value_type&& keyval) { - return doInsert(std::move(keyval)); + return emplace(std::move(keyval)); } // Returns 1 if key is found, 0 otherwise. size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { return 1; @@ -1577,10 +1885,33 @@ public: return 0; } + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type count(const OtherKey& key) const { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { + return 1; + } + return 0; + } + + bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + return 1U == count(key); + } + + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type contains(const OtherKey& key) const { + return 1U == count(key); + } + // Returns a reference to the value found for key. // Throws std::out_of_range if element cannot be found - mapped_type& at(key_type const& key) { - ROBIN_HOOD_TRACE(this); + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q&>::type at(key_type const& key) { + ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { doThrow("key not found"); @@ -1590,8 +1921,10 @@ public: // Returns a reference to the value found for key. // Throws std::out_of_range if element cannot be found - mapped_type const& at(key_type const& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q const&>::type at(key_type const& key) const { + ROBIN_HOOD_TRACE(this) auto kv = mKeyVals + findIdx(key); if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { doThrow("key not found"); @@ -1600,44 +1933,60 @@ public: } const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } template const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type // NOLINT(modernize-use-nodiscard) + find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return const_iterator{mKeyVals + idx, mInfo + idx}; } iterator find(const key_type& key) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } template iterator find(const OtherKey& key, is_transparent_tag /*unused*/) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type find(const OtherKey& key) { + ROBIN_HOOD_TRACE(this) const size_t idx = findIdx(key); return iterator{mKeyVals + idx, mInfo + idx}; } iterator begin() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (empty()) { return end(); } return iterator(mKeyVals, mInfo, fast_forward_tag{}); } const_iterator begin() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return cbegin(); } const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) if (empty()) { return cend(); } @@ -1645,22 +1994,22 @@ public: } iterator end() { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) // no need to supply valid info pointer: end() must not be dereferenced, and only node // pointer is compared. return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; } const_iterator end() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return cend(); } const_iterator cend() const { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; } iterator erase(const_iterator pos) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) // its safe to perform const cast here // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); @@ -1668,7 +2017,7 @@ public: // Erases element at pos, returns iterator to the next element. iterator erase(iterator pos) { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) // we assume that pos always points to a valid entry, and not end(). auto const idx = static_cast(pos.mKeyVals - mKeyVals); @@ -1685,9 +2034,9 @@ public: } size_t erase(const key_type& key) { - ROBIN_HOOD_TRACE(this); - size_t idx; - InfoType info; + ROBIN_HOOD_TRACE(this) + size_t idx{}; + InfoType info{}; keyToIdx(key, &idx, &info); // check while info matches with the source idx @@ -1707,53 +2056,66 @@ public: // reserves space for the specified number of elements. Makes sure the old data fits. // exactly the same as reserve(c). void rehash(size_t c) { - reserve(c); + // forces a reserve + reserve(c, true); } // reserves space for the specified number of elements. Makes sure the old data fits. - // Exactly the same as resize(c). Use resize(0) to shrink to fit. + // Exactly the same as rehash(c). Use rehash(0) to shrink to fit. void reserve(size_t c) { - ROBIN_HOOD_TRACE(this); - auto const minElementsAllowed = (std::max)(c, mNumElements); + // reserve, but don't force rehash + reserve(c, false); + } + + // If possible reallocates the map to a smaller one. This frees the underlying table. + // Does not do anything if load_factor is too large for decreasing the table's size. + void compact() { + ROBIN_HOOD_TRACE(this) auto newSize = InitialNumElements; - while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { + while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) { newSize *= 2; } if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { throwOverflowError(); } - rehashPowerOfTwo(newSize); + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") + + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (newSize < mMask + 1) { + rehashPowerOfTwo(newSize, true); + } } size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return mNumElements; } size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return static_cast(-1); } ROBIN_HOOD(NODISCARD) bool empty() const noexcept { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return 0 == mNumElements; } float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return MaxLoadFactor100 / 100.0F; } // Average number of elements per bucket. Since we allow only 1 per bucket float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return static_cast(size()) / static_cast(mMask + 1); } ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { - ROBIN_HOOD_TRACE(this); + ROBIN_HOOD_TRACE(this) return mMask; } @@ -1766,11 +2128,19 @@ public: return (maxElements / 100) * MaxLoadFactor100; } - ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const { + ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept { + // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load + // 64bit types. return numElements + sizeof(uint64_t); } - // calculation ony allowed for 2^n values + ROBIN_HOOD(NODISCARD) + size_t calcNumElementsWithBuffer(size_t numElements) const noexcept { + auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); + return numElements + (std::min)(maxNumElementsAllowed, (static_cast(0xFF))); + } + + // calculation only allowed for 2^n values ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const { #if ROBIN_HOOD(BITNESS) == 64 return numElements * sizeof(Node) + calcNumBytesInfo(numElements); @@ -1791,29 +2161,76 @@ public: } private: + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + auto it = find(e.first); + return it != end() && it->second == e.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + return find(e) != end(); + } + + void reserve(size_t c, bool forceRehash) { + ROBIN_HOOD_TRACE(this) + auto const minElementsAllowed = (std::max)(c, mNumElements); + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { + throwOverflowError(); + } + + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") + + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (forceRehash || newSize > mMask + 1) { + rehashPowerOfTwo(newSize, false); + } + } + // reserves space for at least the specified number of elements. // only works if numBuckets if power of two - void rehashPowerOfTwo(size_t numBuckets) { - ROBIN_HOOD_TRACE(this); + // True on success, false otherwise + void rehashPowerOfTwo(size_t numBuckets, bool forceFree) { + ROBIN_HOOD_TRACE(this) Node* const oldKeyVals = mKeyVals; uint8_t const* const oldInfo = mInfo; - const size_t oldMaxElements = mMask + 1; + const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); // resize operation: move stuff - init_data(numBuckets); - if (oldMaxElements > 1) { - for (size_t i = 0; i < oldMaxElements; ++i) { + initData(numBuckets); + if (oldMaxElementsWithBuffer > 1) { + for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) { if (oldInfo[i] != 0) { + // might throw an exception, which is really bad since we are in the middle of + // moving stuff. insert_move(std::move(oldKeyVals[i])); // destroy the node but DON'T destroy the data. oldKeyVals[i].~Node(); } } - // don't destroy old data: put it into the pool instead - DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElements)); + // this check is not necessary as it's guarded by the previous if, but it helps + // silence g++'s overeager "attempt to free a non-heap object 'map' + // [-Werror=free-nonheap-object]" warning. + if (oldKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { + // don't destroy old data: put it into the pool instead + if (forceFree) { + std::free(oldKeyVals); + } else { + DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); + } + } } } @@ -1825,103 +2242,115 @@ private: #endif } - void init_data(size_t max_elements) { + template + std::pair try_emplace_impl(OtherKey&& key, Args&&... args) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) Node( + *this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + template + std::pair insertOrAssignImpl(OtherKey&& key, Mapped&& obj) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + mKeyVals[idxAndState.first].getSecond() = std::forward(obj); + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) Node( + *this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(obj))); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(obj))); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + void initData(size_t max_elements) { mNumElements = 0; mMask = max_elements - 1; mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); + auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); + // calloc also zeroes everything + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") mKeyVals = reinterpret_cast( - detail::assertNotNull(calloc(1, calcNumBytesTotal(max_elements)))); - mInfo = reinterpret_cast(mKeyVals + max_elements); + detail::assertNotNull(std::calloc(1, numBytesTotal))); + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); // set sentinel - mInfo[max_elements] = 1; + mInfo[numElementsWithBuffer] = 1; mInfoInc = InitialInfoInc; mInfoHashShift = InitialInfoHashShift; } - template - mapped_type& doCreateByKey(Arg&& key) { - while (true) { - size_t idx; - InfoType info; + enum class InsertionState { overflow_error, key_found, new_node, overwrite_node }; + + // Finds key, and if not already present prepares a spot where to pot the key & value. + // This potentially shifts nodes out of the way, updates mInfo and number of inserted + // elements, so the only operation left to do is create/assign a new node at that spot. + template + std::pair insertKeyPrepareEmptySpot(OtherKey&& key) { + for (int i = 0; i < 256; ++i) { + size_t idx{}; + InfoType info{}; keyToIdx(key, &idx, &info); nextWhileLess(&info, &idx); - // while we potentially have a match. Can't do a do-while here because when mInfo is 0 - // we don't want to skip forward - while (info == mInfo[idx]) { - if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { - // key already exists, do not insert. - return mKeyVals[idx].getSecond(); - } - next(&info, &idx); - } - - // unlikely that this evaluates to true - if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { - increase_size(); - continue; - } - - // key not found, so we are now exactly where we want to insert it. - auto const insertion_idx = idx; - auto const insertion_info = info; - if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { - mMaxNumElementsAllowed = 0; - } - - // find an empty spot - while (0 != mInfo[idx]) { - next(&info, &idx); - } - - auto& l = mKeyVals[insertion_idx]; - if (idx == insertion_idx) { - // put at empty spot. This forwards all arguments into the node where the object is - // constructed exactly where it is needed. - ::new (static_cast(&l)) - Node(*this, std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); - } else { - shiftUp(idx, insertion_idx); - l = Node(*this, std::piecewise_construct, - std::forward_as_tuple(std::forward(key)), std::forward_as_tuple()); - } - - // mKeyVals[idx].getFirst() = std::move(key); - mInfo[insertion_idx] = static_cast(insertion_info); - - ++mNumElements; - return mKeyVals[insertion_idx].getSecond(); - } - } - - // This is exactly the same code as operator[], except for the return values - template - std::pair doInsert(Arg&& keyval) { - while (true) { - size_t idx; - InfoType info; - keyToIdx(keyval.getFirst(), &idx, &info); - nextWhileLess(&info, &idx); - // while we potentially have a match while (info == mInfo[idx]) { - if (WKeyEqual::operator()(keyval.getFirst(), mKeyVals[idx].getFirst())) { + if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { // key already exists, do NOT insert. // see http://en.cppreference.com/w/cpp/container/unordered_map/insert - return std::make_pair(iterator(mKeyVals + idx, mInfo + idx), - false); + return std::make_pair(idx, InsertionState::key_found); } next(&info, &idx); } // unlikely that this evaluates to true if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { - increase_size(); + if (!increase_size()) { + return std::make_pair(size_t(0), InsertionState::overflow_error); + } continue; } @@ -1937,26 +2366,25 @@ private: next(&info, &idx); } - auto& l = mKeyVals[insertion_idx]; - if (idx == insertion_idx) { - ::new (static_cast(&l)) Node(*this, std::forward(keyval)); - } else { + if (idx != insertion_idx) { shiftUp(idx, insertion_idx); - l = Node(*this, std::forward(keyval)); } - // put at empty spot mInfo[insertion_idx] = static_cast(insertion_info); - ++mNumElements; - return std::make_pair(iterator(mKeyVals + insertion_idx, mInfo + insertion_idx), true); + return std::make_pair(insertion_idx, idx == insertion_idx + ? InsertionState::new_node + : InsertionState::overwrite_node); } + + // enough attempts failed, so finally give up. + return std::make_pair(size_t(0), InsertionState::overflow_error); } bool try_increase_info() { ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements << ", maxNumElementsAllowed=" - << calcMaxNumElementsAllowed(mMask + 1)); + << calcMaxNumElementsAllowed(mMask + 1)) if (mInfoInc <= 2) { // need to be > 2 so that shift works (otherwise undefined behavior!) return false; @@ -1967,38 +2395,55 @@ private: // remove one bit of the hash, leaving more space for the distance info. // This is extremely fast because we can operate on 8 bytes at once. ++mInfoHashShift; - auto const data = reinterpret_cast_no_cast_align_warning(mInfo); - auto const numEntries = (mMask + 1) / 8; + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); - for (size_t i = 0; i < numEntries; ++i) { - data[i] = (data[i] >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); + for (size_t i = 0; i < numElementsWithBuffer; i += 8) { + auto val = unaligned_load(mInfo + i); + val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); + std::memcpy(mInfo + i, &val, sizeof(val)); } + // update sentinel, which might have been cleared out! + mInfo[numElementsWithBuffer] = 1; + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); return true; } - void increase_size() { + // True if resize was possible, false otherwise + bool increase_size() { // nothing allocated yet? just allocate InitialNumElements if (0 == mMask) { - init_data(InitialNumElements); - return; + initData(InitialNumElements); + return true; } auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); if (mNumElements < maxNumElementsAllowed && try_increase_info()) { - return; + return true; } ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" << maxNumElementsAllowed << ", load=" << (static_cast(mNumElements) * 100.0 / - (static_cast(mMask) + 1))); - // it seems we have a really bad hash function! don't try to resize again - if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { - throwOverflowError(); - } + (static_cast(mMask) + 1))) - rehashPowerOfTwo((mMask + 1) * 2); + if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { + // we have to resize, even though there would still be plenty of space left! + // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case + // we have to rehash a few times + nextHashMultiplier(); + rehashPowerOfTwo(mMask + 1, true); + } else { + // we've reached the capacity of the map, so the hash seems to work nice. Keep using it. + rehashPowerOfTwo((mMask + 1) * 2, false); + } + return true; + } + + void nextHashMultiplier() { + // adding an *even* number, so that the multiplier will always stay odd. This is necessary + // so that the hash stays a mixing function (and thus doesn't have any information loss). + mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54); } void destroy() { @@ -2007,13 +2452,21 @@ private: return; } - Destroyer::value>{} + Destroyer::value>{} .nodesDoNotDeallocate(*this); - free(mKeyVals); + + // This protection against not deleting mMask shouldn't be needed as it's sufficiently + // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise + // reports a compile error: attempt to free a non-heap object 'fm' + // [-Werror=free-nonheap-object] + if (mKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } } void init() noexcept { - mKeyVals = reinterpret_cast(&mMask); + mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); mInfo = reinterpret_cast(&mMask); mNumElements = 0; mMask = 0; @@ -2023,33 +2476,53 @@ private: } // members are sorted so no padding occurs - Node* mKeyVals = reinterpret_cast(&mMask); // 8 byte 8 - uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 16 - size_t mNumElements = 0; // 8 byte 24 - size_t mMask = 0; // 8 byte 32 - size_t mMaxNumElementsAllowed = 0; // 8 byte 40 - InfoType mInfoInc = InitialInfoInc; // 4 byte 44 - InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 48 - // 16 byte 56 if NodeAllocator + uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8 + Node* mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); // 8 byte 16 + uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 24 + size_t mNumElements = 0; // 8 byte 32 + size_t mMask = 0; // 8 byte 40 + size_t mMaxNumElementsAllowed = 0; // 8 byte 48 + InfoType mInfoInc = InitialInfoInc; // 4 byte 52 + InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56 + // 16 byte 56 if NodeAllocator }; } // namespace detail -template , - typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> -using unordered_flat_map = detail::unordered_map; +// map template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> -using unordered_node_map = detail::unordered_map; +using unordered_flat_map = detail::Table; + +template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +using unordered_node_map = detail::Table; template , typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> using unordered_map = - detail::unordered_map) <= sizeof(size_t) * 6 && - std::is_nothrow_move_constructible>::value && - std::is_nothrow_move_assignable>::value, - MaxLoadFactor100, Key, T, Hash, KeyEqual>; + detail::Table) <= sizeof(size_t) * 6 && + std::is_nothrow_move_constructible>::value && + std::is_nothrow_move_assignable>::value, + MaxLoadFactor100, Key, T, Hash, KeyEqual>; + +// set + +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_flat_set = detail::Table; + +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_node_set = detail::Table; + +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_set = detail::Table::value && + std::is_nothrow_move_assignable::value, + MaxLoadFactor100, Key, void, Hash, KeyEqual>; } // namespace robin_hood