Details | Last modification | View Log | RSS feed
| Rev | Author | Line No. | Line |
|---|---|---|---|
| 14 | pmbaty | 1 | //===- llvm/ADT/DenseMap.h - Dense probed hash table ------------*- C++ -*-===// |
| 2 | // |
||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
||
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
||
| 6 | // |
||
| 7 | //===----------------------------------------------------------------------===// |
||
| 8 | /// |
||
| 9 | /// \file |
||
| 10 | /// This file defines the DenseMap class. |
||
| 11 | /// |
||
| 12 | //===----------------------------------------------------------------------===// |
||
| 13 | |||
| 14 | #ifndef LLVM_ADT_DENSEMAP_H |
||
| 15 | #define LLVM_ADT_DENSEMAP_H |
||
| 16 | |||
| 17 | #include "llvm/ADT/DenseMapInfo.h" |
||
| 18 | #include "llvm/ADT/EpochTracker.h" |
||
| 19 | #include "llvm/Support/AlignOf.h" |
||
| 20 | #include "llvm/Support/Compiler.h" |
||
| 21 | #include "llvm/Support/MathExtras.h" |
||
| 22 | #include "llvm/Support/MemAlloc.h" |
||
| 23 | #include "llvm/Support/ReverseIteration.h" |
||
| 24 | #include "llvm/Support/type_traits.h" |
||
| 25 | #include <algorithm> |
||
| 26 | #include <cassert> |
||
| 27 | #include <cstddef> |
||
| 28 | #include <cstring> |
||
| 29 | #include <initializer_list> |
||
| 30 | #include <iterator> |
||
| 31 | #include <new> |
||
| 32 | #include <type_traits> |
||
| 33 | #include <utility> |
||
| 34 | |||
| 35 | namespace llvm { |
||
| 36 | |||
| 37 | namespace detail { |
||
| 38 | |||
| 39 | // We extend a pair to allow users to override the bucket type with their own |
||
| 40 | // implementation without requiring two members. |
||
| 41 | template <typename KeyT, typename ValueT> |
||
| 42 | struct DenseMapPair : public std::pair<KeyT, ValueT> { |
||
| 43 | using std::pair<KeyT, ValueT>::pair; |
||
| 44 | |||
| 45 | KeyT &getFirst() { return std::pair<KeyT, ValueT>::first; } |
||
| 46 | const KeyT &getFirst() const { return std::pair<KeyT, ValueT>::first; } |
||
| 47 | ValueT &getSecond() { return std::pair<KeyT, ValueT>::second; } |
||
| 48 | const ValueT &getSecond() const { return std::pair<KeyT, ValueT>::second; } |
||
| 49 | }; |
||
| 50 | |||
| 51 | } // end namespace detail |
||
| 52 | |||
| 53 | template <typename KeyT, typename ValueT, |
||
| 54 | typename KeyInfoT = DenseMapInfo<KeyT>, |
||
| 55 | typename Bucket = llvm::detail::DenseMapPair<KeyT, ValueT>, |
||
| 56 | bool IsConst = false> |
||
| 57 | class DenseMapIterator; |
||
| 58 | |||
| 59 | template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, |
||
| 60 | typename BucketT> |
||
| 61 | class DenseMapBase : public DebugEpochBase { |
||
| 62 | template <typename T> |
||
| 63 | using const_arg_type_t = typename const_pointer_or_const_ref<T>::type; |
||
| 64 | |||
| 65 | public: |
||
| 66 | using size_type = unsigned; |
||
| 67 | using key_type = KeyT; |
||
| 68 | using mapped_type = ValueT; |
||
| 69 | using value_type = BucketT; |
||
| 70 | |||
| 71 | using iterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT>; |
||
| 72 | using const_iterator = |
||
| 73 | DenseMapIterator<KeyT, ValueT, KeyInfoT, BucketT, true>; |
||
| 74 | |||
| 75 | inline iterator begin() { |
||
| 76 | // When the map is empty, avoid the overhead of advancing/retreating past |
||
| 77 | // empty buckets. |
||
| 78 | if (empty()) |
||
| 79 | return end(); |
||
| 80 | if (shouldReverseIterate<KeyT>()) |
||
| 81 | return makeIterator(getBucketsEnd() - 1, getBuckets(), *this); |
||
| 82 | return makeIterator(getBuckets(), getBucketsEnd(), *this); |
||
| 83 | } |
||
| 84 | inline iterator end() { |
||
| 85 | return makeIterator(getBucketsEnd(), getBucketsEnd(), *this, true); |
||
| 86 | } |
||
| 87 | inline const_iterator begin() const { |
||
| 88 | if (empty()) |
||
| 89 | return end(); |
||
| 90 | if (shouldReverseIterate<KeyT>()) |
||
| 91 | return makeConstIterator(getBucketsEnd() - 1, getBuckets(), *this); |
||
| 92 | return makeConstIterator(getBuckets(), getBucketsEnd(), *this); |
||
| 93 | } |
||
| 94 | inline const_iterator end() const { |
||
| 95 | return makeConstIterator(getBucketsEnd(), getBucketsEnd(), *this, true); |
||
| 96 | } |
||
| 97 | |||
| 98 | [[nodiscard]] bool empty() const { return getNumEntries() == 0; } |
||
| 99 | unsigned size() const { return getNumEntries(); } |
||
| 100 | |||
| 101 | /// Grow the densemap so that it can contain at least \p NumEntries items |
||
| 102 | /// before resizing again. |
||
| 103 | void reserve(size_type NumEntries) { |
||
| 104 | auto NumBuckets = getMinBucketToReserveForEntries(NumEntries); |
||
| 105 | incrementEpoch(); |
||
| 106 | if (NumBuckets > getNumBuckets()) |
||
| 107 | grow(NumBuckets); |
||
| 108 | } |
||
| 109 | |||
| 110 | void clear() { |
||
| 111 | incrementEpoch(); |
||
| 112 | if (getNumEntries() == 0 && getNumTombstones() == 0) return; |
||
| 113 | |||
| 114 | // If the capacity of the array is huge, and the # elements used is small, |
||
| 115 | // shrink the array. |
||
| 116 | if (getNumEntries() * 4 < getNumBuckets() && getNumBuckets() > 64) { |
||
| 117 | shrink_and_clear(); |
||
| 118 | return; |
||
| 119 | } |
||
| 120 | |||
| 121 | const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); |
||
| 122 | if (std::is_trivially_destructible<ValueT>::value) { |
||
| 123 | // Use a simpler loop when values don't need destruction. |
||
| 124 | for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) |
||
| 125 | P->getFirst() = EmptyKey; |
||
| 126 | } else { |
||
| 127 | unsigned NumEntries = getNumEntries(); |
||
| 128 | for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { |
||
| 129 | if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { |
||
| 130 | if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { |
||
| 131 | P->getSecond().~ValueT(); |
||
| 132 | --NumEntries; |
||
| 133 | } |
||
| 134 | P->getFirst() = EmptyKey; |
||
| 135 | } |
||
| 136 | } |
||
| 137 | assert(NumEntries == 0 && "Node count imbalance!"); |
||
| 138 | (void)NumEntries; |
||
| 139 | } |
||
| 140 | setNumEntries(0); |
||
| 141 | setNumTombstones(0); |
||
| 142 | } |
||
| 143 | |||
| 144 | /// Return 1 if the specified key is in the map, 0 otherwise. |
||
| 145 | size_type count(const_arg_type_t<KeyT> Val) const { |
||
| 146 | const BucketT *TheBucket; |
||
| 147 | return LookupBucketFor(Val, TheBucket) ? 1 : 0; |
||
| 148 | } |
||
| 149 | |||
| 150 | iterator find(const_arg_type_t<KeyT> Val) { |
||
| 151 | BucketT *TheBucket; |
||
| 152 | if (LookupBucketFor(Val, TheBucket)) |
||
| 153 | return makeIterator(TheBucket, |
||
| 154 | shouldReverseIterate<KeyT>() ? getBuckets() |
||
| 155 | : getBucketsEnd(), |
||
| 156 | *this, true); |
||
| 157 | return end(); |
||
| 158 | } |
||
| 159 | const_iterator find(const_arg_type_t<KeyT> Val) const { |
||
| 160 | const BucketT *TheBucket; |
||
| 161 | if (LookupBucketFor(Val, TheBucket)) |
||
| 162 | return makeConstIterator(TheBucket, |
||
| 163 | shouldReverseIterate<KeyT>() ? getBuckets() |
||
| 164 | : getBucketsEnd(), |
||
| 165 | *this, true); |
||
| 166 | return end(); |
||
| 167 | } |
||
| 168 | |||
| 169 | /// Alternate version of find() which allows a different, and possibly |
||
| 170 | /// less expensive, key type. |
||
| 171 | /// The DenseMapInfo is responsible for supplying methods |
||
| 172 | /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key |
||
| 173 | /// type used. |
||
| 174 | template<class LookupKeyT> |
||
| 175 | iterator find_as(const LookupKeyT &Val) { |
||
| 176 | BucketT *TheBucket; |
||
| 177 | if (LookupBucketFor(Val, TheBucket)) |
||
| 178 | return makeIterator(TheBucket, |
||
| 179 | shouldReverseIterate<KeyT>() ? getBuckets() |
||
| 180 | : getBucketsEnd(), |
||
| 181 | *this, true); |
||
| 182 | return end(); |
||
| 183 | } |
||
| 184 | template<class LookupKeyT> |
||
| 185 | const_iterator find_as(const LookupKeyT &Val) const { |
||
| 186 | const BucketT *TheBucket; |
||
| 187 | if (LookupBucketFor(Val, TheBucket)) |
||
| 188 | return makeConstIterator(TheBucket, |
||
| 189 | shouldReverseIterate<KeyT>() ? getBuckets() |
||
| 190 | : getBucketsEnd(), |
||
| 191 | *this, true); |
||
| 192 | return end(); |
||
| 193 | } |
||
| 194 | |||
| 195 | /// lookup - Return the entry for the specified key, or a default |
||
| 196 | /// constructed value if no such entry exists. |
||
| 197 | ValueT lookup(const_arg_type_t<KeyT> Val) const { |
||
| 198 | const BucketT *TheBucket; |
||
| 199 | if (LookupBucketFor(Val, TheBucket)) |
||
| 200 | return TheBucket->getSecond(); |
||
| 201 | return ValueT(); |
||
| 202 | } |
||
| 203 | |||
| 204 | // Inserts key,value pair into the map if the key isn't already in the map. |
||
| 205 | // If the key is already in the map, it returns false and doesn't update the |
||
| 206 | // value. |
||
| 207 | std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) { |
||
| 208 | return try_emplace(KV.first, KV.second); |
||
| 209 | } |
||
| 210 | |||
| 211 | // Inserts key,value pair into the map if the key isn't already in the map. |
||
| 212 | // If the key is already in the map, it returns false and doesn't update the |
||
| 213 | // value. |
||
| 214 | std::pair<iterator, bool> insert(std::pair<KeyT, ValueT> &&KV) { |
||
| 215 | return try_emplace(std::move(KV.first), std::move(KV.second)); |
||
| 216 | } |
||
| 217 | |||
| 218 | // Inserts key,value pair into the map if the key isn't already in the map. |
||
| 219 | // The value is constructed in-place if the key is not in the map, otherwise |
||
| 220 | // it is not moved. |
||
| 221 | template <typename... Ts> |
||
| 222 | std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&... Args) { |
||
| 223 | BucketT *TheBucket; |
||
| 224 | if (LookupBucketFor(Key, TheBucket)) |
||
| 225 | return std::make_pair(makeIterator(TheBucket, |
||
| 226 | shouldReverseIterate<KeyT>() |
||
| 227 | ? getBuckets() |
||
| 228 | : getBucketsEnd(), |
||
| 229 | *this, true), |
||
| 230 | false); // Already in map. |
||
| 231 | |||
| 232 | // Otherwise, insert the new element. |
||
| 233 | TheBucket = |
||
| 234 | InsertIntoBucket(TheBucket, std::move(Key), std::forward<Ts>(Args)...); |
||
| 235 | return std::make_pair(makeIterator(TheBucket, |
||
| 236 | shouldReverseIterate<KeyT>() |
||
| 237 | ? getBuckets() |
||
| 238 | : getBucketsEnd(), |
||
| 239 | *this, true), |
||
| 240 | true); |
||
| 241 | } |
||
| 242 | |||
| 243 | // Inserts key,value pair into the map if the key isn't already in the map. |
||
| 244 | // The value is constructed in-place if the key is not in the map, otherwise |
||
| 245 | // it is not moved. |
||
| 246 | template <typename... Ts> |
||
| 247 | std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&... Args) { |
||
| 248 | BucketT *TheBucket; |
||
| 249 | if (LookupBucketFor(Key, TheBucket)) |
||
| 250 | return std::make_pair(makeIterator(TheBucket, |
||
| 251 | shouldReverseIterate<KeyT>() |
||
| 252 | ? getBuckets() |
||
| 253 | : getBucketsEnd(), |
||
| 254 | *this, true), |
||
| 255 | false); // Already in map. |
||
| 256 | |||
| 257 | // Otherwise, insert the new element. |
||
| 258 | TheBucket = InsertIntoBucket(TheBucket, Key, std::forward<Ts>(Args)...); |
||
| 259 | return std::make_pair(makeIterator(TheBucket, |
||
| 260 | shouldReverseIterate<KeyT>() |
||
| 261 | ? getBuckets() |
||
| 262 | : getBucketsEnd(), |
||
| 263 | *this, true), |
||
| 264 | true); |
||
| 265 | } |
||
| 266 | |||
| 267 | /// Alternate version of insert() which allows a different, and possibly |
||
| 268 | /// less expensive, key type. |
||
| 269 | /// The DenseMapInfo is responsible for supplying methods |
||
| 270 | /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key |
||
| 271 | /// type used. |
||
| 272 | template <typename LookupKeyT> |
||
| 273 | std::pair<iterator, bool> insert_as(std::pair<KeyT, ValueT> &&KV, |
||
| 274 | const LookupKeyT &Val) { |
||
| 275 | BucketT *TheBucket; |
||
| 276 | if (LookupBucketFor(Val, TheBucket)) |
||
| 277 | return std::make_pair(makeIterator(TheBucket, |
||
| 278 | shouldReverseIterate<KeyT>() |
||
| 279 | ? getBuckets() |
||
| 280 | : getBucketsEnd(), |
||
| 281 | *this, true), |
||
| 282 | false); // Already in map. |
||
| 283 | |||
| 284 | // Otherwise, insert the new element. |
||
| 285 | TheBucket = InsertIntoBucketWithLookup(TheBucket, std::move(KV.first), |
||
| 286 | std::move(KV.second), Val); |
||
| 287 | return std::make_pair(makeIterator(TheBucket, |
||
| 288 | shouldReverseIterate<KeyT>() |
||
| 289 | ? getBuckets() |
||
| 290 | : getBucketsEnd(), |
||
| 291 | *this, true), |
||
| 292 | true); |
||
| 293 | } |
||
| 294 | |||
| 295 | /// insert - Range insertion of pairs. |
||
| 296 | template<typename InputIt> |
||
| 297 | void insert(InputIt I, InputIt E) { |
||
| 298 | for (; I != E; ++I) |
||
| 299 | insert(*I); |
||
| 300 | } |
||
| 301 | |||
| 302 | bool erase(const KeyT &Val) { |
||
| 303 | BucketT *TheBucket; |
||
| 304 | if (!LookupBucketFor(Val, TheBucket)) |
||
| 305 | return false; // not in map. |
||
| 306 | |||
| 307 | TheBucket->getSecond().~ValueT(); |
||
| 308 | TheBucket->getFirst() = getTombstoneKey(); |
||
| 309 | decrementNumEntries(); |
||
| 310 | incrementNumTombstones(); |
||
| 311 | return true; |
||
| 312 | } |
||
| 313 | void erase(iterator I) { |
||
| 314 | BucketT *TheBucket = &*I; |
||
| 315 | TheBucket->getSecond().~ValueT(); |
||
| 316 | TheBucket->getFirst() = getTombstoneKey(); |
||
| 317 | decrementNumEntries(); |
||
| 318 | incrementNumTombstones(); |
||
| 319 | } |
||
| 320 | |||
| 321 | value_type& FindAndConstruct(const KeyT &Key) { |
||
| 322 | BucketT *TheBucket; |
||
| 323 | if (LookupBucketFor(Key, TheBucket)) |
||
| 324 | return *TheBucket; |
||
| 325 | |||
| 326 | return *InsertIntoBucket(TheBucket, Key); |
||
| 327 | } |
||
| 328 | |||
| 329 | ValueT &operator[](const KeyT &Key) { |
||
| 330 | return FindAndConstruct(Key).second; |
||
| 331 | } |
||
| 332 | |||
| 333 | value_type& FindAndConstruct(KeyT &&Key) { |
||
| 334 | BucketT *TheBucket; |
||
| 335 | if (LookupBucketFor(Key, TheBucket)) |
||
| 336 | return *TheBucket; |
||
| 337 | |||
| 338 | return *InsertIntoBucket(TheBucket, std::move(Key)); |
||
| 339 | } |
||
| 340 | |||
| 341 | ValueT &operator[](KeyT &&Key) { |
||
| 342 | return FindAndConstruct(std::move(Key)).second; |
||
| 343 | } |
||
| 344 | |||
| 345 | /// isPointerIntoBucketsArray - Return true if the specified pointer points |
||
| 346 | /// somewhere into the DenseMap's array of buckets (i.e. either to a key or |
||
| 347 | /// value in the DenseMap). |
||
| 348 | bool isPointerIntoBucketsArray(const void *Ptr) const { |
||
| 349 | return Ptr >= getBuckets() && Ptr < getBucketsEnd(); |
||
| 350 | } |
||
| 351 | |||
| 352 | /// getPointerIntoBucketsArray() - Return an opaque pointer into the buckets |
||
| 353 | /// array. In conjunction with the previous method, this can be used to |
||
| 354 | /// determine whether an insertion caused the DenseMap to reallocate. |
||
| 355 | const void *getPointerIntoBucketsArray() const { return getBuckets(); } |
||
| 356 | |||
| 357 | protected: |
||
| 358 | DenseMapBase() = default; |
||
| 359 | |||
| 360 | void destroyAll() { |
||
| 361 | if (getNumBuckets() == 0) // Nothing to do. |
||
| 362 | return; |
||
| 363 | |||
| 364 | const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); |
||
| 365 | for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { |
||
| 366 | if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && |
||
| 367 | !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) |
||
| 368 | P->getSecond().~ValueT(); |
||
| 369 | P->getFirst().~KeyT(); |
||
| 370 | } |
||
| 371 | } |
||
| 372 | |||
| 373 | void initEmpty() { |
||
| 374 | setNumEntries(0); |
||
| 375 | setNumTombstones(0); |
||
| 376 | |||
| 377 | assert((getNumBuckets() & (getNumBuckets()-1)) == 0 && |
||
| 378 | "# initial buckets must be a power of two!"); |
||
| 379 | const KeyT EmptyKey = getEmptyKey(); |
||
| 380 | for (BucketT *B = getBuckets(), *E = getBucketsEnd(); B != E; ++B) |
||
| 381 | ::new (&B->getFirst()) KeyT(EmptyKey); |
||
| 382 | } |
||
| 383 | |||
| 384 | /// Returns the number of buckets to allocate to ensure that the DenseMap can |
||
| 385 | /// accommodate \p NumEntries without need to grow(). |
||
| 386 | unsigned getMinBucketToReserveForEntries(unsigned NumEntries) { |
||
| 387 | // Ensure that "NumEntries * 4 < NumBuckets * 3" |
||
| 388 | if (NumEntries == 0) |
||
| 389 | return 0; |
||
| 390 | // +1 is required because of the strict equality. |
||
| 391 | // For example if NumEntries is 48, we need to return 401. |
||
| 392 | return NextPowerOf2(NumEntries * 4 / 3 + 1); |
||
| 393 | } |
||
| 394 | |||
| 395 | void moveFromOldBuckets(BucketT *OldBucketsBegin, BucketT *OldBucketsEnd) { |
||
| 396 | initEmpty(); |
||
| 397 | |||
| 398 | // Insert all the old elements. |
||
| 399 | const KeyT EmptyKey = getEmptyKey(); |
||
| 400 | const KeyT TombstoneKey = getTombstoneKey(); |
||
| 401 | for (BucketT *B = OldBucketsBegin, *E = OldBucketsEnd; B != E; ++B) { |
||
| 402 | if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) && |
||
| 403 | !KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) { |
||
| 404 | // Insert the key/value into the new table. |
||
| 405 | BucketT *DestBucket; |
||
| 406 | bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket); |
||
| 407 | (void)FoundVal; // silence warning. |
||
| 408 | assert(!FoundVal && "Key already in new map?"); |
||
| 409 | DestBucket->getFirst() = std::move(B->getFirst()); |
||
| 410 | ::new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond())); |
||
| 411 | incrementNumEntries(); |
||
| 412 | |||
| 413 | // Free the value. |
||
| 414 | B->getSecond().~ValueT(); |
||
| 415 | } |
||
| 416 | B->getFirst().~KeyT(); |
||
| 417 | } |
||
| 418 | } |
||
| 419 | |||
| 420 | template <typename OtherBaseT> |
||
| 421 | void copyFrom( |
||
| 422 | const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) { |
||
| 423 | assert(&other != this); |
||
| 424 | assert(getNumBuckets() == other.getNumBuckets()); |
||
| 425 | |||
| 426 | setNumEntries(other.getNumEntries()); |
||
| 427 | setNumTombstones(other.getNumTombstones()); |
||
| 428 | |||
| 429 | if (std::is_trivially_copyable<KeyT>::value && |
||
| 430 | std::is_trivially_copyable<ValueT>::value) |
||
| 431 | memcpy(reinterpret_cast<void *>(getBuckets()), other.getBuckets(), |
||
| 432 | getNumBuckets() * sizeof(BucketT)); |
||
| 433 | else |
||
| 434 | for (size_t i = 0; i < getNumBuckets(); ++i) { |
||
| 435 | ::new (&getBuckets()[i].getFirst()) |
||
| 436 | KeyT(other.getBuckets()[i].getFirst()); |
||
| 437 | if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) && |
||
| 438 | !KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey())) |
||
| 439 | ::new (&getBuckets()[i].getSecond()) |
||
| 440 | ValueT(other.getBuckets()[i].getSecond()); |
||
| 441 | } |
||
| 442 | } |
||
| 443 | |||
| 444 | static unsigned getHashValue(const KeyT &Val) { |
||
| 445 | return KeyInfoT::getHashValue(Val); |
||
| 446 | } |
||
| 447 | |||
| 448 | template<typename LookupKeyT> |
||
| 449 | static unsigned getHashValue(const LookupKeyT &Val) { |
||
| 450 | return KeyInfoT::getHashValue(Val); |
||
| 451 | } |
||
| 452 | |||
| 453 | static const KeyT getEmptyKey() { |
||
| 454 | static_assert(std::is_base_of<DenseMapBase, DerivedT>::value, |
||
| 455 | "Must pass the derived type to this template!"); |
||
| 456 | return KeyInfoT::getEmptyKey(); |
||
| 457 | } |
||
| 458 | |||
| 459 | static const KeyT getTombstoneKey() { |
||
| 460 | return KeyInfoT::getTombstoneKey(); |
||
| 461 | } |
||
| 462 | |||
| 463 | private: |
||
| 464 | iterator makeIterator(BucketT *P, BucketT *E, |
||
| 465 | DebugEpochBase &Epoch, |
||
| 466 | bool NoAdvance=false) { |
||
| 467 | if (shouldReverseIterate<KeyT>()) { |
||
| 468 | BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; |
||
| 469 | return iterator(B, E, Epoch, NoAdvance); |
||
| 470 | } |
||
| 471 | return iterator(P, E, Epoch, NoAdvance); |
||
| 472 | } |
||
| 473 | |||
| 474 | const_iterator makeConstIterator(const BucketT *P, const BucketT *E, |
||
| 475 | const DebugEpochBase &Epoch, |
||
| 476 | const bool NoAdvance=false) const { |
||
| 477 | if (shouldReverseIterate<KeyT>()) { |
||
| 478 | const BucketT *B = P == getBucketsEnd() ? getBuckets() : P + 1; |
||
| 479 | return const_iterator(B, E, Epoch, NoAdvance); |
||
| 480 | } |
||
| 481 | return const_iterator(P, E, Epoch, NoAdvance); |
||
| 482 | } |
||
| 483 | |||
| 484 | unsigned getNumEntries() const { |
||
| 485 | return static_cast<const DerivedT *>(this)->getNumEntries(); |
||
| 486 | } |
||
| 487 | |||
| 488 | void setNumEntries(unsigned Num) { |
||
| 489 | static_cast<DerivedT *>(this)->setNumEntries(Num); |
||
| 490 | } |
||
| 491 | |||
| 492 | void incrementNumEntries() { |
||
| 493 | setNumEntries(getNumEntries() + 1); |
||
| 494 | } |
||
| 495 | |||
| 496 | void decrementNumEntries() { |
||
| 497 | setNumEntries(getNumEntries() - 1); |
||
| 498 | } |
||
| 499 | |||
| 500 | unsigned getNumTombstones() const { |
||
| 501 | return static_cast<const DerivedT *>(this)->getNumTombstones(); |
||
| 502 | } |
||
| 503 | |||
| 504 | void setNumTombstones(unsigned Num) { |
||
| 505 | static_cast<DerivedT *>(this)->setNumTombstones(Num); |
||
| 506 | } |
||
| 507 | |||
| 508 | void incrementNumTombstones() { |
||
| 509 | setNumTombstones(getNumTombstones() + 1); |
||
| 510 | } |
||
| 511 | |||
| 512 | void decrementNumTombstones() { |
||
| 513 | setNumTombstones(getNumTombstones() - 1); |
||
| 514 | } |
||
| 515 | |||
| 516 | const BucketT *getBuckets() const { |
||
| 517 | return static_cast<const DerivedT *>(this)->getBuckets(); |
||
| 518 | } |
||
| 519 | |||
| 520 | BucketT *getBuckets() { |
||
| 521 | return static_cast<DerivedT *>(this)->getBuckets(); |
||
| 522 | } |
||
| 523 | |||
| 524 | unsigned getNumBuckets() const { |
||
| 525 | return static_cast<const DerivedT *>(this)->getNumBuckets(); |
||
| 526 | } |
||
| 527 | |||
| 528 | BucketT *getBucketsEnd() { |
||
| 529 | return getBuckets() + getNumBuckets(); |
||
| 530 | } |
||
| 531 | |||
| 532 | const BucketT *getBucketsEnd() const { |
||
| 533 | return getBuckets() + getNumBuckets(); |
||
| 534 | } |
||
| 535 | |||
| 536 | void grow(unsigned AtLeast) { |
||
| 537 | static_cast<DerivedT *>(this)->grow(AtLeast); |
||
| 538 | } |
||
| 539 | |||
| 540 | void shrink_and_clear() { |
||
| 541 | static_cast<DerivedT *>(this)->shrink_and_clear(); |
||
| 542 | } |
||
| 543 | |||
| 544 | template <typename KeyArg, typename... ValueArgs> |
||
| 545 | BucketT *InsertIntoBucket(BucketT *TheBucket, KeyArg &&Key, |
||
| 546 | ValueArgs &&... Values) { |
||
| 547 | TheBucket = InsertIntoBucketImpl(Key, Key, TheBucket); |
||
| 548 | |||
| 549 | TheBucket->getFirst() = std::forward<KeyArg>(Key); |
||
| 550 | ::new (&TheBucket->getSecond()) ValueT(std::forward<ValueArgs>(Values)...); |
||
| 551 | return TheBucket; |
||
| 552 | } |
||
| 553 | |||
| 554 | template <typename LookupKeyT> |
||
| 555 | BucketT *InsertIntoBucketWithLookup(BucketT *TheBucket, KeyT &&Key, |
||
| 556 | ValueT &&Value, LookupKeyT &Lookup) { |
||
| 557 | TheBucket = InsertIntoBucketImpl(Key, Lookup, TheBucket); |
||
| 558 | |||
| 559 | TheBucket->getFirst() = std::move(Key); |
||
| 560 | ::new (&TheBucket->getSecond()) ValueT(std::move(Value)); |
||
| 561 | return TheBucket; |
||
| 562 | } |
||
| 563 | |||
| 564 | template <typename LookupKeyT> |
||
| 565 | BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup, |
||
| 566 | BucketT *TheBucket) { |
||
| 567 | incrementEpoch(); |
||
| 568 | |||
| 569 | // If the load of the hash table is more than 3/4, or if fewer than 1/8 of |
||
| 570 | // the buckets are empty (meaning that many are filled with tombstones), |
||
| 571 | // grow the table. |
||
| 572 | // |
||
| 573 | // The later case is tricky. For example, if we had one empty bucket with |
||
| 574 | // tons of tombstones, failing lookups (e.g. for insertion) would have to |
||
| 575 | // probe almost the entire table until it found the empty bucket. If the |
||
| 576 | // table completely filled with tombstones, no lookup would ever succeed, |
||
| 577 | // causing infinite loops in lookup. |
||
| 578 | unsigned NewNumEntries = getNumEntries() + 1; |
||
| 579 | unsigned NumBuckets = getNumBuckets(); |
||
| 580 | if (LLVM_UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) { |
||
| 581 | this->grow(NumBuckets * 2); |
||
| 582 | LookupBucketFor(Lookup, TheBucket); |
||
| 583 | NumBuckets = getNumBuckets(); |
||
| 584 | } else if (LLVM_UNLIKELY(NumBuckets-(NewNumEntries+getNumTombstones()) <= |
||
| 585 | NumBuckets/8)) { |
||
| 586 | this->grow(NumBuckets); |
||
| 587 | LookupBucketFor(Lookup, TheBucket); |
||
| 588 | } |
||
| 589 | assert(TheBucket); |
||
| 590 | |||
| 591 | // Only update the state after we've grown our bucket space appropriately |
||
| 592 | // so that when growing buckets we have self-consistent entry count. |
||
| 593 | incrementNumEntries(); |
||
| 594 | |||
| 595 | // If we are writing over a tombstone, remember this. |
||
| 596 | const KeyT EmptyKey = getEmptyKey(); |
||
| 597 | if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey)) |
||
| 598 | decrementNumTombstones(); |
||
| 599 | |||
| 600 | return TheBucket; |
||
| 601 | } |
||
| 602 | |||
| 603 | /// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in |
||
| 604 | /// FoundBucket. If the bucket contains the key and a value, this returns |
||
| 605 | /// true, otherwise it returns a bucket with an empty marker or tombstone and |
||
| 606 | /// returns false. |
||
| 607 | template<typename LookupKeyT> |
||
| 608 | bool LookupBucketFor(const LookupKeyT &Val, |
||
| 609 | const BucketT *&FoundBucket) const { |
||
| 610 | const BucketT *BucketsPtr = getBuckets(); |
||
| 611 | const unsigned NumBuckets = getNumBuckets(); |
||
| 612 | |||
| 613 | if (NumBuckets == 0) { |
||
| 614 | FoundBucket = nullptr; |
||
| 615 | return false; |
||
| 616 | } |
||
| 617 | |||
| 618 | // FoundTombstone - Keep track of whether we find a tombstone while probing. |
||
| 619 | const BucketT *FoundTombstone = nullptr; |
||
| 620 | const KeyT EmptyKey = getEmptyKey(); |
||
| 621 | const KeyT TombstoneKey = getTombstoneKey(); |
||
| 622 | assert(!KeyInfoT::isEqual(Val, EmptyKey) && |
||
| 623 | !KeyInfoT::isEqual(Val, TombstoneKey) && |
||
| 624 | "Empty/Tombstone value shouldn't be inserted into map!"); |
||
| 625 | |||
| 626 | unsigned BucketNo = getHashValue(Val) & (NumBuckets-1); |
||
| 627 | unsigned ProbeAmt = 1; |
||
| 628 | while (true) { |
||
| 629 | const BucketT *ThisBucket = BucketsPtr + BucketNo; |
||
| 630 | // Found Val's bucket? If so, return it. |
||
| 631 | if (LLVM_LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) { |
||
| 632 | FoundBucket = ThisBucket; |
||
| 633 | return true; |
||
| 634 | } |
||
| 635 | |||
| 636 | // If we found an empty bucket, the key doesn't exist in the set. |
||
| 637 | // Insert it and return the default value. |
||
| 638 | if (LLVM_LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) { |
||
| 639 | // If we've already seen a tombstone while probing, fill it in instead |
||
| 640 | // of the empty bucket we eventually probed to. |
||
| 641 | FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket; |
||
| 642 | return false; |
||
| 643 | } |
||
| 644 | |||
| 645 | // If this is a tombstone, remember it. If Val ends up not in the map, we |
||
| 646 | // prefer to return it than something that would require more probing. |
||
| 647 | if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) && |
||
| 648 | !FoundTombstone) |
||
| 649 | FoundTombstone = ThisBucket; // Remember the first tombstone found. |
||
| 650 | |||
| 651 | // Otherwise, it's a hash collision or a tombstone, continue quadratic |
||
| 652 | // probing. |
||
| 653 | BucketNo += ProbeAmt++; |
||
| 654 | BucketNo &= (NumBuckets-1); |
||
| 655 | } |
||
| 656 | } |
||
| 657 | |||
| 658 | template <typename LookupKeyT> |
||
| 659 | bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) { |
||
| 660 | const BucketT *ConstFoundBucket; |
||
| 661 | bool Result = const_cast<const DenseMapBase *>(this) |
||
| 662 | ->LookupBucketFor(Val, ConstFoundBucket); |
||
| 663 | FoundBucket = const_cast<BucketT *>(ConstFoundBucket); |
||
| 664 | return Result; |
||
| 665 | } |
||
| 666 | |||
| 667 | public: |
||
| 668 | /// Return the approximate size (in bytes) of the actual map. |
||
| 669 | /// This is just the raw memory used by DenseMap. |
||
| 670 | /// If entries are pointers to objects, the size of the referenced objects |
||
| 671 | /// are not included. |
||
| 672 | size_t getMemorySize() const { |
||
| 673 | return getNumBuckets() * sizeof(BucketT); |
||
| 674 | } |
||
| 675 | }; |
||
| 676 | |||
| 677 | /// Equality comparison for DenseMap. |
||
| 678 | /// |
||
| 679 | /// Iterates over elements of LHS confirming that each (key, value) pair in LHS |
||
| 680 | /// is also in RHS, and that no additional pairs are in RHS. |
||
| 681 | /// Equivalent to N calls to RHS.find and N value comparisons. Amortized |
||
| 682 | /// complexity is linear, worst case is O(N^2) (if every hash collides). |
||
| 683 | template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, |
||
| 684 | typename BucketT> |
||
| 685 | bool operator==( |
||
| 686 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS, |
||
| 687 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) { |
||
| 688 | if (LHS.size() != RHS.size()) |
||
| 689 | return false; |
||
| 690 | |||
| 691 | for (auto &KV : LHS) { |
||
| 692 | auto I = RHS.find(KV.first); |
||
| 693 | if (I == RHS.end() || I->second != KV.second) |
||
| 694 | return false; |
||
| 695 | } |
||
| 696 | |||
| 697 | return true; |
||
| 698 | } |
||
| 699 | |||
| 700 | /// Inequality comparison for DenseMap. |
||
| 701 | /// |
||
| 702 | /// Equivalent to !(LHS == RHS). See operator== for performance notes. |
||
| 703 | template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT, |
||
| 704 | typename BucketT> |
||
| 705 | bool operator!=( |
||
| 706 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS, |
||
| 707 | const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) { |
||
| 708 | return !(LHS == RHS); |
||
| 709 | } |
||
| 710 | |||
| 711 | template <typename KeyT, typename ValueT, |
||
| 712 | typename KeyInfoT = DenseMapInfo<KeyT>, |
||
| 713 | typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>> |
||
| 714 | class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>, |
||
| 715 | KeyT, ValueT, KeyInfoT, BucketT> { |
||
| 716 | friend class DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
||
| 717 | |||
| 718 | // Lift some types from the dependent base class into this class for |
||
| 719 | // simplicity of referring to them. |
||
| 720 | using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
||
| 721 | |||
| 722 | BucketT *Buckets; |
||
| 723 | unsigned NumEntries; |
||
| 724 | unsigned NumTombstones; |
||
| 725 | unsigned NumBuckets; |
||
| 726 | |||
| 727 | public: |
||
| 728 | /// Create a DenseMap with an optional \p InitialReserve that guarantee that |
||
| 729 | /// this number of elements can be inserted in the map without grow() |
||
| 730 | explicit DenseMap(unsigned InitialReserve = 0) { init(InitialReserve); } |
||
| 731 | |||
| 732 | DenseMap(const DenseMap &other) : BaseT() { |
||
| 733 | init(0); |
||
| 734 | copyFrom(other); |
||
| 735 | } |
||
| 736 | |||
| 737 | DenseMap(DenseMap &&other) : BaseT() { |
||
| 738 | init(0); |
||
| 739 | swap(other); |
||
| 740 | } |
||
| 741 | |||
| 742 | template<typename InputIt> |
||
| 743 | DenseMap(const InputIt &I, const InputIt &E) { |
||
| 744 | init(std::distance(I, E)); |
||
| 745 | this->insert(I, E); |
||
| 746 | } |
||
| 747 | |||
| 748 | DenseMap(std::initializer_list<typename BaseT::value_type> Vals) { |
||
| 749 | init(Vals.size()); |
||
| 750 | this->insert(Vals.begin(), Vals.end()); |
||
| 751 | } |
||
| 752 | |||
| 753 | ~DenseMap() { |
||
| 754 | this->destroyAll(); |
||
| 755 | deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); |
||
| 756 | } |
||
| 757 | |||
| 758 | void swap(DenseMap& RHS) { |
||
| 759 | this->incrementEpoch(); |
||
| 760 | RHS.incrementEpoch(); |
||
| 761 | std::swap(Buckets, RHS.Buckets); |
||
| 762 | std::swap(NumEntries, RHS.NumEntries); |
||
| 763 | std::swap(NumTombstones, RHS.NumTombstones); |
||
| 764 | std::swap(NumBuckets, RHS.NumBuckets); |
||
| 765 | } |
||
| 766 | |||
| 767 | DenseMap& operator=(const DenseMap& other) { |
||
| 768 | if (&other != this) |
||
| 769 | copyFrom(other); |
||
| 770 | return *this; |
||
| 771 | } |
||
| 772 | |||
| 773 | DenseMap& operator=(DenseMap &&other) { |
||
| 774 | this->destroyAll(); |
||
| 775 | deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); |
||
| 776 | init(0); |
||
| 777 | swap(other); |
||
| 778 | return *this; |
||
| 779 | } |
||
| 780 | |||
| 781 | void copyFrom(const DenseMap& other) { |
||
| 782 | this->destroyAll(); |
||
| 783 | deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT)); |
||
| 784 | if (allocateBuckets(other.NumBuckets)) { |
||
| 785 | this->BaseT::copyFrom(other); |
||
| 786 | } else { |
||
| 787 | NumEntries = 0; |
||
| 788 | NumTombstones = 0; |
||
| 789 | } |
||
| 790 | } |
||
| 791 | |||
| 792 | void init(unsigned InitNumEntries) { |
||
| 793 | auto InitBuckets = BaseT::getMinBucketToReserveForEntries(InitNumEntries); |
||
| 794 | if (allocateBuckets(InitBuckets)) { |
||
| 795 | this->BaseT::initEmpty(); |
||
| 796 | } else { |
||
| 797 | NumEntries = 0; |
||
| 798 | NumTombstones = 0; |
||
| 799 | } |
||
| 800 | } |
||
| 801 | |||
| 802 | void grow(unsigned AtLeast) { |
||
| 803 | unsigned OldNumBuckets = NumBuckets; |
||
| 804 | BucketT *OldBuckets = Buckets; |
||
| 805 | |||
| 806 | allocateBuckets(std::max<unsigned>(64, static_cast<unsigned>(NextPowerOf2(AtLeast-1)))); |
||
| 807 | assert(Buckets); |
||
| 808 | if (!OldBuckets) { |
||
| 809 | this->BaseT::initEmpty(); |
||
| 810 | return; |
||
| 811 | } |
||
| 812 | |||
| 813 | this->moveFromOldBuckets(OldBuckets, OldBuckets+OldNumBuckets); |
||
| 814 | |||
| 815 | // Free the old table. |
||
| 816 | deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets, |
||
| 817 | alignof(BucketT)); |
||
| 818 | } |
||
| 819 | |||
| 820 | void shrink_and_clear() { |
||
| 821 | unsigned OldNumBuckets = NumBuckets; |
||
| 822 | unsigned OldNumEntries = NumEntries; |
||
| 823 | this->destroyAll(); |
||
| 824 | |||
| 825 | // Reduce the number of buckets. |
||
| 826 | unsigned NewNumBuckets = 0; |
||
| 827 | if (OldNumEntries) |
||
| 828 | NewNumBuckets = std::max(64, 1 << (Log2_32_Ceil(OldNumEntries) + 1)); |
||
| 829 | if (NewNumBuckets == NumBuckets) { |
||
| 830 | this->BaseT::initEmpty(); |
||
| 831 | return; |
||
| 832 | } |
||
| 833 | |||
| 834 | deallocate_buffer(Buckets, sizeof(BucketT) * OldNumBuckets, |
||
| 835 | alignof(BucketT)); |
||
| 836 | init(NewNumBuckets); |
||
| 837 | } |
||
| 838 | |||
| 839 | private: |
||
| 840 | unsigned getNumEntries() const { |
||
| 841 | return NumEntries; |
||
| 842 | } |
||
| 843 | |||
| 844 | void setNumEntries(unsigned Num) { |
||
| 845 | NumEntries = Num; |
||
| 846 | } |
||
| 847 | |||
| 848 | unsigned getNumTombstones() const { |
||
| 849 | return NumTombstones; |
||
| 850 | } |
||
| 851 | |||
| 852 | void setNumTombstones(unsigned Num) { |
||
| 853 | NumTombstones = Num; |
||
| 854 | } |
||
| 855 | |||
| 856 | BucketT *getBuckets() const { |
||
| 857 | return Buckets; |
||
| 858 | } |
||
| 859 | |||
| 860 | unsigned getNumBuckets() const { |
||
| 861 | return NumBuckets; |
||
| 862 | } |
||
| 863 | |||
| 864 | bool allocateBuckets(unsigned Num) { |
||
| 865 | NumBuckets = Num; |
||
| 866 | if (NumBuckets == 0) { |
||
| 867 | Buckets = nullptr; |
||
| 868 | return false; |
||
| 869 | } |
||
| 870 | |||
| 871 | Buckets = static_cast<BucketT *>( |
||
| 872 | allocate_buffer(sizeof(BucketT) * NumBuckets, alignof(BucketT))); |
||
| 873 | return true; |
||
| 874 | } |
||
| 875 | }; |
||
| 876 | |||
| 877 | template <typename KeyT, typename ValueT, unsigned InlineBuckets = 4, |
||
| 878 | typename KeyInfoT = DenseMapInfo<KeyT>, |
||
| 879 | typename BucketT = llvm::detail::DenseMapPair<KeyT, ValueT>> |
||
| 880 | class SmallDenseMap |
||
| 881 | : public DenseMapBase< |
||
| 882 | SmallDenseMap<KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT>, KeyT, |
||
| 883 | ValueT, KeyInfoT, BucketT> { |
||
| 884 | friend class DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
||
| 885 | |||
| 886 | // Lift some types from the dependent base class into this class for |
||
| 887 | // simplicity of referring to them. |
||
| 888 | using BaseT = DenseMapBase<SmallDenseMap, KeyT, ValueT, KeyInfoT, BucketT>; |
||
| 889 | |||
| 890 | static_assert(isPowerOf2_64(InlineBuckets), |
||
| 891 | "InlineBuckets must be a power of 2."); |
||
| 892 | |||
| 893 | unsigned Small : 1; |
||
| 894 | unsigned NumEntries : 31; |
||
| 895 | unsigned NumTombstones; |
||
| 896 | |||
| 897 | struct LargeRep { |
||
| 898 | BucketT *Buckets; |
||
| 899 | unsigned NumBuckets; |
||
| 900 | }; |
||
| 901 | |||
| 902 | /// A "union" of an inline bucket array and the struct representing |
||
| 903 | /// a large bucket. This union will be discriminated by the 'Small' bit. |
||
| 904 | AlignedCharArrayUnion<BucketT[InlineBuckets], LargeRep> storage; |
||
| 905 | |||
| 906 | public: |
||
| 907 | explicit SmallDenseMap(unsigned NumInitBuckets = 0) { |
||
| 908 | if (NumInitBuckets > InlineBuckets) |
||
| 909 | NumInitBuckets = NextPowerOf2(NumInitBuckets - 1); |
||
| 910 | init(NumInitBuckets); |
||
| 911 | } |
||
| 912 | |||
| 913 | SmallDenseMap(const SmallDenseMap &other) : BaseT() { |
||
| 914 | init(0); |
||
| 915 | copyFrom(other); |
||
| 916 | } |
||
| 917 | |||
| 918 | SmallDenseMap(SmallDenseMap &&other) : BaseT() { |
||
| 919 | init(0); |
||
| 920 | swap(other); |
||
| 921 | } |
||
| 922 | |||
| 923 | template<typename InputIt> |
||
| 924 | SmallDenseMap(const InputIt &I, const InputIt &E) { |
||
| 925 | init(NextPowerOf2(std::distance(I, E))); |
||
| 926 | this->insert(I, E); |
||
| 927 | } |
||
| 928 | |||
| 929 | SmallDenseMap(std::initializer_list<typename BaseT::value_type> Vals) |
||
| 930 | : SmallDenseMap(Vals.begin(), Vals.end()) {} |
||
| 931 | |||
| 932 | ~SmallDenseMap() { |
||
| 933 | this->destroyAll(); |
||
| 934 | deallocateBuckets(); |
||
| 935 | } |
||
| 936 | |||
| 937 | void swap(SmallDenseMap& RHS) { |
||
| 938 | unsigned TmpNumEntries = RHS.NumEntries; |
||
| 939 | RHS.NumEntries = NumEntries; |
||
| 940 | NumEntries = TmpNumEntries; |
||
| 941 | std::swap(NumTombstones, RHS.NumTombstones); |
||
| 942 | |||
| 943 | const KeyT EmptyKey = this->getEmptyKey(); |
||
| 944 | const KeyT TombstoneKey = this->getTombstoneKey(); |
||
| 945 | if (Small && RHS.Small) { |
||
| 946 | // If we're swapping inline bucket arrays, we have to cope with some of |
||
| 947 | // the tricky bits of DenseMap's storage system: the buckets are not |
||
| 948 | // fully initialized. Thus we swap every key, but we may have |
||
| 949 | // a one-directional move of the value. |
||
| 950 | for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { |
||
| 951 | BucketT *LHSB = &getInlineBuckets()[i], |
||
| 952 | *RHSB = &RHS.getInlineBuckets()[i]; |
||
| 953 | bool hasLHSValue = (!KeyInfoT::isEqual(LHSB->getFirst(), EmptyKey) && |
||
| 954 | !KeyInfoT::isEqual(LHSB->getFirst(), TombstoneKey)); |
||
| 955 | bool hasRHSValue = (!KeyInfoT::isEqual(RHSB->getFirst(), EmptyKey) && |
||
| 956 | !KeyInfoT::isEqual(RHSB->getFirst(), TombstoneKey)); |
||
| 957 | if (hasLHSValue && hasRHSValue) { |
||
| 958 | // Swap together if we can... |
||
| 959 | std::swap(*LHSB, *RHSB); |
||
| 960 | continue; |
||
| 961 | } |
||
| 962 | // Swap separately and handle any asymmetry. |
||
| 963 | std::swap(LHSB->getFirst(), RHSB->getFirst()); |
||
| 964 | if (hasLHSValue) { |
||
| 965 | ::new (&RHSB->getSecond()) ValueT(std::move(LHSB->getSecond())); |
||
| 966 | LHSB->getSecond().~ValueT(); |
||
| 967 | } else if (hasRHSValue) { |
||
| 968 | ::new (&LHSB->getSecond()) ValueT(std::move(RHSB->getSecond())); |
||
| 969 | RHSB->getSecond().~ValueT(); |
||
| 970 | } |
||
| 971 | } |
||
| 972 | return; |
||
| 973 | } |
||
| 974 | if (!Small && !RHS.Small) { |
||
| 975 | std::swap(getLargeRep()->Buckets, RHS.getLargeRep()->Buckets); |
||
| 976 | std::swap(getLargeRep()->NumBuckets, RHS.getLargeRep()->NumBuckets); |
||
| 977 | return; |
||
| 978 | } |
||
| 979 | |||
| 980 | SmallDenseMap &SmallSide = Small ? *this : RHS; |
||
| 981 | SmallDenseMap &LargeSide = Small ? RHS : *this; |
||
| 982 | |||
| 983 | // First stash the large side's rep and move the small side across. |
||
| 984 | LargeRep TmpRep = std::move(*LargeSide.getLargeRep()); |
||
| 985 | LargeSide.getLargeRep()->~LargeRep(); |
||
| 986 | LargeSide.Small = true; |
||
| 987 | // This is similar to the standard move-from-old-buckets, but the bucket |
||
| 988 | // count hasn't actually rotated in this case. So we have to carefully |
||
| 989 | // move construct the keys and values into their new locations, but there |
||
| 990 | // is no need to re-hash things. |
||
| 991 | for (unsigned i = 0, e = InlineBuckets; i != e; ++i) { |
||
| 992 | BucketT *NewB = &LargeSide.getInlineBuckets()[i], |
||
| 993 | *OldB = &SmallSide.getInlineBuckets()[i]; |
||
| 994 | ::new (&NewB->getFirst()) KeyT(std::move(OldB->getFirst())); |
||
| 995 | OldB->getFirst().~KeyT(); |
||
| 996 | if (!KeyInfoT::isEqual(NewB->getFirst(), EmptyKey) && |
||
| 997 | !KeyInfoT::isEqual(NewB->getFirst(), TombstoneKey)) { |
||
| 998 | ::new (&NewB->getSecond()) ValueT(std::move(OldB->getSecond())); |
||
| 999 | OldB->getSecond().~ValueT(); |
||
| 1000 | } |
||
| 1001 | } |
||
| 1002 | |||
| 1003 | // The hard part of moving the small buckets across is done, just move |
||
| 1004 | // the TmpRep into its new home. |
||
| 1005 | SmallSide.Small = false; |
||
| 1006 | new (SmallSide.getLargeRep()) LargeRep(std::move(TmpRep)); |
||
| 1007 | } |
||
| 1008 | |||
| 1009 | SmallDenseMap& operator=(const SmallDenseMap& other) { |
||
| 1010 | if (&other != this) |
||
| 1011 | copyFrom(other); |
||
| 1012 | return *this; |
||
| 1013 | } |
||
| 1014 | |||
| 1015 | SmallDenseMap& operator=(SmallDenseMap &&other) { |
||
| 1016 | this->destroyAll(); |
||
| 1017 | deallocateBuckets(); |
||
| 1018 | init(0); |
||
| 1019 | swap(other); |
||
| 1020 | return *this; |
||
| 1021 | } |
||
| 1022 | |||
| 1023 | void copyFrom(const SmallDenseMap& other) { |
||
| 1024 | this->destroyAll(); |
||
| 1025 | deallocateBuckets(); |
||
| 1026 | Small = true; |
||
| 1027 | if (other.getNumBuckets() > InlineBuckets) { |
||
| 1028 | Small = false; |
||
| 1029 | new (getLargeRep()) LargeRep(allocateBuckets(other.getNumBuckets())); |
||
| 1030 | } |
||
| 1031 | this->BaseT::copyFrom(other); |
||
| 1032 | } |
||
| 1033 | |||
| 1034 | void init(unsigned InitBuckets) { |
||
| 1035 | Small = true; |
||
| 1036 | if (InitBuckets > InlineBuckets) { |
||
| 1037 | Small = false; |
||
| 1038 | new (getLargeRep()) LargeRep(allocateBuckets(InitBuckets)); |
||
| 1039 | } |
||
| 1040 | this->BaseT::initEmpty(); |
||
| 1041 | } |
||
| 1042 | |||
| 1043 | void grow(unsigned AtLeast) { |
||
| 1044 | if (AtLeast > InlineBuckets) |
||
| 1045 | AtLeast = std::max<unsigned>(64, NextPowerOf2(AtLeast-1)); |
||
| 1046 | |||
| 1047 | if (Small) { |
||
| 1048 | // First move the inline buckets into a temporary storage. |
||
| 1049 | AlignedCharArrayUnion<BucketT[InlineBuckets]> TmpStorage; |
||
| 1050 | BucketT *TmpBegin = reinterpret_cast<BucketT *>(&TmpStorage); |
||
| 1051 | BucketT *TmpEnd = TmpBegin; |
||
| 1052 | |||
| 1053 | // Loop over the buckets, moving non-empty, non-tombstones into the |
||
| 1054 | // temporary storage. Have the loop move the TmpEnd forward as it goes. |
||
| 1055 | const KeyT EmptyKey = this->getEmptyKey(); |
||
| 1056 | const KeyT TombstoneKey = this->getTombstoneKey(); |
||
| 1057 | for (BucketT *P = getBuckets(), *E = P + InlineBuckets; P != E; ++P) { |
||
| 1058 | if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && |
||
| 1059 | !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { |
||
| 1060 | assert(size_t(TmpEnd - TmpBegin) < InlineBuckets && |
||
| 1061 | "Too many inline buckets!"); |
||
| 1062 | ::new (&TmpEnd->getFirst()) KeyT(std::move(P->getFirst())); |
||
| 1063 | ::new (&TmpEnd->getSecond()) ValueT(std::move(P->getSecond())); |
||
| 1064 | ++TmpEnd; |
||
| 1065 | P->getSecond().~ValueT(); |
||
| 1066 | } |
||
| 1067 | P->getFirst().~KeyT(); |
||
| 1068 | } |
||
| 1069 | |||
| 1070 | // AtLeast == InlineBuckets can happen if there are many tombstones, |
||
| 1071 | // and grow() is used to remove them. Usually we always switch to the |
||
| 1072 | // large rep here. |
||
| 1073 | if (AtLeast > InlineBuckets) { |
||
| 1074 | Small = false; |
||
| 1075 | new (getLargeRep()) LargeRep(allocateBuckets(AtLeast)); |
||
| 1076 | } |
||
| 1077 | this->moveFromOldBuckets(TmpBegin, TmpEnd); |
||
| 1078 | return; |
||
| 1079 | } |
||
| 1080 | |||
| 1081 | LargeRep OldRep = std::move(*getLargeRep()); |
||
| 1082 | getLargeRep()->~LargeRep(); |
||
| 1083 | if (AtLeast <= InlineBuckets) { |
||
| 1084 | Small = true; |
||
| 1085 | } else { |
||
| 1086 | new (getLargeRep()) LargeRep(allocateBuckets(AtLeast)); |
||
| 1087 | } |
||
| 1088 | |||
| 1089 | this->moveFromOldBuckets(OldRep.Buckets, OldRep.Buckets+OldRep.NumBuckets); |
||
| 1090 | |||
| 1091 | // Free the old table. |
||
| 1092 | deallocate_buffer(OldRep.Buckets, sizeof(BucketT) * OldRep.NumBuckets, |
||
| 1093 | alignof(BucketT)); |
||
| 1094 | } |
||
| 1095 | |||
| 1096 | void shrink_and_clear() { |
||
| 1097 | unsigned OldSize = this->size(); |
||
| 1098 | this->destroyAll(); |
||
| 1099 | |||
| 1100 | // Reduce the number of buckets. |
||
| 1101 | unsigned NewNumBuckets = 0; |
||
| 1102 | if (OldSize) { |
||
| 1103 | NewNumBuckets = 1 << (Log2_32_Ceil(OldSize) + 1); |
||
| 1104 | if (NewNumBuckets > InlineBuckets && NewNumBuckets < 64u) |
||
| 1105 | NewNumBuckets = 64; |
||
| 1106 | } |
||
| 1107 | if ((Small && NewNumBuckets <= InlineBuckets) || |
||
| 1108 | (!Small && NewNumBuckets == getLargeRep()->NumBuckets)) { |
||
| 1109 | this->BaseT::initEmpty(); |
||
| 1110 | return; |
||
| 1111 | } |
||
| 1112 | |||
| 1113 | deallocateBuckets(); |
||
| 1114 | init(NewNumBuckets); |
||
| 1115 | } |
||
| 1116 | |||
| 1117 | private: |
||
| 1118 | unsigned getNumEntries() const { |
||
| 1119 | return NumEntries; |
||
| 1120 | } |
||
| 1121 | |||
| 1122 | void setNumEntries(unsigned Num) { |
||
| 1123 | // NumEntries is hardcoded to be 31 bits wide. |
||
| 1124 | assert(Num < (1U << 31) && "Cannot support more than 1<<31 entries"); |
||
| 1125 | NumEntries = Num; |
||
| 1126 | } |
||
| 1127 | |||
| 1128 | unsigned getNumTombstones() const { |
||
| 1129 | return NumTombstones; |
||
| 1130 | } |
||
| 1131 | |||
| 1132 | void setNumTombstones(unsigned Num) { |
||
| 1133 | NumTombstones = Num; |
||
| 1134 | } |
||
| 1135 | |||
| 1136 | const BucketT *getInlineBuckets() const { |
||
| 1137 | assert(Small); |
||
| 1138 | // Note that this cast does not violate aliasing rules as we assert that |
||
| 1139 | // the memory's dynamic type is the small, inline bucket buffer, and the |
||
| 1140 | // 'storage' is a POD containing a char buffer. |
||
| 1141 | return reinterpret_cast<const BucketT *>(&storage); |
||
| 1142 | } |
||
| 1143 | |||
| 1144 | BucketT *getInlineBuckets() { |
||
| 1145 | return const_cast<BucketT *>( |
||
| 1146 | const_cast<const SmallDenseMap *>(this)->getInlineBuckets()); |
||
| 1147 | } |
||
| 1148 | |||
| 1149 | const LargeRep *getLargeRep() const { |
||
| 1150 | assert(!Small); |
||
| 1151 | // Note, same rule about aliasing as with getInlineBuckets. |
||
| 1152 | return reinterpret_cast<const LargeRep *>(&storage); |
||
| 1153 | } |
||
| 1154 | |||
| 1155 | LargeRep *getLargeRep() { |
||
| 1156 | return const_cast<LargeRep *>( |
||
| 1157 | const_cast<const SmallDenseMap *>(this)->getLargeRep()); |
||
| 1158 | } |
||
| 1159 | |||
| 1160 | const BucketT *getBuckets() const { |
||
| 1161 | return Small ? getInlineBuckets() : getLargeRep()->Buckets; |
||
| 1162 | } |
||
| 1163 | |||
| 1164 | BucketT *getBuckets() { |
||
| 1165 | return const_cast<BucketT *>( |
||
| 1166 | const_cast<const SmallDenseMap *>(this)->getBuckets()); |
||
| 1167 | } |
||
| 1168 | |||
| 1169 | unsigned getNumBuckets() const { |
||
| 1170 | return Small ? InlineBuckets : getLargeRep()->NumBuckets; |
||
| 1171 | } |
||
| 1172 | |||
| 1173 | void deallocateBuckets() { |
||
| 1174 | if (Small) |
||
| 1175 | return; |
||
| 1176 | |||
| 1177 | deallocate_buffer(getLargeRep()->Buckets, |
||
| 1178 | sizeof(BucketT) * getLargeRep()->NumBuckets, |
||
| 1179 | alignof(BucketT)); |
||
| 1180 | getLargeRep()->~LargeRep(); |
||
| 1181 | } |
||
| 1182 | |||
| 1183 | LargeRep allocateBuckets(unsigned Num) { |
||
| 1184 | assert(Num > InlineBuckets && "Must allocate more buckets than are inline"); |
||
| 1185 | LargeRep Rep = {static_cast<BucketT *>(allocate_buffer( |
||
| 1186 | sizeof(BucketT) * Num, alignof(BucketT))), |
||
| 1187 | Num}; |
||
| 1188 | return Rep; |
||
| 1189 | } |
||
| 1190 | }; |
||
| 1191 | |||
| 1192 | template <typename KeyT, typename ValueT, typename KeyInfoT, typename Bucket, |
||
| 1193 | bool IsConst> |
||
| 1194 | class DenseMapIterator : DebugEpochBase::HandleBase { |
||
| 1195 | friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>; |
||
| 1196 | friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, false>; |
||
| 1197 | |||
| 1198 | public: |
||
| 1199 | using difference_type = ptrdiff_t; |
||
| 1200 | using value_type = std::conditional_t<IsConst, const Bucket, Bucket>; |
||
| 1201 | using pointer = value_type *; |
||
| 1202 | using reference = value_type &; |
||
| 1203 | using iterator_category = std::forward_iterator_tag; |
||
| 1204 | |||
| 1205 | private: |
||
| 1206 | pointer Ptr = nullptr; |
||
| 1207 | pointer End = nullptr; |
||
| 1208 | |||
| 1209 | public: |
||
| 1210 | DenseMapIterator() = default; |
||
| 1211 | |||
| 1212 | DenseMapIterator(pointer Pos, pointer E, const DebugEpochBase &Epoch, |
||
| 1213 | bool NoAdvance = false) |
||
| 1214 | : DebugEpochBase::HandleBase(&Epoch), Ptr(Pos), End(E) { |
||
| 1215 | assert(isHandleInSync() && "invalid construction!"); |
||
| 1216 | |||
| 1217 | if (NoAdvance) return; |
||
| 1218 | if (shouldReverseIterate<KeyT>()) { |
||
| 1219 | RetreatPastEmptyBuckets(); |
||
| 1220 | return; |
||
| 1221 | } |
||
| 1222 | AdvancePastEmptyBuckets(); |
||
| 1223 | } |
||
| 1224 | |||
| 1225 | // Converting ctor from non-const iterators to const iterators. SFINAE'd out |
||
| 1226 | // for const iterator destinations so it doesn't end up as a user defined copy |
||
| 1227 | // constructor. |
||
| 1228 | template <bool IsConstSrc, |
||
| 1229 | typename = std::enable_if_t<!IsConstSrc && IsConst>> |
||
| 1230 | DenseMapIterator( |
||
| 1231 | const DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, IsConstSrc> &I) |
||
| 1232 | : DebugEpochBase::HandleBase(I), Ptr(I.Ptr), End(I.End) {} |
||
| 1233 | |||
| 1234 | reference operator*() const { |
||
| 1235 | assert(isHandleInSync() && "invalid iterator access!"); |
||
| 1236 | assert(Ptr != End && "dereferencing end() iterator"); |
||
| 1237 | if (shouldReverseIterate<KeyT>()) |
||
| 1238 | return Ptr[-1]; |
||
| 1239 | return *Ptr; |
||
| 1240 | } |
||
| 1241 | pointer operator->() const { |
||
| 1242 | assert(isHandleInSync() && "invalid iterator access!"); |
||
| 1243 | assert(Ptr != End && "dereferencing end() iterator"); |
||
| 1244 | if (shouldReverseIterate<KeyT>()) |
||
| 1245 | return &(Ptr[-1]); |
||
| 1246 | return Ptr; |
||
| 1247 | } |
||
| 1248 | |||
| 1249 | friend bool operator==(const DenseMapIterator &LHS, |
||
| 1250 | const DenseMapIterator &RHS) { |
||
| 1251 | assert((!LHS.Ptr || LHS.isHandleInSync()) && "handle not in sync!"); |
||
| 1252 | assert((!RHS.Ptr || RHS.isHandleInSync()) && "handle not in sync!"); |
||
| 1253 | assert(LHS.getEpochAddress() == RHS.getEpochAddress() && |
||
| 1254 | "comparing incomparable iterators!"); |
||
| 1255 | return LHS.Ptr == RHS.Ptr; |
||
| 1256 | } |
||
| 1257 | |||
| 1258 | friend bool operator!=(const DenseMapIterator &LHS, |
||
| 1259 | const DenseMapIterator &RHS) { |
||
| 1260 | return !(LHS == RHS); |
||
| 1261 | } |
||
| 1262 | |||
| 1263 | inline DenseMapIterator& operator++() { // Preincrement |
||
| 1264 | assert(isHandleInSync() && "invalid iterator access!"); |
||
| 1265 | assert(Ptr != End && "incrementing end() iterator"); |
||
| 1266 | if (shouldReverseIterate<KeyT>()) { |
||
| 1267 | --Ptr; |
||
| 1268 | RetreatPastEmptyBuckets(); |
||
| 1269 | return *this; |
||
| 1270 | } |
||
| 1271 | ++Ptr; |
||
| 1272 | AdvancePastEmptyBuckets(); |
||
| 1273 | return *this; |
||
| 1274 | } |
||
| 1275 | DenseMapIterator operator++(int) { // Postincrement |
||
| 1276 | assert(isHandleInSync() && "invalid iterator access!"); |
||
| 1277 | DenseMapIterator tmp = *this; ++*this; return tmp; |
||
| 1278 | } |
||
| 1279 | |||
| 1280 | private: |
||
| 1281 | void AdvancePastEmptyBuckets() { |
||
| 1282 | assert(Ptr <= End); |
||
| 1283 | const KeyT Empty = KeyInfoT::getEmptyKey(); |
||
| 1284 | const KeyT Tombstone = KeyInfoT::getTombstoneKey(); |
||
| 1285 | |||
| 1286 | while (Ptr != End && (KeyInfoT::isEqual(Ptr->getFirst(), Empty) || |
||
| 1287 | KeyInfoT::isEqual(Ptr->getFirst(), Tombstone))) |
||
| 1288 | ++Ptr; |
||
| 1289 | } |
||
| 1290 | |||
| 1291 | void RetreatPastEmptyBuckets() { |
||
| 1292 | assert(Ptr >= End); |
||
| 1293 | const KeyT Empty = KeyInfoT::getEmptyKey(); |
||
| 1294 | const KeyT Tombstone = KeyInfoT::getTombstoneKey(); |
||
| 1295 | |||
| 1296 | while (Ptr != End && (KeyInfoT::isEqual(Ptr[-1].getFirst(), Empty) || |
||
| 1297 | KeyInfoT::isEqual(Ptr[-1].getFirst(), Tombstone))) |
||
| 1298 | --Ptr; |
||
| 1299 | } |
||
| 1300 | }; |
||
| 1301 | |||
| 1302 | template <typename KeyT, typename ValueT, typename KeyInfoT> |
||
| 1303 | inline size_t capacity_in_bytes(const DenseMap<KeyT, ValueT, KeyInfoT> &X) { |
||
| 1304 | return X.getMemorySize(); |
||
| 1305 | } |
||
| 1306 | |||
| 1307 | } // end namespace llvm |
||
| 1308 | |||
| 1309 | #endif // LLVM_ADT_DENSEMAP_H |