From e44bf8557682f12930b4fc0dbbdc83f5e574019e Mon Sep 17 00:00:00 2001 From: Gareth Lloyd Date: Fri, 8 Mar 2024 15:45:16 +0000 Subject: [PATCH] PoolResource2 --- src/utils/memory.cpp | 97 +++++++++++++++++++++++-- src/utils/memory.hpp | 163 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 4 deletions(-) diff --git a/src/utils/memory.cpp b/src/utils/memory.cpp index 49eb8f3b0..e5df27664 100644 --- a/src/utils/memory.cpp +++ b/src/utils/memory.cpp @@ -259,10 +259,10 @@ void *PoolResource::DoAllocate(size_t bytes, size_t alignment) { void PoolResource::DoDeallocate(void *p, size_t bytes, size_t alignment) { size_t block_size = std::max(bytes, alignment); - MG_ASSERT(block_size % alignment == 0, - "PoolResource shouldn't serve allocation requests where bytes aren't " - "a multiple of alignment"); - if (block_size > max_block_size_) { + DMG_ASSERT(block_size % alignment == 0, + "PoolResource shouldn't serve allocation requests where bytes aren't " + "a multiple of alignment"); + if (block_size > max_block_size_) [[unlikely]] { // Deallocate a big block. BigBlock big_block{bytes, alignment, p}; auto it = std::lower_bound(unpooled_.begin(), unpooled_.end(), big_block, @@ -317,4 +317,93 @@ MemoryResource *NullMemoryResource() noexcept { return &res; } +namespace impl { + +/// 1 bit sensitivity test +static_assert(bin_index<1>(9u) == 0); +static_assert(bin_index<1>(10u) == 0); +static_assert(bin_index<1>(11u) == 0); +static_assert(bin_index<1>(12u) == 0); +static_assert(bin_index<1>(13u) == 0); +static_assert(bin_index<1>(14u) == 0); +static_assert(bin_index<1>(15u) == 0); +static_assert(bin_index<1>(16u) == 0); + +static_assert(bin_index<1>(17u) == 1); +static_assert(bin_index<1>(18u) == 1); +static_assert(bin_index<1>(19u) == 1); +static_assert(bin_index<1>(20u) == 1); +static_assert(bin_index<1>(21u) == 1); +static_assert(bin_index<1>(22u) == 1); +static_assert(bin_index<1>(23u) == 1); +static_assert(bin_index<1>(24u) == 1); +static_assert(bin_index<1>(25u) == 1); +static_assert(bin_index<1>(26u) == 1); +static_assert(bin_index<1>(27u) == 1); +static_assert(bin_index<1>(28u) == 1); +static_assert(bin_index<1>(29u) == 1); +static_assert(bin_index<1>(30u) == 1); +static_assert(bin_index<1>(31u) == 1); +static_assert(bin_index<1>(32u) == 1); + +/// 2 bit sensitivity test + +static_assert(bin_index<2>(9u) == 0); +static_assert(bin_index<2>(10u) == 0); +static_assert(bin_index<2>(11u) == 0); +static_assert(bin_index<2>(12u) == 0); + +static_assert(bin_index<2>(13u) == 1); +static_assert(bin_index<2>(14u) == 1); +static_assert(bin_index<2>(15u) == 1); +static_assert(bin_index<2>(16u) == 1); + +static_assert(bin_index<2>(17u) == 2); +static_assert(bin_index<2>(18u) == 2); +static_assert(bin_index<2>(19u) == 2); +static_assert(bin_index<2>(20u) == 2); +static_assert(bin_index<2>(21u) == 2); +static_assert(bin_index<2>(22u) == 2); +static_assert(bin_index<2>(23u) == 2); +static_assert(bin_index<2>(24u) == 2); + +} // namespace impl + +void *PoolResource2::DoAllocate(size_t bytes, size_t alignment) { + // Take the max of `bytes` and `alignment` so that we simplify handling + // alignment requests. + size_t block_size = std::max(bytes, alignment); + // Check that we have received a regular allocation request with non-padded + // structs/classes in play. These will always have + // `sizeof(T) % alignof(T) == 0`. Special requests which don't have that + // property can never be correctly handled with contiguous blocks. We would + // have to write a general-purpose allocator which has to behave as complex + // as malloc/free. + if (block_size % alignment != 0) throw BadAlloc("Requested bytes must be a multiple of alignment"); + + if (pools_4bit_.is_above_upper_bound(block_size)) return unpooled_memory_->Allocate(bytes, alignment); + if (pools_2bit_.is_size_handled(block_size)) return pools_2bit_.allocate(block_size); + if (pools_3bit_.is_size_handled(block_size)) return pools_3bit_.allocate(block_size); + if (pools_4bit_.is_size_handled(block_size)) return pools_4bit_.allocate(block_size); + DMG_ASSERT(block_size <= 8); + return pool_8_.Allocate(); +} +void PoolResource2::DoDeallocate(void *p, size_t bytes, size_t alignment) { + size_t block_size = std::max(bytes, alignment); + DMG_ASSERT(block_size % alignment == 0); + + if (pools_4bit_.is_above_upper_bound(block_size)) { + unpooled_memory_->Deallocate(p, bytes, alignment); + } else if (pools_2bit_.is_size_handled(block_size)) { + pools_2bit_.deallocate(p, block_size); + } else if (pools_3bit_.is_size_handled(block_size)) { + pools_3bit_.deallocate(p, block_size); + } else if (pools_4bit_.is_size_handled(block_size)) { + pools_4bit_.deallocate(p, block_size); + } else { + DMG_ASSERT(block_size <= 8); + pool_8_.Deallocate(p); + } +} +bool PoolResource2::DoIsEqual(MemoryResource const &other) const noexcept { return this == &other; } } // namespace memgraph::utils diff --git a/src/utils/memory.hpp b/src/utils/memory.hpp index 2ed6991c0..3013b1b73 100644 --- a/src/utils/memory.hpp +++ b/src/utils/memory.hpp @@ -15,6 +15,7 @@ #pragma once +#include #include #include #include @@ -449,8 +450,170 @@ class Pool final { void Release(); }; +// C++ overloads for clz +constexpr auto clz(unsigned int x) { return __builtin_clz(x); } +constexpr auto clz(unsigned long x) { return __builtin_clzl(x); } +constexpr auto clz(unsigned long long x) { return __builtin_clzll(x); } + +template +constexpr auto bits_sizeof = sizeof(T) * CHAR_BIT; + +/// 0-based bit index of the most significant bit assumed that `n` != 0 +template +constexpr auto msb_index(T n) { + return bits_sizeof - clz(n) - T(1); +} + +/* This function will in O(1) time provide a bin index based on: + * B - the number of most significant bits to be sensitive to + * LB - the value that should be considered below the consideration for bin index of 0 (LB is exclusive) + * + * lets say we were: + * - sensitive to two bits (B == 2) + * - lowest bin is for 8 (LB == 8) + * + * our bin indexes would look like: + * 0 - 0000'1100 12 + * 1 - 0001'0000 16 + * 2 - 0001'1000 24 + * 3 - 0010'0000 32 + * 4 - 0011'0000 48 + * 5 - 0100'0000 64 + * 6 - 0110'0000 96 + * 7 - 1000'0000 128 + * 8 - 1100'0000 192 + * ... + * + * Example: + * Given n == 70, we want to return the bin index to the first value which is + * larger than n. + * bin_index<2,8>(70) => 6, as 64 (index 5) < 70 and 70 <= 96 (index 6) + */ +template +constexpr std::size_t bin_index(std::size_t n) { + static_assert(B >= 1U, "Needs to be sensitive to at least one bit"); + static_assert(LB != 0U, "Lower bound need to be non-zero"); + DMG_ASSERT(n > LB); + + // We will alway be sensitive to at least the MSB + // exponent tells us how many bits we need to use to select within a level + constexpr auto kExponent = B - 1U; + // 2^exponent gives the size of each level + constexpr auto kSize = 1U << kExponent; + // offset help adjust results down to be inline with bin_index(LB) == 0 + constexpr auto kOffset = msb_index(LB); + + auto const msb_idx = msb_index(n); + DMG_ASSERT(msb_idx != 0); + + auto const mask = (1u << msb_idx) - 1u; + auto const under = n & mask; + auto const selector = under >> (msb_idx - kExponent); + + auto const rest = under & (mask >> kExponent); + auto const no_overflow = rest == 0U; + + auto const msb_level = kSize * (msb_idx - kOffset); + return msb_level + selector - no_overflow; +} + +// This is the inverse opperation for bin_index +// bin_size(bin_index(X)-1) < X <= bin_size(bin_index(X)) +template +std::size_t bin_size(std::size_t idx) { + constexpr auto kExponent = B - 1U; + constexpr auto kSize = 1U << kExponent; + constexpr auto kOffset = msb_index(LB); + + // no need to optimise `/` or `%` compiler can see `kSize` is a power of 2 + auto const level = (idx + 1) / kSize; + auto const sub_level = (idx + 1) % kSize; + return (1U << (level + kOffset)) | (sub_level << (level + kOffset - kExponent)); +} + +template +struct MultiPool { + static_assert(LB < UB, "lower bound must be less than upper bound"); + static_assert(IsPow2(LB) && IsPow2(UB), "Design untested for non powers of 2"); + + // upper bound is inclusive + static bool is_size_handled(std::size_t size) { return LB < size && size <= UB; } + static bool is_above_upper_bound(std::size_t size) { return UB < size; } + + static constexpr auto n_bins = bin_index(UB) + 1U; + + MultiPool(uint8_t blocks_per_chunk, MemoryResource *memory, MemoryResource *internal_memory) + : blocks_per_chunk_{blocks_per_chunk}, memory_{memory}, internal_memory_{internal_memory} {} + + ~MultiPool() { + if (pools_) { + auto pool_alloc = Allocator(internal_memory_); + for (auto i = 0U; i != n_bins; ++i) { + pool_alloc.destroy(&pools_[i]); + } + pool_alloc.deallocate(pools_, n_bins); + } + } + + void *allocate(std::size_t bytes) { + auto idx = bin_index(bytes); + if (!pools_) initialise_pools(); + return pools_[idx].Allocate(); + } + + void deallocate(void *ptr, std::size_t bytes) { + auto idx = bin_index(bytes); + pools_[idx].Deallocate(ptr); + } + + private: + void initialise_pools() { + auto pool_alloc = Allocator(internal_memory_); + auto pools = pool_alloc.allocate(n_bins); + try { + for (auto i = 0U; i != n_bins; ++i) { + auto block_size = bin_size(i); + pool_alloc.construct(&pools[i], block_size, blocks_per_chunk_, memory_); + } + pools_ = pools; + } catch (...) { + pool_alloc.deallocate(pools, n_bins); + throw; + } + } + + Pool *pools_{}; + uint8_t blocks_per_chunk_{}; + MemoryResource *memory_{}; + MemoryResource *internal_memory_{}; +}; + } // namespace impl +class PoolResource2 final : public MemoryResource { + public: + PoolResource2(uint8_t blocks_per_chunk, MemoryResource *memory = NewDeleteResource(), + MemoryResource *internal_memory = NewDeleteResource()) + : pool_8_(8, blocks_per_chunk, memory), + pools_2bit_(blocks_per_chunk, memory, internal_memory), + pools_3bit_(blocks_per_chunk, memory, internal_memory), + pools_4bit_(blocks_per_chunk, memory, internal_memory), + unpooled_memory_{internal_memory} {} + ~PoolResource2() override = default; + + private: + void *DoAllocate(size_t bytes, size_t alignment) override; + void DoDeallocate(void *p, size_t bytes, size_t alignment) override; + bool DoIsEqual(MemoryResource const &other) const noexcept override; + + private: + impl::Pool pool_8_; + impl::MultiPool<2, 8, 128> pools_2bit_; + impl::MultiPool<3, 128, 512> pools_3bit_; + impl::MultiPool<4, 512, 1024> pools_4bit_; + MemoryResource *unpooled_memory_; +}; + /// MemoryResource which serves allocation requests for different block sizes. /// /// PoolResource is not thread-safe!