From 7438b7ff13b3fd70068357c73ff40e8c71843169 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 3 Sep 2026 09:59:51 -0700 Subject: [PATCH] Make SizeMap calculation faster for small sizes Index sizes <= 1024 directly by size, this increases the lookup array size by 896 bytes, but removes 2 instructions from the fast path: 7e5449: 48 8d 47 07 leaq 0x7(%rdi), %rax 7e544d: 48 c1 e8 03 shrq $0x3, %rax PiperOrigin-RevId: 975784186 --- tcmalloc/sizemap.cc | 10 +++--- tcmalloc/sizemap.h | 75 +++++++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/tcmalloc/sizemap.cc b/tcmalloc/sizemap.cc index 8eb1298f7..68c9bc7db 100644 --- a/tcmalloc/sizemap.cc +++ b/tcmalloc/sizemap.cc @@ -36,6 +36,7 @@ #include "tcmalloc/parameters.h" #include "tcmalloc/span.h" #include "tcmalloc/static_vars.h" +#include "tcmalloc/tcmalloc_policy.h" GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc { @@ -230,9 +231,10 @@ bool SizeMap::ValidSizeClasses(absl::Span size_classes) { // Initialize the mapping arrays bool SizeMap::Init(absl::Span size_classes) { // Do some sanity checking on add_amount[]/shift_amount[]/class_array[] - TC_CHECK_EQ(ClassIndex(0), 0); - TC_CHECK_EQ(ClassIndex(kMaxSize), kClassArraySize - 1); - TC_CHECK_EQ(ClassIndex(kLargeSize) + 1, ClassIndex(kLargeSize + 1)); + TC_CHECK_EQ(ClassIndex(CppPolicy(), 0), 0); + TC_CHECK_EQ(ClassIndex(CppPolicy(), kMaxSize), kClassArraySize - 1); + TC_CHECK_EQ(ClassIndex(CppPolicy(), kLargeSize) + 1, + ClassIndex(CppPolicy(), kLargeSize + 1)); static_assert(kSmallSizeAlignment <= static_cast(kAlignment)); static_assert(static_cast(kAlignment) <= 16); @@ -243,7 +245,7 @@ bool SizeMap::Init(absl::Span size_classes) { // Fill in the canonical class array in region 0. for (int c = 1, s = 0; c < kNumClasses && s <= kMaxSize; c++) { for (; s <= class_to_size_[c]; s += kSmallSizeAlignment) { - class_array_[ClassIndex(s)] = c; + class_array_[ClassIndex(CppPolicy(), s)] = c; } } diff --git a/tcmalloc/sizemap.h b/tcmalloc/sizemap.h index 00f170fca..cf99840a9 100644 --- a/tcmalloc/sizemap.h +++ b/tcmalloc/sizemap.h @@ -80,7 +80,7 @@ class SizeMap { // 1025 (1025 + 127 + (120<<7)) / 128 129 // ... // 32768 (32768 + 127 + (120<<7)) / 128 376 - static constexpr int kSmallSizeAlignment = 8; + static constexpr int kSmallSizeAlignment = 1; static constexpr size_t kClassArraySize = kLargeSize / kSmallSizeAlignment + (kMaxSize - kLargeSize) / kLargeSizeAlignment + 1; @@ -136,10 +136,13 @@ class SizeMap { // If size is no more than kMaxSize, compute index of the // class_array[] entry for it, putting the class index in output // parameter idx and returning true. Otherwise return false. - ABSL_ATTRIBUTE_ALWAYS_INLINE static inline bool ClassIndexMaybe(size_t s, - size_t& idx) { + template + ABSL_ATTRIBUTE_ALWAYS_INLINE bool ClassIndexMaybe(Policy policy, size_t s, + size_t& idx, + size_t& size_class) const { if (ABSL_PREDICT_TRUE(s <= kLargeSize)) { idx = (s + kSmallSizeAlignment - 1) / kSmallSizeAlignment; + size_class = SizeClassForIndex(policy, idx); return true; } else if (ABSL_PREDICT_TRUE(s <= kMaxSize)) { idx = ((s + kLargeSizeAlignment - 1 + @@ -147,7 +150,7 @@ class SizeMap { kLargeSize / kLargeSizeAlignment) * kLargeSizeAlignment) / kLargeSizeAlignment); - + size_class = SizeClassForIndex(policy, idx); // TODO(b/64294063): Add a dummy statement to keep the tail of the fast // and the slow paths from being deduplicated, turning the shifts into // a variable-width shift (shr reg, cl on x86); these are especially @@ -158,19 +161,47 @@ class SizeMap { // // See SLOW_PATH_BARRIER() in tcmalloc.cc for more information // about this technique. - asm volatile(""); + asm(""); return true; } return false; } - ABSL_ATTRIBUTE_ALWAYS_INLINE static inline size_t ClassIndex(size_t s) { - size_t ret; - TC_CHECK(ClassIndexMaybe(s, ret)); + template + ABSL_ATTRIBUTE_ALWAYS_INLINE size_t ClassIndex(Policy policy, + size_t s) const { + size_t ret, size_class; + TC_CHECK(ClassIndexMaybe(policy, s, ret, size_class)); return ret; } + template + ABSL_ATTRIBUTE_ALWAYS_INLINE size_t SizeClassForIndex(Policy policy, + size_t idx) const { + // Note, if security heap partitioning is enabled, only data (region 0) + // is added to the cold heap. See the comment for kTotalClassArraySize + // for more details. + if (kHasColdClasses && policy.is_cold()) { + TC_ASSERT(policy.allocation_type() == AllocationType::New); + TC_ASSERT_LT(idx + (policy.security_partition() + kColdRegionsStart) * + kClassArraySize, + kTotalClassArraySize); + return class_array_[idx + + (policy.security_partition() + kColdRegionsStart) * + kClassArraySize]; + } + constexpr size_t kTypeOffset = + policy.allocation_type() != AllocationType::New ? kSecurityPartitions + : 0; + TC_ASSERT_LT( + idx + (policy.security_partition() + kTypeOffset) * kClassArraySize, + kTotalClassArraySize); + return class_array_[idx + (policy.security_partition() + kTypeOffset) * + kClassArraySize] + + policy.scaled_numa_partition(); + } + // Set the specified class_array_ region from region 0 adjusting all // values by `adjust`. void SetClassArrayRegion(size_t region, CompactSizeClass adjust); @@ -237,34 +268,10 @@ class SizeMap { return {false}; } - size_t idx; - if (ABSL_PREDICT_FALSE(!ClassIndexMaybe(size, idx))) { + size_t idx, size_class; + if (ABSL_PREDICT_FALSE(!ClassIndexMaybe(policy, size, idx, size_class))) { return {false}; } - size_t size_class; - // Note, if security heap partitioning is enabled, only data (region 0) - // is added to the cold heap. See the comment for kTotalClassArraySize - // for more details. - if (kHasColdClasses && policy.is_cold()) { - TC_ASSERT(policy.allocation_type() == AllocationType::New); - TC_ASSERT_LT(idx + (policy.security_partition() + kColdRegionsStart) * - kClassArraySize, - kTotalClassArraySize); - size_class = - class_array_[idx + (policy.security_partition() + kColdRegionsStart) * - kClassArraySize]; - } else { - constexpr size_t kTypeOffset = - policy.allocation_type() != AllocationType::New ? kSecurityPartitions - : 0; - TC_ASSERT_LT( - idx + (policy.security_partition() + kTypeOffset) * kClassArraySize, - kTotalClassArraySize); - size_class = - class_array_[idx + (policy.security_partition() + kTypeOffset) * - kClassArraySize] + - policy.scaled_numa_partition(); - } // Don't search for suitably aligned class for operator new // (when alignment is statically known to be no greater than kAlignment).