From c6d7afb94ce836155f50329a72f512a0a9bd8052 Mon Sep 17 00:00:00 2001 From: Chris Kennelly Date: Mon, 31 Aug 2026 09:52:18 -0700 Subject: [PATCH] Optimize HugeRegion::Release using RangeTracker::PrevFreeRange. Add Bitmap::PrevFreeRange and RangeTracker::PrevFreeRange to scan free ranges in reverse. Use PrevFreeRange during adaptive release to scan from back to front directly, avoiding candidate buffering and flattening control flow. Legacy behavior is preserved behind TCMALLOC_INTERNAL_LEGACY_LOCKING. PiperOrigin-RevId: 973934966 --- tcmalloc/huge_region.h | 48 +++++++++++++-- tcmalloc/huge_region_test.cc | 60 +++++++++++++++++++ tcmalloc/internal/range_tracker.h | 34 +++++++++++ tcmalloc/internal/range_tracker_test.cc | 80 +++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 4 deletions(-) diff --git a/tcmalloc/huge_region.h b/tcmalloc/huge_region.h index 4bfae15db..8424bf347 100644 --- a/tcmalloc/huge_region.h +++ b/tcmalloc/huge_region.h @@ -331,18 +331,17 @@ inline void HugeRegion::Put(Range r, bool release) { // sophisticated mechanism similar to Filler/Cache, that accounts for a recent // peak while releasing pages. inline HugeLength HugeRegion::Release(Length desired, bool adaptive_release) { - if (desired == Length(0)) return NHugePages(0); - const Length free_yet_backed = free_backed_count_.in_pages(); const Length to_release = std::min(desired, free_yet_backed); + if (to_release == Length(0)) return NHugePages(0); - HugeLength release_target = NHugePages(0); bool should_unback[kNumHugePages] = {}; +#ifdef TCMALLOC_INTERNAL_LEGACY_LOCKING + HugeLength release_target = NHugePages(0); const int start = adaptive_release ? kNumHugePages - 1 : 0; const int end = adaptive_release ? -1 : kNumHugePages; const int step = adaptive_release ? -1 : 1; - // TODO(b/73749855): Consider optimizing this search by consulting tracker_. for (int i = start; i != end; i += step) { if (backed_[i] && pages_used_[i] == Length(0)) { should_unback[i] = true; @@ -351,6 +350,47 @@ inline HugeLength HugeRegion::Release(Length desired, bool adaptive_release) { if (release_target.in_pages() >= to_release) break; } +#else + const size_t needed = HLFromPages(to_release).raw_num(); + size_t released = 0; + + if (adaptive_release) { + size_t index = tracker_.size(), n; + while (tracker_.PrevFreeRange(index, &index, &n)) { + if (n < kPagesPerHugePage.raw_num()) continue; + const size_t first_hp = (index + kPagesPerHugePage.raw_num() - 1) / + kPagesPerHugePage.raw_num(); + const size_t last_hp = (index + n) / kPagesPerHugePage.raw_num(); + for (size_t i = last_hp; i > first_hp; --i) { + const size_t hp = i - 1; + if (!backed_[hp]) continue; + TC_ASSERT_EQ(pages_used_[hp], Length(0)); + should_unback[hp] = true; + if (++released == needed) break; + } + if (released == needed) break; + } + } else { + size_t index = 0, n; + while (tracker_.NextFreeRange(index, &index, &n)) { + if (n < kPagesPerHugePage.raw_num()) { + index += n; + continue; + } + const size_t first_hp = (index + kPagesPerHugePage.raw_num() - 1) / + kPagesPerHugePage.raw_num(); + const size_t last_hp = (index + n) / kPagesPerHugePage.raw_num(); + for (size_t hp = first_hp; hp < last_hp; ++hp) { + if (!backed_[hp]) continue; + TC_ASSERT_EQ(pages_used_[hp], Length(0)); + should_unback[hp] = true; + if (++released == needed) break; + } + if (released == needed) break; + index += n; + } + } +#endif // TCMALLOC_INTERNAL_LEGACY_LOCKING return UnbackHugepages(should_unback); } diff --git a/tcmalloc/huge_region_test.cc b/tcmalloc/huge_region_test.cc index 148a791c5..d25672412 100644 --- a/tcmalloc/huge_region_test.cc +++ b/tcmalloc/huge_region_test.cc @@ -315,6 +315,66 @@ TEST_F(HugeRegionTest, ReleaseAdaptive) { CheckMock(); } +TEST_F(HugeRegionTest, ReleaseAdaptiveFragmented) { + const Length n = kPagesPerHugePage; + bool from_released; + std::optional allocs[8]; + for (int i = 0; i < 8; ++i) { + allocs[i] = Allocate(n, &from_released); + EXPECT_TRUE(from_released); + } + + // Delete hugepages: 0, 1, 3, 4, 6, 7 while keeping 2 and 5 allocated. + Delete(*allocs[0]); + Delete(*allocs[1]); + Delete(*allocs[3]); + Delete(*allocs[4]); + Delete(*allocs[6]); + Delete(*allocs[7]); + + // Free backed hugepages are: [0, 1], [3, 4], [6, 7]. + // Reverse order should release: 7, 6 from the last range, then 4 from the + // middle range. + ExpectUnback({p_ + NHugePages(4), NHugePages(1)}); + ExpectUnback({p_ + NHugePages(6), NHugePages(2)}); + EXPECT_EQ(NHugePages(3), region_.Release(NHugePages(3).in_pages(), + /*adaptive_release=*/true)); + CheckMock(); + + Delete(*allocs[2]); + Delete(*allocs[5]); +} + +TEST_F(HugeRegionTest, ReleaseNonAdaptiveFragmented) { + const Length n = kPagesPerHugePage; + bool from_released; + std::optional allocs[8]; + for (int i = 0; i < 8; ++i) { + allocs[i] = Allocate(n, &from_released); + EXPECT_TRUE(from_released); + } + + // Delete hugepages: 0, 1, 3, 4, 6, 7 while keeping 2 and 5 allocated. + Delete(*allocs[0]); + Delete(*allocs[1]); + Delete(*allocs[3]); + Delete(*allocs[4]); + Delete(*allocs[6]); + Delete(*allocs[7]); + + // Free backed hugepages are: [0, 1], [3, 4], [6, 7]. + // Forward order should release: 0, 1 from the first range, then 3 from the + // middle range. + ExpectUnback({p_, NHugePages(2)}); + ExpectUnback({p_ + NHugePages(3), NHugePages(1)}); + EXPECT_EQ(NHugePages(3), region_.Release(NHugePages(3).in_pages(), + /*adaptive_release=*/false)); + CheckMock(); + + Delete(*allocs[2]); + Delete(*allocs[5]); +} + TEST_F(HugeRegionTest, ReleaseFailure) { const Length n = kPagesPerHugePage; bool from_released; diff --git a/tcmalloc/internal/range_tracker.h b/tcmalloc/internal/range_tracker.h index c7f4be154..df6fc9ddc 100644 --- a/tcmalloc/internal/range_tracker.h +++ b/tcmalloc/internal/range_tracker.h @@ -81,6 +81,12 @@ class Bitmap { // If there is at least one free range at or after , // put it in *index, *length and return true; else return false. bool NextFreeRange(size_t start, size_t* index, size_t* length) const; + // If there is at least one free range before , + // put it in *index, *length and return true; else return false. + // To iterate backwards through all free ranges: + // size_t index = bitmap.size(), n; + // while (bitmap.PrevFreeRange(index, &index, &n)) { ... } + bool PrevFreeRange(size_t end, size_t* index, size_t* length) const; // Returns index of the first {true, false} bit >= index, or N if none. size_t FindSet(size_t index) const; @@ -166,6 +172,12 @@ class RangeTracker { // If there is at least one free range at or after , // put it in *index, *length and return true; else return false. bool NextFreeRange(size_t start, size_t* index, size_t* length) const; + // If there is at least one free range before , + // put it in *index, *length and return true; else return false. + // To iterate backwards through all free ranges: + // size_t index = tracker.size(), n; + // while (tracker.PrevFreeRange(index, &index, &n)) { ... } + bool PrevFreeRange(size_t end, size_t* index, size_t* length) const; void Clear(); @@ -331,6 +343,14 @@ inline bool RangeTracker::NextFreeRange(size_t start, size_t* index, return bits_.NextFreeRange(start, index, length); } +// If there is at least one free range before , +// put it in *index, *length and return true; else return false. +template +inline bool RangeTracker::PrevFreeRange(size_t end, size_t* index, + size_t* length) const { + return bits_.PrevFreeRange(end, index, length); +} + template inline void RangeTracker::Clear() { bits_.Clear(); @@ -495,6 +515,20 @@ inline bool Bitmap::NextFreeRange(size_t start, size_t* index, return true; } +template +inline bool Bitmap::PrevFreeRange(size_t end, size_t* index, + size_t* length) const { + if (end == 0) return false; + if (end > N) end = N; + ssize_t j = FindClearBackwards(end - 1); + if (j < 0) return false; + ssize_t i = FindSetBackwards(j); + size_t range_start = (i < 0) ? 0 : static_cast(i + 1); + *index = range_start; + *length = static_cast(j + 1) - range_start; + return true; +} + template inline size_t Bitmap::FindSet(size_t index) const { return FindValue(index); diff --git a/tcmalloc/internal/range_tracker_test.cc b/tcmalloc/internal/range_tracker_test.cc index 245a3bdaa..ce1c7a9b9 100644 --- a/tcmalloc/internal/range_tracker_test.cc +++ b/tcmalloc/internal/range_tracker_test.cc @@ -314,6 +314,71 @@ TEST_F(BitmapTest, PopBatch) { EXPECT_TRUE(map.IsZero()); } +TEST_F(BitmapTest, PrevFreeRange) { + // Empty bitmap (all free). + { + Bitmap<253> map; + size_t index = map.size(), len = 0; + EXPECT_TRUE(map.PrevFreeRange(index, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 253); + EXPECT_FALSE(map.PrevFreeRange(index, &index, &len)); + } + + // Full bitmap (no free ranges). + { + Bitmap<253> map; + map.SetRange(0, 253); + size_t index = map.size(), len = 0; + EXPECT_FALSE(map.PrevFreeRange(index, &index, &len)); + } + + // Bounds and edge conditions. + { + Bitmap<64> map; + size_t index = 0, len = 0; + EXPECT_FALSE(map.PrevFreeRange(0, &index, &len)); + + // end > N clamps to N. + EXPECT_TRUE(map.PrevFreeRange(100, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 64); + + // Partial query: end inside a free range [0, 64). + EXPECT_TRUE(map.PrevFreeRange(30, &index, &len)); + EXPECT_EQ(index, 0); + EXPECT_EQ(len, 30); + } + + // Multiple ranges matching forward traversal in reverse. + { + Bitmap<253> map; + map.SetRange(0, 253); + // Open free ranges: [10, 20), [50, 80), [120, 150), [200, 253) + map.ClearRange(10, 10); + map.ClearRange(50, 30); + map.ClearRange(120, 30); + map.ClearRange(200, 53); + + std::vector> forward_ranges; + size_t f_index = 0, f_len; + while (map.NextFreeRange(f_index, &f_index, &f_len)) { + forward_ranges.push_back({f_index, f_len}); + f_index += f_len; + } + EXPECT_THAT(forward_ranges, ElementsAre(Pair(10, 10), Pair(50, 30), + Pair(120, 30), Pair(200, 53))); + + std::vector> backward_ranges; + size_t b_index = map.size(), b_len; + while (map.PrevFreeRange(b_index, &b_index, &b_len)) { + backward_ranges.push_back({b_index, b_len}); + } + EXPECT_THAT(backward_ranges, ElementsAre(Pair(200, 53), Pair(120, 30), + Pair(50, 30), Pair(10, 10))); + } +} + class RangeTrackerTest : public ::testing::Test { protected: std::vector> FreeRanges() { @@ -325,6 +390,14 @@ class RangeTrackerTest : public ::testing::Test { } return ret; } + std::vector> FreeRangesBackwards() { + std::vector> ret; + size_t index = range_.size(), len; + while (range_.PrevFreeRange(index, &index, &len)) { + ret.push_back({index, len}); + } + return ret; + } static constexpr size_t kBits = 1017; RangeTracker range_; }; @@ -334,23 +407,28 @@ TEST_F(RangeTrackerTest, Trivial) { EXPECT_EQ(0, range_.used()); EXPECT_EQ(kBits, range_.longest_free()); EXPECT_THAT(FreeRanges(), ElementsAre(Pair(0, kBits))); + EXPECT_THAT(FreeRangesBackwards(), ElementsAre(Pair(0, kBits))); ASSERT_EQ(0, range_.FindAndMark(kBits)); EXPECT_EQ(0, range_.longest_free()); EXPECT_EQ(kBits, range_.used()); EXPECT_THAT(FreeRanges(), ElementsAre()); + EXPECT_THAT(FreeRangesBackwards(), ElementsAre()); range_.Unmark(0, 100); EXPECT_EQ(100, range_.longest_free()); EXPECT_EQ(kBits - 100, range_.used()); EXPECT_THAT(FreeRanges(), ElementsAre(Pair(0, 100))); + EXPECT_THAT(FreeRangesBackwards(), ElementsAre(Pair(0, 100))); // non-contiguous - shouldn't increase longest range_.Unmark(200, 100); EXPECT_EQ(100, range_.longest_free()); EXPECT_EQ(kBits - 200, range_.used()); EXPECT_THAT(FreeRanges(), ElementsAre(Pair(0, 100), Pair(200, 100))); + EXPECT_THAT(FreeRangesBackwards(), ElementsAre(Pair(200, 100), Pair(0, 100))); range_.Unmark(100, 100); EXPECT_EQ(300, range_.longest_free()); EXPECT_EQ(kBits - 300, range_.used()); EXPECT_THAT(FreeRanges(), ElementsAre(Pair(0, 300))); + EXPECT_THAT(FreeRangesBackwards(), ElementsAre(Pair(0, 300))); } TEST_F(RangeTrackerTest, Mark) { @@ -359,6 +437,8 @@ TEST_F(RangeTrackerTest, Mark) { EXPECT_EQ(range_.used(), 100); EXPECT_EQ(range_.longest_free(), kBits - 200); EXPECT_THAT(FreeRanges(), ElementsAre(Pair(0, 100), Pair(200, kBits - 200))); + EXPECT_THAT(FreeRangesBackwards(), + ElementsAre(Pair(200, kBits - 200), Pair(0, 100))); range_.Unmark(100, 100); EXPECT_EQ(range_.used(), 0); EXPECT_EQ(range_.longest_free(), kBits);