From 749ae0982e31016ef02b287427c4fbd124a7d369 Mon Sep 17 00:00:00 2001 From: Connal de Souza Date: Fri, 11 Sep 2026 11:46:52 -0700 Subject: [PATCH] Fix slab prefetch address on Pop fast path. The result is at current - 1, the pointer to the next object is at current -2 (where the PrefetchNextObject targets), and the slab address to prefetch is at current - 3. PiperOrigin-RevId: 979934875 --- tcmalloc/internal/percpu_tcmalloc.h | 16 ++++++++-------- .../testing/fast_path.release+insecure.golden | 2 +- tcmalloc/testing/fast_path.release.golden | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tcmalloc/internal/percpu_tcmalloc.h b/tcmalloc/internal/percpu_tcmalloc.h index ce8ace58b..b947a1800 100644 --- a/tcmalloc/internal/percpu_tcmalloc.h +++ b/tcmalloc/internal/percpu_tcmalloc.h @@ -835,7 +835,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( TC_ASSERT_NE(size_class, 0); void* next; void* result; - uintptr_t scratch, current; + uintptr_t tcmalloc_slabs_addr, current; asm goto(TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Pop) // scratch = tcmalloc_slabs; @@ -855,7 +855,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( "subl $0xffff0001, (%[scratch], %[size_class], 4)\n" // Commit "5:\n" - : [result] "=&r"(result), [scratch] "=&r"(scratch), + : [result] "=&r"(result), [scratch] "=&r"(tcmalloc_slabs_addr), [current] "=&r"(current), [next] "=&r"(next) : TCMALLOC_RSEQ_INPUTS, [begin_mark_mask] "n"(kBeginMark), [size_class] "r"(size_class) @@ -865,9 +865,9 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( TC_ASSERT(result); TSANAcquire(result); - // The next pop will be from current-1, but because we prefetch the previous - // element we've already just read that, so prefetch current-2. - PrefetchSlabMemory(scratch + (current - 2) * sizeof(void*)); + // The next pop will be from current-2, but because we prefetch the previous + // element we've already just read that, so prefetch current-3. + PrefetchSlabMemory(tcmalloc_slabs_addr + (current - 3) * sizeof(void*)); PrefetchNextObject(next); return AssumeNotNull(result); underflow_path: @@ -884,7 +884,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( void* region_start; void* prefetch; uintptr_t scratch; - uintptr_t current; + uintptr_t current_plus_slabs_addr; asm goto( TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Pop) // region_start = tcmalloc_slabs; @@ -908,7 +908,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( // Commit "5:\n" : [result] "=&r"(result), [prefetch] "=&r"(prefetch), - [current] "=&r"(current), + [current] "=&r"(current_plus_slabs_addr), // Temps [region_start] "=&r"(region_start), [scratch] "=&r"(scratch) // Real inputs @@ -921,7 +921,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( // The next pop will be from current-2, but because we prefetch the previous // element we've already just read that, so prefetch current-3. - PrefetchSlabMemory(current - 3 * sizeof(void*)); + PrefetchSlabMemory(current_plus_slabs_addr - 3 * sizeof(void*)); PrefetchNextObject(prefetch); return AssumeNotNull(result); underflow_path: diff --git a/tcmalloc/testing/fast_path.release+insecure.golden b/tcmalloc/testing/fast_path.release+insecure.golden index 258d82064..901698d70 100644 --- a/tcmalloc/testing/fast_path.release+insecure.golden +++ b/tcmalloc/testing/fast_path.release+insecure.golden @@ -12,7 +12,7 @@ new(align)->size - - 5 - 6 - 7 - - - 2 - new(align,cold)->size - 1 6 - 6 - 8 - - - 2 - 1 new(align,nothrow) - - 5 - 6 - 6 - - - 2 - 1 new(cold)->size - 1 4 - 4 - 7 - - - 2 - 1 -new(cold,token_1)->size - 1 4 - 4 - 7 - - - 2 - 1 +new(cold,token_1)->size - - 4 - 5 - 7 - - - 2 - 1 new(nothrow) - - 3 - 4 - 5 - - - 2 - 1 new(token_1) - - 3 - 4 - 5 - - - 2 - 1 new(token_1)->size - - 3 - 4 - 6 - - - 2 - 1 diff --git a/tcmalloc/testing/fast_path.release.golden b/tcmalloc/testing/fast_path.release.golden index 258d82064..901698d70 100644 --- a/tcmalloc/testing/fast_path.release.golden +++ b/tcmalloc/testing/fast_path.release.golden @@ -12,7 +12,7 @@ new(align)->size - - 5 - 6 - 7 - - - 2 - new(align,cold)->size - 1 6 - 6 - 8 - - - 2 - 1 new(align,nothrow) - - 5 - 6 - 6 - - - 2 - 1 new(cold)->size - 1 4 - 4 - 7 - - - 2 - 1 -new(cold,token_1)->size - 1 4 - 4 - 7 - - - 2 - 1 +new(cold,token_1)->size - - 4 - 5 - 7 - - - 2 - 1 new(nothrow) - - 3 - 4 - 5 - - - 2 - 1 new(token_1) - - 3 - 4 - 5 - - - 2 - 1 new(token_1)->size - - 3 - 4 - 6 - - - 2 - 1