From cefec58e9a1a1fdd77e24659845bec357507bb9f Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 2 Jul 2026 03:40:38 -0700 Subject: [PATCH] Remove fragmentation profile The fragmentation profile is not useful in the current form. The proxy allocation stacks are not correlated and are not the cause of external fragmentation. They are just random objects of a particular size class. We don't see that these profiles are used in practice. External fragmentation of more of a global tcmalloc property that depends on CFL span prioritization, etc. The information about external fragmentation is available in a more complete form in mallocz CFL info, and if necessary can be crossed with heapz for representative stacks. PiperOrigin-RevId: 941602092 --- tcmalloc/allocation_sampling.cc | 49 ---------- tcmalloc/allocation_sampling.h | 89 +++---------------- tcmalloc/central_freelist_test.cc | 41 --------- tcmalloc/common.h | 11 --- tcmalloc/experimental_pow2_size_class.cc | 4 +- tcmalloc/internal/logging.h | 9 -- tcmalloc/legacy_size_classes.cc | 30 +++---- tcmalloc/malloc_extension.h | 3 +- .../reuse_relaxed_below_64_size_classes.cc | 14 +-- tcmalloc/size_classes.cc | 16 ++-- tcmalloc/span.cc | 24 ----- tcmalloc/span.h | 4 - tcmalloc/span_test.cc | 10 --- tcmalloc/tcmalloc.cc | 24 ++--- tcmalloc/testing/profile_test.cc | 68 -------------- 15 files changed, 58 insertions(+), 338 deletions(-) diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index eedc410d3..5af8d2358 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -48,41 +48,6 @@ GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc::tcmalloc_internal { -std::unique_ptr DumpFragmentationProfile(Static& state) { - auto profile = std::make_unique(ProfileType::kFragmentation); - state.sampled_allocation_recorder().Iterate( - [&state, &profile](const SampledAllocation& sampled_allocation) { - // Compute fragmentation to charge to this sample: - const StackTrace& t = sampled_allocation.sampled_stack; - if (t.proxy == nullptr) { - // There is just one object per-span, and neighboring spans - // can be released back to the system, so we charge no - // fragmentation to this sampled object. - return; - } - - // Fetch the span on which the proxy lives so we can examine its - // co-residents. - const PageId p = PageIdContaining(t.proxy); - Span* span = state.pagemap().GetDescriptor(p); - if (span == nullptr || span == &state.invalid_span()) { - // Avoid crashes in production mode code, but report in tests. - TC_ASSERT_NE(span, nullptr); - TC_ASSERT_NE(span, &state.invalid_span()); - return; - } - - const double frag = span->Fragmentation(t.allocated_size); - if (frag > 0) { - // Associate the memory warmth with the actual object, not the proxy. - // The residency information (t.span_start_address) is likely not very - // useful, but we might as well pass it along. - profile->AddTrace(frag, t); - } - }); - return profile; -} - std::unique_ptr DumpHeapProfile(Static& state) { auto profile = std::make_unique(ProfileType::kHeap); profile->SetStartTime(absl::Now()); @@ -93,19 +58,5 @@ std::unique_ptr DumpHeapProfile(Static& state) { return profile; } -ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr, - size_t size_class) { - if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) { - state.cpu_cache().Deallocate(ptr, size_class); - } else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent(); - ABSL_PREDICT_TRUE(cache)) { - cache->Deallocate(ptr, size_class); - } else { - // This thread doesn't have thread-cache yet or already. Delete directly - // into transfer cache. - state.transfer_cache().InsertRange(size_class, absl::Span(&ptr, 1)); - } -} - } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 7013d5a0c..7db362dcc 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -39,17 +39,6 @@ namespace tcmalloc::tcmalloc_internal { class Static; -// This function computes a profile that maps a live stack trace to -// the number of bytes of central-cache memory pinned by an allocation -// at that stack trace. -// In the case when span is hosting >= 1 number of small objects (t.proxy != -// nullptr), we call span::Fragmentation() and read `span->allocated_`. It is -// safe to do so since we hold the per-sample lock while iterating over sampled -// allocations. It prevents the sampled allocation that has the proxy object to -// complete deallocation, thus `proxy` can not be returned to the span yet. It -// thus prevents the central free list to return the span to the page heap. -std::unique_ptr DumpFragmentationProfile(Static& state); - std::unique_ptr DumpHeapProfile(Static& state); #if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ // For RSEQ enabled builds, we declare the sampler in percpu.h so that we can @@ -78,16 +67,9 @@ inline Sampler& GetThreadSampler() { #endif } -void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); - // Performs sampling for already occurred allocation of object. // -// For very small object sizes, object is used as 'proxy' and full -// page with sampled marked is allocated instead. -// -// For medium-sized objects that have single instance per span, -// they're simply freed and fresh page span is allocated to represent -// sampling. +// For small object sizes, we allocate a new object in a sampled span. // // For large objects (i.e. allocated with do_malloc_pages) they are // also fully reused and their span is marked as sampled. @@ -106,12 +88,10 @@ void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); template ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, void* absl_nullable obj, Span* absl_nullable span) { - TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) || - (size_class == 0 && obj == nullptr && span != nullptr)); + size_t size_class, Span* absl_nullable span) { + TC_CHECK_EQ(size_class != 0, span == nullptr); StackTrace stack_trace; - stack_trace.proxy = nullptr; stack_trace.requested_size = requested_size; // Grab the stack trace outside the heap lock. stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0); @@ -142,8 +122,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( : MemoryTag::kSampled; size_t capacity = 0; if (size_class != 0) { - TC_ASSERT_EQ(size_class, - state.pagemap().sizeclass(PageIdContainingTagged(obj))); state.per_size_class_counts()[size_class].Add(allocation_estimate); stack_trace.allocated_size = state.sizemap().class_to_size(size_class); @@ -172,24 +150,10 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( } capacity = requested_size; } else if ((span = state.page_allocator().New( - num_pages, {1, AccessDensityPrediction::kSparse}, tag)) == - nullptr) { + num_pages, {1, AccessDensityPrediction::kSparse}, tag))) { capacity = stack_trace.allocated_size; - return {obj, capacity}; } else { - capacity = stack_trace.allocated_size; - } - - size_t span_size = - Length(state.sizemap().class_to_pages(size_class)).in_bytes(); - size_t objects_per_span = stack_trace.allocated_size != 0 - ? span_size / stack_trace.allocated_size - : -1ul; - - if (objects_per_span != 1) { - TC_ASSERT_GT(objects_per_span, 1); - stack_trace.proxy = obj; - obj = nullptr; + return {}; } } else { // Set stack_trace.allocated_size to the exact size for a page allocation. @@ -255,12 +219,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( state.peak_heap_tracker().MaybeSaveSample(); - if (obj != nullptr) { - // We are not maintaining precise statistics on malloc hit/miss rates at our - // cache tiers. We can deallocate into our ordinary cache. - TC_ASSERT_NE(size_class, 0); - FreeProxyObject(state, obj, size_class); - } TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0); return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc : span->start_address(), @@ -275,16 +233,15 @@ template static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, Span* span) { - return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr, - span); + return SampleifyAllocation(state, policy, requested_size, weight, 0, span); } template static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, sized_ptr_t res) { + size_t size_class) { return SampleifyAllocation(state, policy, requested_size, weight, size_class, - res.p, nullptr); + nullptr); } // Rewrite type so that the allocation type falls into one of the categories we @@ -334,7 +291,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0); - void* const proxy = sampled_allocation->sampled_stack.proxy; const size_t weight = sampled_allocation->sampled_stack.weight; const size_t requested_size = sampled_allocation->sampled_stack.requested_size; @@ -377,18 +333,14 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, sampled_allocation->sampled_stack.depth)); } - const size_t allocated_alignment = static_cast( - sampled_allocation->sampled_stack.requested_alignment.value_or( - std::align_val_t{1})); - const std::optional deallocated_alignment = - policy.has_explicit_alignment() - ? std::make_optional(policy.align()) - : std::nullopt; - if ((size.has_value() || policy.allocation_type() == AllocationType::New)) { const bool type_mismatch = policy.allocation_type() != sampled_allocation->sampled_stack.allocation_type; + const std::optional deallocated_alignment = + policy.has_explicit_alignment() + ? std::make_optional(policy.align()) + : std::nullopt; const bool alignment_mismatch = deallocated_alignment != sampled_allocation->sampled_stack.requested_alignment; @@ -439,23 +391,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, MallocHook::InvokeSampledDeleteHook(sampled_alloc); state.deallocation_samples.ReportFree(sampled_alloc_handle); - - if (proxy) { - const auto proxy_policy = policy.InSamePartitionAs(proxy); - size_t size_class; - if (AccessFromPointer(proxy) == AllocationAccess::kCold) { - size_class = state.sizemap().SizeClass( - proxy_policy.AccessAsCold().AlignAs(allocated_alignment), - allocated_size); - } else { - size_class = state.sizemap().SizeClass( - proxy_policy.AccessAsHot().AlignAs(allocated_alignment), - allocated_size); - } - TC_ASSERT_EQ(size_class, - state.pagemap().sizeclass(PageIdContainingTagged(proxy))); - FreeProxyObject(state, proxy, size_class); - } } } // namespace tcmalloc::tcmalloc_internal diff --git a/tcmalloc/central_freelist_test.cc b/tcmalloc/central_freelist_test.cc index 08eabc9e5..ad5f71e5d 100644 --- a/tcmalloc/central_freelist_test.cc +++ b/tcmalloc/central_freelist_test.cc @@ -1076,47 +1076,6 @@ TEST_P(CentralFreeListTest, PassSpanDensityToPageheap) { test_function(e.objects_per_span(), AccessDensityPrediction::kDense); } -TEST_P(CentralFreeListTest, SpanFragmentation) { -#if ABSL_HAVE_HWADDRESS_SANITIZER - GTEST_SKIP() - << "Skipping under HWASan, which uses the top bits of the pointer."; -#endif - - // This test is primarily exercising Span itself to model how tcmalloc.cc uses - // it, but this gives us a self-contained (and sanitizable) implementation of - // the CentralFreeList. - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); - // Allocate one object from the CFL to allocate a span. - void* initial; - int got = e.central_freelist().RemoveRange(absl::MakeSpan(&initial, 1)); - ASSERT_EQ(got, 1); - - Span* const span = e.central_freelist().forwarder().MapObjectToSpan(initial); - const size_t object_size = - e.central_freelist().forwarder().class_to_size(TypeParam::kSizeClass); - - ThreadManager fragmentation; - fragmentation.Start(1, [&](int) { - if (e.objects_per_span() != 1) { - benchmark::DoNotOptimize(span->Fragmentation(object_size)); - } - }); - - ThreadManager cfl; - cfl.Start(1, [&](int) { - void* next; - int got = e.central_freelist().RemoveRange(absl::MakeSpan(&next, 1)); - e.central_freelist().InsertRange(absl::MakeSpan(&next, got)); - }); - - absl::SleepFor(absl::Milliseconds(50)); - - fragmentation.Stop(); - cfl.Stop(); - - e.central_freelist().InsertRange(absl::MakeSpan(&initial, 1)); -} - TEST_P(CentralFreeListTest, SpanLifetimeWithLongLivedSpans) { #if ABSL_HAVE_HWADDRESS_SANITIZER GTEST_SKIP() diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 4e7165c73..704861807 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -248,17 +248,6 @@ enum class AllocationAccess { kCold, }; -inline AllocationAccess AccessFromPointer(void* ptr) { - if (!kHasExpandedClasses) { - TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold); - return AllocationAccess::kHot; - } - - return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold) - ? AllocationAccess::kCold - : AllocationAccess::kHot; -} - inline MemoryTag MultiNormalTag(size_t partition) { switch (partition) { case 0: diff --git a/tcmalloc/experimental_pow2_size_class.cc b/tcmalloc/experimental_pow2_size_class.cc index be170038b..820be1e33 100644 --- a/tcmalloc/experimental_pow2_size_class.cc +++ b/tcmalloc/experimental_pow2_size_class.cc @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index f2b675bc2..c0cd16d04 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -64,15 +64,6 @@ struct StackTrace { // memory block. AllocHandle sampled_alloc_handle; - // For small sampled objects, we allocate a full span to hold the - // sampled object. However to avoid disturbing fragmentation - // profiles, in such cases we also allocate a small proxy object - // using the normal mechanism. - // - // proxy field is defined only for heap sample stack traces. - // For heap samples, proxy==NULL iff size > kMaxSize. - void* proxy; - uintptr_t requested_size; std::optional requested_alignment; uintptr_t allocated_size; // size after sizeclass/page rounding diff --git a/tcmalloc/legacy_size_classes.cc b/tcmalloc/legacy_size_classes.cc index 26d70179e..38f70e817 100644 --- a/tcmalloc/legacy_size_classes.cc +++ b/tcmalloc/legacy_size_classes.cc @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 38 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 39 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 40 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 41 9 2.33% 0.43% 16.67% + { 896, 8192, 32}, // 41 9 2.33% 0.42% 16.67% { 1024, 8192, 32}, // 42 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 43 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 43 14 1.95% 0.42% 12.50% { 1280, 16384, 32}, // 44 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 45 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 46 10 6.61% 0.43% 9.09% @@ -228,7 +228,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -296,17 +296,17 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 46 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 47 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 48 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 49 81 1.15% 12.54% 13.64% + { 3200, 262144, 20}, // 49 81 1.15% 12.53% 13.64% { 3584, 262144, 18}, // 50 73 0.22% 12.53% 12.00% { 4096, 262144, 16}, // 51 64 0.02% 12.53% 14.29% { 4224, 262144, 15}, // 52 62 0.12% 12.53% 3.12% - { 4736, 262144, 13}, // 53 55 0.66% 12.54% 12.12% + { 4736, 262144, 13}, // 53 55 0.66% 12.53% 12.12% { 5120, 262144, 12}, // 54 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 55 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 56 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 57 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 58 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 59 34 0.41% 12.54% 9.09% + { 7680, 262144, 8}, // 59 34 0.41% 12.53% 9.09% { 8192, 262144, 8}, // 60 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 61 28 0.22% 12.53% 14.06% { 11392, 262144, 5}, // 62 23 0.07% 12.53% 21.92% @@ -330,7 +330,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% @@ -439,7 +439,7 @@ static constexpr SizeClassInfo List[] = { { 368, 8192, 32}, // 24 22 1.94% 0.42% 4.55% { 384, 8192, 32}, // 25 21 2.33% 0.42% 4.35% { 400, 8192, 32}, // 26 20 3.10% 0.42% 4.17% - { 416, 8192, 32}, // 27 19 4.26% 0.43% 4.00% + { 416, 8192, 32}, // 27 19 4.26% 0.42% 4.00% { 448, 8192, 32}, // 28 18 2.33% 0.42% 7.69% { 480, 8192, 32}, // 29 17 1.16% 0.42% 7.14% { 512, 8192, 32}, // 30 16 0.78% 0.42% 6.67% @@ -447,9 +447,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 32 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 33 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 34 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 35 9 2.33% 0.43% 16.67% + { 896, 8192, 32}, // 35 9 2.33% 0.42% 16.67% { 1024, 8192, 32}, // 36 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 37 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 37 14 1.95% 0.42% 12.50% { 1280, 16384, 32}, // 38 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 39 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 40 10 6.61% 0.43% 9.09% @@ -583,7 +583,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -647,19 +647,19 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 42 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 43 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 44 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 45 81 1.15% 12.54% 13.64% + { 3200, 262144, 20}, // 45 81 1.15% 12.53% 13.64% { 3584, 262144, 18}, // 46 73 0.22% 12.53% 12.00% { 3840, 262144, 17}, // 47 68 0.41% 12.53% 7.14% { 4096, 262144, 16}, // 48 64 0.02% 12.53% 6.67% { 4224, 262144, 15}, // 49 62 0.12% 12.53% 3.12% { 4480, 262144, 14}, // 50 58 0.90% 12.54% 6.06% - { 4736, 262144, 13}, // 51 55 0.66% 12.54% 5.71% + { 4736, 262144, 13}, // 51 55 0.66% 12.53% 5.71% { 5120, 262144, 12}, // 52 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 53 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 54 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 55 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 56 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 57 34 0.41% 12.54% 9.09% + { 7680, 262144, 8}, // 57 34 0.41% 12.53% 9.09% { 8192, 262144, 8}, // 58 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 59 28 0.22% 12.53% 14.06% { 10368, 262144, 6}, // 60 25 1.15% 12.54% 10.96% @@ -685,7 +685,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% diff --git a/tcmalloc/malloc_extension.h b/tcmalloc/malloc_extension.h index 0ed8e50dd..cf545012a 100644 --- a/tcmalloc/malloc_extension.h +++ b/tcmalloc/malloc_extension.h @@ -151,7 +151,8 @@ enum class ProfileType { kHeap, // Fragmentation report - kFragmentation, + kFragmentation ABSL_DEPRECATED( + "Fragmentation profiles are no longer collected"), // Sample of objects that were live at a recent peak of total heap usage. The // specifics of when exactly this profile is collected are subject to change. diff --git a/tcmalloc/reuse_relaxed_below_64_size_classes.cc b/tcmalloc/reuse_relaxed_below_64_size_classes.cc index b00ce9eaa..03846e8e3 100644 --- a/tcmalloc/reuse_relaxed_below_64_size_classes.cc +++ b/tcmalloc/reuse_relaxed_below_64_size_classes.cc @@ -78,9 +78,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 18 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 19 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 20 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 21 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 21 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 22 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 23 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 23 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 24 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 25 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 26 4 0.78% 0.42% 14.29% @@ -227,7 +227,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -308,9 +308,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 17 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 18 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 19 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 20 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 20 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 21 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 22 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 22 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 23 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 24 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 25 4 0.78% 0.42% 14.29% @@ -393,7 +393,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -457,7 +457,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/size_classes.cc b/tcmalloc/size_classes.cc index a61a0b0e5..951a012c8 100644 --- a/tcmalloc/size_classes.cc +++ b/tcmalloc/size_classes.cc @@ -76,9 +76,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -161,7 +161,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -225,7 +225,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -305,9 +305,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -390,7 +390,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -454,7 +454,7 @@ static constexpr SizeClassInfo List[] = { { 74880, 524288, 2}, // 43 7 0.04% 12.53% 14.26% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/span.cc b/tcmalloc/span.cc index 1e5764ce3..7d92aa74d 100644 --- a/tcmalloc/span.cc +++ b/tcmalloc/span.cc @@ -81,30 +81,6 @@ SampledAllocation* Span::Unsample() { return sampled_allocation; } -double Span::Fragmentation(size_t object_size) const { - if (object_size == 0) { - // Avoid crashes in production mode code, but report in tests. - TC_ASSERT_NE(object_size, 0); - return 0; - } - const size_t span_objects = bytes_in_span() / object_size; - const size_t live = allocated_.load(std::memory_order_relaxed); - if (live == 0) { - // Avoid crashes in production mode code, but report in tests. - TC_ASSERT_NE(live, 0); - return 0; - } - // Assume that all in-use objects in this span are spread evenly - // through this span. So charge the free space in span evenly - // to each of the live objects. - // A note on units here: StackTraceTable::AddTrace(1, *t) - // represents usage (of whatever kind: heap space, allocation, - // fragmentation) of 1 object of size t->allocated_size. - // So we want to report here the number of objects we are "responsible" - // for pinning - NOT bytes. - return static_cast(span_objects - live) / live; -} - // Freelist organization. // // Partially full spans in CentralFreeList contain a list of free objects diff --git a/tcmalloc/span.h b/tcmalloc/span.h index 0e2b827a6..d10020fdf 100644 --- a/tcmalloc/span.h +++ b/tcmalloc/span.h @@ -183,10 +183,6 @@ class ABSL_CACHELINE_ALIGNED Span final : public SpanList::Elem { // Total memory bytes in the span. size_t bytes_in_span() const; - // Returns internal fragmentation of the span. - // REQUIRES: this is a SMALL_OBJECT span. - double Fragmentation(size_t object_size) const; - // Returns number of objects allocated in the span. uint16_t Allocated() const { return allocated_.load(std::memory_order_relaxed); diff --git a/tcmalloc/span_test.cc b/tcmalloc/span_test.cc index 41e463fd8..81f313ede 100644 --- a/tcmalloc/span_test.cc +++ b/tcmalloc/span_test.cc @@ -129,11 +129,6 @@ TEST_P(SpanTest, FreelistBasic) { for (;;) { size_t n = span_.FreelistPopBatch(absl::MakeSpan(batch, want), size_); popped += n; - EXPECT_NEAR( - span_.Fragmentation(size_), - static_cast(objects_per_span_) / static_cast(popped) - - 1., - 1e-5); EXPECT_EQ(span_.FreelistEmpty(size_), popped == objects_per_span_); for (size_t i = 0; i < n; ++i) { void* p = batch[i]; @@ -192,11 +187,6 @@ TEST_P(SpanTest, FreelistBasicObjIdx) { for (;;) { size_t n = span_.FreelistPopBatch(absl::MakeSpan(batch, want), size_); popped += n; - EXPECT_NEAR( - span_.Fragmentation(size_), - static_cast(objects_per_span_) / static_cast(popped) - - 1., - 1e-5); EXPECT_EQ(span_.FreelistEmpty(size_), popped == objects_per_span_); for (size_t i = 0; i < n; ++i) { void* p = batch[i]; diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index f3158e992..14b9ccccd 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -119,6 +119,7 @@ #include "tcmalloc/sampler.h" #include "tcmalloc/segv_handler.h" #include "tcmalloc/span.h" +#include "tcmalloc/stack_trace_table.h" #include "tcmalloc/static_vars.h" #include "tcmalloc/stats.h" #include "tcmalloc/tcmalloc_policy.h" @@ -278,7 +279,8 @@ extern "C" const ProfileBase* MallocExtension_Internal_SnapshotCurrent( case ProfileType::kHeap: return DumpHeapProfile(tc_globals).release(); case ProfileType::kFragmentation: - return DumpFragmentationProfile(tc_globals).release(); + // Fragmentation profiles are no longer collected. + return nullptr; case ProfileType::kPeakHeap: return tc_globals.peak_heap_tracker().DumpSample().release(); default: @@ -1192,21 +1194,19 @@ alloc_small_sampled_hooks_or_perthread(size_t size, size_t size_class, size_class = ret.size_class; TC_CHECK(ret.is_small); } - void* res; - // If we are here because of sampling, try AllocateFast first. - if (ABSL_PREDICT_TRUE(weight == 0) || - (res = tc_globals.cpu_cache().AllocateFast(size_class)) == nullptr) { + __sized_ptr_t ptr; + if (ABSL_PREDICT_FALSE(weight != 0)) { + ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class); + } else { if (UsePerCpuCache(tc_globals)) { - res = tc_globals.cpu_cache().AllocateSlow(size_class); + ptr.p = tc_globals.cpu_cache().AllocateSlow(size_class); } else { - res = ThreadCache::GetCache()->Allocate(size_class); + ptr.p = ThreadCache::GetCache()->Allocate(size_class); } - if (ABSL_PREDICT_FALSE(res == nullptr)) return policy.handle_oom(size); + ptr.n = tc_globals.sizemap().class_to_size(size_class); } - __sized_ptr_t ptr = {res, tc_globals.sizemap().class_to_size(size_class)}; - if (ABSL_PREDICT_FALSE(weight != 0)) { - ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class, - ptr); + if (ABSL_PREDICT_FALSE(ptr.p == nullptr)) { + return policy.handle_oom(size); } if (Policy::invoke_hooks()) { // Size returning tcmallocs call NewHooks with capacity as requested_size. diff --git a/tcmalloc/testing/profile_test.cc b/tcmalloc/testing/profile_test.cc index 73e18880a..c603bd440 100644 --- a/tcmalloc/testing/profile_test.cc +++ b/tcmalloc/testing/profile_test.cc @@ -310,74 +310,6 @@ TEST(AllocationSampleTest, SampleAccuracy) { } } -TEST(FragmentationzTest, Accuracy) { - // Increase sampling rate to decrease flakiness. - ScopedProfileSamplingInterval ps(512 * 1024); - // Disable GWP-ASan, since it allocates different sizes than normal samples. - ScopedGuardedSamplingInterval gs(-1); - - // a fairly odd allocation size - will be rounded to 128. This lets - // us find our record in the table. - static const size_t kItemSize = 115; - // allocate about 3.5 GiB: - static const size_t kNumItems = 32 * 1024 * 1024; - - std::vector> keep; - std::vector> drop; - // hint expected sizes: - drop.reserve(kNumItems * 8 / 10); - keep.reserve(kNumItems * 2 / 10); - - // We allocate many items, then free 80% of them "randomly". (To - // decrease noise and speed up, we just keep every 5th one exactly.) - for (int i = 0; i < kNumItems; ++i) { - // Ideally we should use a malloc() here, for consistency; but unique_ptr - // doesn't come with a have a "free()" deleter; use ::operator new instead. - (i % 5 == 0 ? keep : drop) - .push_back(std::unique_ptr( - static_cast(::operator new[](kItemSize)))); - } - drop.resize(0); - - // there are at least 64 items per span here. (8/10)^64 = 6.2e-7 ~= 0 - // probability we actually managed to free a page; every page is fragmented. - // We still have 20% or so of it allocated, so we should see 80% of it - // charged to these allocations as fragmentations. - auto profile = MallocExtension::SnapshotCurrent(ProfileType::kFragmentation); - - // Pull out the fragmentationz entry corresponding to this - size_t requested_size = 0; - size_t allocated_size = 0; - size_t sum = 0; - size_t count = 0; - profile.Iterate([&](const Profile::Sample& e) { - if (e.requested_size != kItemSize) return; - - if (requested_size == 0) { - allocated_size = e.allocated_size; - requested_size = e.requested_size; - } else { - // we will usually have single entry in - // profile, but in builds without optimization - // our fast-path code causes same call-site to - // have two different stack traces. Thus we - // expect and deal with second entry for same - // allocation. - EXPECT_EQ(requested_size, e.requested_size); - EXPECT_EQ(allocated_size, e.allocated_size); - } - sum += e.sum; - count += e.count; - }); - - double frag_bytes = sum; - double real_frag_bytes = - static_cast(allocated_size * kNumItems) * 0.8; - // We should be pretty close with this much data. - EXPECT_NEAR(real_frag_bytes, frag_bytes, real_frag_bytes * 0.15) - << " sum = " << sum << " allocated = " << allocated_size - << " requested = " << requested_size << " count = " << count; -} extern "C" { void* __alloc_token_0__ZnwmRKSt9nothrow_t(size_t size, std::nothrow_t);