diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index eedc410d3..5af8d2358 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -48,41 +48,6 @@ GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc::tcmalloc_internal { -std::unique_ptr DumpFragmentationProfile(Static& state) { - auto profile = std::make_unique(ProfileType::kFragmentation); - state.sampled_allocation_recorder().Iterate( - [&state, &profile](const SampledAllocation& sampled_allocation) { - // Compute fragmentation to charge to this sample: - const StackTrace& t = sampled_allocation.sampled_stack; - if (t.proxy == nullptr) { - // There is just one object per-span, and neighboring spans - // can be released back to the system, so we charge no - // fragmentation to this sampled object. - return; - } - - // Fetch the span on which the proxy lives so we can examine its - // co-residents. - const PageId p = PageIdContaining(t.proxy); - Span* span = state.pagemap().GetDescriptor(p); - if (span == nullptr || span == &state.invalid_span()) { - // Avoid crashes in production mode code, but report in tests. - TC_ASSERT_NE(span, nullptr); - TC_ASSERT_NE(span, &state.invalid_span()); - return; - } - - const double frag = span->Fragmentation(t.allocated_size); - if (frag > 0) { - // Associate the memory warmth with the actual object, not the proxy. - // The residency information (t.span_start_address) is likely not very - // useful, but we might as well pass it along. - profile->AddTrace(frag, t); - } - }); - return profile; -} - std::unique_ptr DumpHeapProfile(Static& state) { auto profile = std::make_unique(ProfileType::kHeap); profile->SetStartTime(absl::Now()); @@ -93,19 +58,5 @@ std::unique_ptr DumpHeapProfile(Static& state) { return profile; } -ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr, - size_t size_class) { - if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) { - state.cpu_cache().Deallocate(ptr, size_class); - } else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent(); - ABSL_PREDICT_TRUE(cache)) { - cache->Deallocate(ptr, size_class); - } else { - // This thread doesn't have thread-cache yet or already. Delete directly - // into transfer cache. - state.transfer_cache().InsertRange(size_class, absl::Span(&ptr, 1)); - } -} - } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 7013d5a0c..7db362dcc 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -39,17 +39,6 @@ namespace tcmalloc::tcmalloc_internal { class Static; -// This function computes a profile that maps a live stack trace to -// the number of bytes of central-cache memory pinned by an allocation -// at that stack trace. -// In the case when span is hosting >= 1 number of small objects (t.proxy != -// nullptr), we call span::Fragmentation() and read `span->allocated_`. It is -// safe to do so since we hold the per-sample lock while iterating over sampled -// allocations. It prevents the sampled allocation that has the proxy object to -// complete deallocation, thus `proxy` can not be returned to the span yet. It -// thus prevents the central free list to return the span to the page heap. -std::unique_ptr DumpFragmentationProfile(Static& state); - std::unique_ptr DumpHeapProfile(Static& state); #if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ // For RSEQ enabled builds, we declare the sampler in percpu.h so that we can @@ -78,16 +67,9 @@ inline Sampler& GetThreadSampler() { #endif } -void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); - // Performs sampling for already occurred allocation of object. // -// For very small object sizes, object is used as 'proxy' and full -// page with sampled marked is allocated instead. -// -// For medium-sized objects that have single instance per span, -// they're simply freed and fresh page span is allocated to represent -// sampling. +// For small object sizes, we allocate a new object in a sampled span. // // For large objects (i.e. allocated with do_malloc_pages) they are // also fully reused and their span is marked as sampled. @@ -106,12 +88,10 @@ void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); template ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, void* absl_nullable obj, Span* absl_nullable span) { - TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) || - (size_class == 0 && obj == nullptr && span != nullptr)); + size_t size_class, Span* absl_nullable span) { + TC_CHECK_EQ(size_class != 0, span == nullptr); StackTrace stack_trace; - stack_trace.proxy = nullptr; stack_trace.requested_size = requested_size; // Grab the stack trace outside the heap lock. stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0); @@ -142,8 +122,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( : MemoryTag::kSampled; size_t capacity = 0; if (size_class != 0) { - TC_ASSERT_EQ(size_class, - state.pagemap().sizeclass(PageIdContainingTagged(obj))); state.per_size_class_counts()[size_class].Add(allocation_estimate); stack_trace.allocated_size = state.sizemap().class_to_size(size_class); @@ -172,24 +150,10 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( } capacity = requested_size; } else if ((span = state.page_allocator().New( - num_pages, {1, AccessDensityPrediction::kSparse}, tag)) == - nullptr) { + num_pages, {1, AccessDensityPrediction::kSparse}, tag))) { capacity = stack_trace.allocated_size; - return {obj, capacity}; } else { - capacity = stack_trace.allocated_size; - } - - size_t span_size = - Length(state.sizemap().class_to_pages(size_class)).in_bytes(); - size_t objects_per_span = stack_trace.allocated_size != 0 - ? span_size / stack_trace.allocated_size - : -1ul; - - if (objects_per_span != 1) { - TC_ASSERT_GT(objects_per_span, 1); - stack_trace.proxy = obj; - obj = nullptr; + return {}; } } else { // Set stack_trace.allocated_size to the exact size for a page allocation. @@ -255,12 +219,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( state.peak_heap_tracker().MaybeSaveSample(); - if (obj != nullptr) { - // We are not maintaining precise statistics on malloc hit/miss rates at our - // cache tiers. We can deallocate into our ordinary cache. - TC_ASSERT_NE(size_class, 0); - FreeProxyObject(state, obj, size_class); - } TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0); return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc : span->start_address(), @@ -275,16 +233,15 @@ template static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, Span* span) { - return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr, - span); + return SampleifyAllocation(state, policy, requested_size, weight, 0, span); } template static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, sized_ptr_t res) { + size_t size_class) { return SampleifyAllocation(state, policy, requested_size, weight, size_class, - res.p, nullptr); + nullptr); } // Rewrite type so that the allocation type falls into one of the categories we @@ -334,7 +291,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0); - void* const proxy = sampled_allocation->sampled_stack.proxy; const size_t weight = sampled_allocation->sampled_stack.weight; const size_t requested_size = sampled_allocation->sampled_stack.requested_size; @@ -377,18 +333,14 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, sampled_allocation->sampled_stack.depth)); } - const size_t allocated_alignment = static_cast( - sampled_allocation->sampled_stack.requested_alignment.value_or( - std::align_val_t{1})); - const std::optional deallocated_alignment = - policy.has_explicit_alignment() - ? std::make_optional(policy.align()) - : std::nullopt; - if ((size.has_value() || policy.allocation_type() == AllocationType::New)) { const bool type_mismatch = policy.allocation_type() != sampled_allocation->sampled_stack.allocation_type; + const std::optional deallocated_alignment = + policy.has_explicit_alignment() + ? std::make_optional(policy.align()) + : std::nullopt; const bool alignment_mismatch = deallocated_alignment != sampled_allocation->sampled_stack.requested_alignment; @@ -439,23 +391,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, MallocHook::InvokeSampledDeleteHook(sampled_alloc); state.deallocation_samples.ReportFree(sampled_alloc_handle); - - if (proxy) { - const auto proxy_policy = policy.InSamePartitionAs(proxy); - size_t size_class; - if (AccessFromPointer(proxy) == AllocationAccess::kCold) { - size_class = state.sizemap().SizeClass( - proxy_policy.AccessAsCold().AlignAs(allocated_alignment), - allocated_size); - } else { - size_class = state.sizemap().SizeClass( - proxy_policy.AccessAsHot().AlignAs(allocated_alignment), - allocated_size); - } - TC_ASSERT_EQ(size_class, - state.pagemap().sizeclass(PageIdContainingTagged(proxy))); - FreeProxyObject(state, proxy, size_class); - } } } // namespace tcmalloc::tcmalloc_internal diff --git a/tcmalloc/central_freelist_test.cc b/tcmalloc/central_freelist_test.cc index 08eabc9e5..ad5f71e5d 100644 --- a/tcmalloc/central_freelist_test.cc +++ b/tcmalloc/central_freelist_test.cc @@ -1076,47 +1076,6 @@ TEST_P(CentralFreeListTest, PassSpanDensityToPageheap) { test_function(e.objects_per_span(), AccessDensityPrediction::kDense); } -TEST_P(CentralFreeListTest, SpanFragmentation) { -#if ABSL_HAVE_HWADDRESS_SANITIZER - GTEST_SKIP() - << "Skipping under HWASan, which uses the top bits of the pointer."; -#endif - - // This test is primarily exercising Span itself to model how tcmalloc.cc uses - // it, but this gives us a self-contained (and sanitizable) implementation of - // the CentralFreeList. - TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move); - // Allocate one object from the CFL to allocate a span. - void* initial; - int got = e.central_freelist().RemoveRange(absl::MakeSpan(&initial, 1)); - ASSERT_EQ(got, 1); - - Span* const span = e.central_freelist().forwarder().MapObjectToSpan(initial); - const size_t object_size = - e.central_freelist().forwarder().class_to_size(TypeParam::kSizeClass); - - ThreadManager fragmentation; - fragmentation.Start(1, [&](int) { - if (e.objects_per_span() != 1) { - benchmark::DoNotOptimize(span->Fragmentation(object_size)); - } - }); - - ThreadManager cfl; - cfl.Start(1, [&](int) { - void* next; - int got = e.central_freelist().RemoveRange(absl::MakeSpan(&next, 1)); - e.central_freelist().InsertRange(absl::MakeSpan(&next, got)); - }); - - absl::SleepFor(absl::Milliseconds(50)); - - fragmentation.Stop(); - cfl.Stop(); - - e.central_freelist().InsertRange(absl::MakeSpan(&initial, 1)); -} - TEST_P(CentralFreeListTest, SpanLifetimeWithLongLivedSpans) { #if ABSL_HAVE_HWADDRESS_SANITIZER GTEST_SKIP() diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 4e7165c73..704861807 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -248,17 +248,6 @@ enum class AllocationAccess { kCold, }; -inline AllocationAccess AccessFromPointer(void* ptr) { - if (!kHasExpandedClasses) { - TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold); - return AllocationAccess::kHot; - } - - return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold) - ? AllocationAccess::kCold - : AllocationAccess::kHot; -} - inline MemoryTag MultiNormalTag(size_t partition) { switch (partition) { case 0: diff --git a/tcmalloc/experimental_pow2_size_class.cc b/tcmalloc/experimental_pow2_size_class.cc index be170038b..820be1e33 100644 --- a/tcmalloc/experimental_pow2_size_class.cc +++ b/tcmalloc/experimental_pow2_size_class.cc @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index f2b675bc2..c0cd16d04 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -64,15 +64,6 @@ struct StackTrace { // memory block. AllocHandle sampled_alloc_handle; - // For small sampled objects, we allocate a full span to hold the - // sampled object. However to avoid disturbing fragmentation - // profiles, in such cases we also allocate a small proxy object - // using the normal mechanism. - // - // proxy field is defined only for heap sample stack traces. - // For heap samples, proxy==NULL iff size > kMaxSize. - void* proxy; - uintptr_t requested_size; std::optional requested_alignment; uintptr_t allocated_size; // size after sizeclass/page rounding diff --git a/tcmalloc/legacy_size_classes.cc b/tcmalloc/legacy_size_classes.cc index 26d70179e..38f70e817 100644 --- a/tcmalloc/legacy_size_classes.cc +++ b/tcmalloc/legacy_size_classes.cc @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 38 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 39 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 40 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 41 9 2.33% 0.43% 16.67% + { 896, 8192, 32}, // 41 9 2.33% 0.42% 16.67% { 1024, 8192, 32}, // 42 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 43 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 43 14 1.95% 0.42% 12.50% { 1280, 16384, 32}, // 44 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 45 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 46 10 6.61% 0.43% 9.09% @@ -228,7 +228,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -296,17 +296,17 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 46 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 47 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 48 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 49 81 1.15% 12.54% 13.64% + { 3200, 262144, 20}, // 49 81 1.15% 12.53% 13.64% { 3584, 262144, 18}, // 50 73 0.22% 12.53% 12.00% { 4096, 262144, 16}, // 51 64 0.02% 12.53% 14.29% { 4224, 262144, 15}, // 52 62 0.12% 12.53% 3.12% - { 4736, 262144, 13}, // 53 55 0.66% 12.54% 12.12% + { 4736, 262144, 13}, // 53 55 0.66% 12.53% 12.12% { 5120, 262144, 12}, // 54 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 55 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 56 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 57 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 58 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 59 34 0.41% 12.54% 9.09% + { 7680, 262144, 8}, // 59 34 0.41% 12.53% 9.09% { 8192, 262144, 8}, // 60 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 61 28 0.22% 12.53% 14.06% { 11392, 262144, 5}, // 62 23 0.07% 12.53% 21.92% @@ -330,7 +330,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% @@ -439,7 +439,7 @@ static constexpr SizeClassInfo List[] = { { 368, 8192, 32}, // 24 22 1.94% 0.42% 4.55% { 384, 8192, 32}, // 25 21 2.33% 0.42% 4.35% { 400, 8192, 32}, // 26 20 3.10% 0.42% 4.17% - { 416, 8192, 32}, // 27 19 4.26% 0.43% 4.00% + { 416, 8192, 32}, // 27 19 4.26% 0.42% 4.00% { 448, 8192, 32}, // 28 18 2.33% 0.42% 7.69% { 480, 8192, 32}, // 29 17 1.16% 0.42% 7.14% { 512, 8192, 32}, // 30 16 0.78% 0.42% 6.67% @@ -447,9 +447,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 32 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 33 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 34 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 35 9 2.33% 0.43% 16.67% + { 896, 8192, 32}, // 35 9 2.33% 0.42% 16.67% { 1024, 8192, 32}, // 36 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 37 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 37 14 1.95% 0.42% 12.50% { 1280, 16384, 32}, // 38 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 39 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 40 10 6.61% 0.43% 9.09% @@ -583,7 +583,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -647,19 +647,19 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 42 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 43 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 44 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 45 81 1.15% 12.54% 13.64% + { 3200, 262144, 20}, // 45 81 1.15% 12.53% 13.64% { 3584, 262144, 18}, // 46 73 0.22% 12.53% 12.00% { 3840, 262144, 17}, // 47 68 0.41% 12.53% 7.14% { 4096, 262144, 16}, // 48 64 0.02% 12.53% 6.67% { 4224, 262144, 15}, // 49 62 0.12% 12.53% 3.12% { 4480, 262144, 14}, // 50 58 0.90% 12.54% 6.06% - { 4736, 262144, 13}, // 51 55 0.66% 12.54% 5.71% + { 4736, 262144, 13}, // 51 55 0.66% 12.53% 5.71% { 5120, 262144, 12}, // 52 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 53 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 54 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 55 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 56 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 57 34 0.41% 12.54% 9.09% + { 7680, 262144, 8}, // 57 34 0.41% 12.53% 9.09% { 8192, 262144, 8}, // 58 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 59 28 0.22% 12.53% 14.06% { 10368, 262144, 6}, // 60 25 1.15% 12.54% 10.96% @@ -685,7 +685,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% diff --git a/tcmalloc/malloc_extension.h b/tcmalloc/malloc_extension.h index 0ed8e50dd..cf545012a 100644 --- a/tcmalloc/malloc_extension.h +++ b/tcmalloc/malloc_extension.h @@ -151,7 +151,8 @@ enum class ProfileType { kHeap, // Fragmentation report - kFragmentation, + kFragmentation ABSL_DEPRECATED( + "Fragmentation profiles are no longer collected"), // Sample of objects that were live at a recent peak of total heap usage. The // specifics of when exactly this profile is collected are subject to change. diff --git a/tcmalloc/reuse_relaxed_below_64_size_classes.cc b/tcmalloc/reuse_relaxed_below_64_size_classes.cc index b00ce9eaa..03846e8e3 100644 --- a/tcmalloc/reuse_relaxed_below_64_size_classes.cc +++ b/tcmalloc/reuse_relaxed_below_64_size_classes.cc @@ -78,9 +78,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 18 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 19 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 20 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 21 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 21 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 22 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 23 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 23 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 24 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 25 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 26 4 0.78% 0.42% 14.29% @@ -227,7 +227,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -308,9 +308,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 17 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 18 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 19 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 20 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 20 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 21 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 22 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 22 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 23 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 24 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 25 4 0.78% 0.42% 14.29% @@ -393,7 +393,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -457,7 +457,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/size_classes.cc b/tcmalloc/size_classes.cc index a61a0b0e5..951a012c8 100644 --- a/tcmalloc/size_classes.cc +++ b/tcmalloc/size_classes.cc @@ -76,9 +76,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -161,7 +161,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -225,7 +225,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -305,9 +305,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -390,7 +390,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -454,7 +454,7 @@ static constexpr SizeClassInfo List[] = { { 74880, 524288, 2}, // 43 7 0.04% 12.53% 14.26% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/span.cc b/tcmalloc/span.cc index 1e5764ce3..7d92aa74d 100644 --- a/tcmalloc/span.cc +++ b/tcmalloc/span.cc @@ -81,30 +81,6 @@ SampledAllocation* Span::Unsample() { return sampled_allocation; } -double Span::Fragmentation(size_t object_size) const { - if (object_size == 0) { - // Avoid crashes in production mode code, but report in tests. - TC_ASSERT_NE(object_size, 0); - return 0; - } - const size_t span_objects = bytes_in_span() / object_size; - const size_t live = allocated_.load(std::memory_order_relaxed); - if (live == 0) { - // Avoid crashes in production mode code, but report in tests. - TC_ASSERT_NE(live, 0); - return 0; - } - // Assume that all in-use objects in this span are spread evenly - // through this span. So charge the free space in span evenly - // to each of the live objects. - // A note on units here: StackTraceTable::AddTrace(1, *t) - // represents usage (of whatever kind: heap space, allocation, - // fragmentation) of 1 object of size t->allocated_size. - // So we want to report here the number of objects we are "responsible" - // for pinning - NOT bytes. - return static_cast(span_objects - live) / live; -} - // Freelist organization. // // Partially full spans in CentralFreeList contain a list of free objects diff --git a/tcmalloc/span.h b/tcmalloc/span.h index 0e2b827a6..d10020fdf 100644 --- a/tcmalloc/span.h +++ b/tcmalloc/span.h @@ -183,10 +183,6 @@ class ABSL_CACHELINE_ALIGNED Span final : public SpanList::Elem { // Total memory bytes in the span. size_t bytes_in_span() const; - // Returns internal fragmentation of the span. - // REQUIRES: this is a SMALL_OBJECT span. - double Fragmentation(size_t object_size) const; - // Returns number of objects allocated in the span. uint16_t Allocated() const { return allocated_.load(std::memory_order_relaxed); diff --git a/tcmalloc/span_test.cc b/tcmalloc/span_test.cc index 41e463fd8..81f313ede 100644 --- a/tcmalloc/span_test.cc +++ b/tcmalloc/span_test.cc @@ -129,11 +129,6 @@ TEST_P(SpanTest, FreelistBasic) { for (;;) { size_t n = span_.FreelistPopBatch(absl::MakeSpan(batch, want), size_); popped += n; - EXPECT_NEAR( - span_.Fragmentation(size_), - static_cast(objects_per_span_) / static_cast(popped) - - 1., - 1e-5); EXPECT_EQ(span_.FreelistEmpty(size_), popped == objects_per_span_); for (size_t i = 0; i < n; ++i) { void* p = batch[i]; @@ -192,11 +187,6 @@ TEST_P(SpanTest, FreelistBasicObjIdx) { for (;;) { size_t n = span_.FreelistPopBatch(absl::MakeSpan(batch, want), size_); popped += n; - EXPECT_NEAR( - span_.Fragmentation(size_), - static_cast(objects_per_span_) / static_cast(popped) - - 1., - 1e-5); EXPECT_EQ(span_.FreelistEmpty(size_), popped == objects_per_span_); for (size_t i = 0; i < n; ++i) { void* p = batch[i]; diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index f3158e992..14b9ccccd 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -119,6 +119,7 @@ #include "tcmalloc/sampler.h" #include "tcmalloc/segv_handler.h" #include "tcmalloc/span.h" +#include "tcmalloc/stack_trace_table.h" #include "tcmalloc/static_vars.h" #include "tcmalloc/stats.h" #include "tcmalloc/tcmalloc_policy.h" @@ -278,7 +279,8 @@ extern "C" const ProfileBase* MallocExtension_Internal_SnapshotCurrent( case ProfileType::kHeap: return DumpHeapProfile(tc_globals).release(); case ProfileType::kFragmentation: - return DumpFragmentationProfile(tc_globals).release(); + // Fragmentation profiles are no longer collected. + return nullptr; case ProfileType::kPeakHeap: return tc_globals.peak_heap_tracker().DumpSample().release(); default: @@ -1192,21 +1194,19 @@ alloc_small_sampled_hooks_or_perthread(size_t size, size_t size_class, size_class = ret.size_class; TC_CHECK(ret.is_small); } - void* res; - // If we are here because of sampling, try AllocateFast first. - if (ABSL_PREDICT_TRUE(weight == 0) || - (res = tc_globals.cpu_cache().AllocateFast(size_class)) == nullptr) { + __sized_ptr_t ptr; + if (ABSL_PREDICT_FALSE(weight != 0)) { + ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class); + } else { if (UsePerCpuCache(tc_globals)) { - res = tc_globals.cpu_cache().AllocateSlow(size_class); + ptr.p = tc_globals.cpu_cache().AllocateSlow(size_class); } else { - res = ThreadCache::GetCache()->Allocate(size_class); + ptr.p = ThreadCache::GetCache()->Allocate(size_class); } - if (ABSL_PREDICT_FALSE(res == nullptr)) return policy.handle_oom(size); + ptr.n = tc_globals.sizemap().class_to_size(size_class); } - __sized_ptr_t ptr = {res, tc_globals.sizemap().class_to_size(size_class)}; - if (ABSL_PREDICT_FALSE(weight != 0)) { - ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class, - ptr); + if (ABSL_PREDICT_FALSE(ptr.p == nullptr)) { + return policy.handle_oom(size); } if (Policy::invoke_hooks()) { // Size returning tcmallocs call NewHooks with capacity as requested_size. diff --git a/tcmalloc/testing/profile_test.cc b/tcmalloc/testing/profile_test.cc index 73e18880a..c603bd440 100644 --- a/tcmalloc/testing/profile_test.cc +++ b/tcmalloc/testing/profile_test.cc @@ -310,74 +310,6 @@ TEST(AllocationSampleTest, SampleAccuracy) { } } -TEST(FragmentationzTest, Accuracy) { - // Increase sampling rate to decrease flakiness. - ScopedProfileSamplingInterval ps(512 * 1024); - // Disable GWP-ASan, since it allocates different sizes than normal samples. - ScopedGuardedSamplingInterval gs(-1); - - // a fairly odd allocation size - will be rounded to 128. This lets - // us find our record in the table. - static const size_t kItemSize = 115; - // allocate about 3.5 GiB: - static const size_t kNumItems = 32 * 1024 * 1024; - - std::vector> keep; - std::vector> drop; - // hint expected sizes: - drop.reserve(kNumItems * 8 / 10); - keep.reserve(kNumItems * 2 / 10); - - // We allocate many items, then free 80% of them "randomly". (To - // decrease noise and speed up, we just keep every 5th one exactly.) - for (int i = 0; i < kNumItems; ++i) { - // Ideally we should use a malloc() here, for consistency; but unique_ptr - // doesn't come with a have a "free()" deleter; use ::operator new instead. - (i % 5 == 0 ? keep : drop) - .push_back(std::unique_ptr( - static_cast(::operator new[](kItemSize)))); - } - drop.resize(0); - - // there are at least 64 items per span here. (8/10)^64 = 6.2e-7 ~= 0 - // probability we actually managed to free a page; every page is fragmented. - // We still have 20% or so of it allocated, so we should see 80% of it - // charged to these allocations as fragmentations. - auto profile = MallocExtension::SnapshotCurrent(ProfileType::kFragmentation); - - // Pull out the fragmentationz entry corresponding to this - size_t requested_size = 0; - size_t allocated_size = 0; - size_t sum = 0; - size_t count = 0; - profile.Iterate([&](const Profile::Sample& e) { - if (e.requested_size != kItemSize) return; - - if (requested_size == 0) { - allocated_size = e.allocated_size; - requested_size = e.requested_size; - } else { - // we will usually have single entry in - // profile, but in builds without optimization - // our fast-path code causes same call-site to - // have two different stack traces. Thus we - // expect and deal with second entry for same - // allocation. - EXPECT_EQ(requested_size, e.requested_size); - EXPECT_EQ(allocated_size, e.allocated_size); - } - sum += e.sum; - count += e.count; - }); - - double frag_bytes = sum; - double real_frag_bytes = - static_cast(allocated_size * kNumItems) * 0.8; - // We should be pretty close with this much data. - EXPECT_NEAR(real_frag_bytes, frag_bytes, real_frag_bytes * 0.15) - << " sum = " << sum << " allocated = " << allocated_size - << " requested = " << requested_size << " count = " << count; -} extern "C" { void* __alloc_token_0__ZnwmRKSt9nothrow_t(size_t size, std::nothrow_t);