Skip to content

Remove fragmentation profile #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Remove fragmentation profile #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 0 additions & 49 deletions tcmalloc/allocation_sampling.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,41 +48,6 @@
GOOGLE_MALLOC_SECTION_BEGIN
namespace tcmalloc::tcmalloc_internal {

std::unique_ptr<const ProfileBase> DumpFragmentationProfile(Static& state) {
auto profile = std::make_unique<StackTraceTable>(ProfileType::kFragmentation);
state.sampled_allocation_recorder().Iterate(
[&state, &profile](const SampledAllocation& sampled_allocation) {
// Compute fragmentation to charge to this sample:
const StackTrace& t = sampled_allocation.sampled_stack;
if (t.proxy == nullptr) {
// There is just one object per-span, and neighboring spans
// can be released back to the system, so we charge no
// fragmentation to this sampled object.
return;
}

// Fetch the span on which the proxy lives so we can examine its
// co-residents.
const PageId p = PageIdContaining(t.proxy);
Span* span = state.pagemap().GetDescriptor(p);
if (span == nullptr || span == &state.invalid_span()) {
// Avoid crashes in production mode code, but report in tests.
TC_ASSERT_NE(span, nullptr);
TC_ASSERT_NE(span, &state.invalid_span());
return;
}

const double frag = span->Fragmentation(t.allocated_size);
if (frag > 0) {
// Associate the memory warmth with the actual object, not the proxy.
// The residency information (t.span_start_address) is likely not very
// useful, but we might as well pass it along.
profile->AddTrace(frag, t);
}
});
return profile;
}

std::unique_ptr<const ProfileBase> DumpHeapProfile(Static& state) {
auto profile = std::make_unique<StackTraceTable>(ProfileType::kHeap);
profile->SetStartTime(absl::Now());
Expand All @@ -93,19 +58,5 @@ std::unique_ptr<const ProfileBase> DumpHeapProfile(Static& state) {
return profile;
}

ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr,
size_t size_class) {
if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) {
state.cpu_cache().Deallocate(ptr, size_class);
} else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent();
ABSL_PREDICT_TRUE(cache)) {
cache->Deallocate(ptr, size_class);
} else {
// This thread doesn't have thread-cache yet or already. Delete directly
// into transfer cache.
state.transfer_cache().InsertRange(size_class, absl::Span<void*>(&ptr, 1));
}
}

} // namespace tcmalloc::tcmalloc_internal
GOOGLE_MALLOC_SECTION_END
89 changes: 12 additions & 77 deletions tcmalloc/allocation_sampling.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,6 @@ namespace tcmalloc::tcmalloc_internal {

class Static;

// This function computes a profile that maps a live stack trace to
// the number of bytes of central-cache memory pinned by an allocation
// at that stack trace.
// In the case when span is hosting >= 1 number of small objects (t.proxy !=
// nullptr), we call span::Fragmentation() and read `span->allocated_`. It is
// safe to do so since we hold the per-sample lock while iterating over sampled
// allocations. It prevents the sampled allocation that has the proxy object to
// complete deallocation, thus `proxy` can not be returned to the span yet. It
// thus prevents the central free list to return the span to the page heap.
std::unique_ptr<const ProfileBase> DumpFragmentationProfile(Static& state);

std::unique_ptr<const ProfileBase> DumpHeapProfile(Static& state);
#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ
// For RSEQ enabled builds, we declare the sampler in percpu.h so that we can
Expand Down Expand Up @@ -78,16 +67,9 @@ inline Sampler& GetThreadSampler() {
#endif
}

void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class);

// Performs sampling for already occurred allocation of object.
//
// For very small object sizes, object is used as 'proxy' and full
// page with sampled marked is allocated instead.
//
// For medium-sized objects that have single instance per span,
// they're simply freed and fresh page span is allocated to represent
// sampling.
// For small object sizes, we allocate a new object in a sampled span.
//
// For large objects (i.e. allocated with do_malloc_pages) they are
// also fully reused and their span is marked as sampled.
Expand All @@ -106,12 +88,10 @@ void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class);
template <typename Policy>
ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
Static& state, Policy policy, size_t requested_size, size_t weight,
size_t size_class, void* absl_nullable obj, Span* absl_nullable span) {
TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) ||
(size_class == 0 && obj == nullptr && span != nullptr));
size_t size_class, Span* absl_nullable span) {
TC_CHECK_EQ(size_class != 0, span == nullptr);

StackTrace stack_trace;
stack_trace.proxy = nullptr;
stack_trace.requested_size = requested_size;
// Grab the stack trace outside the heap lock.
stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0);
Expand Down Expand Up @@ -142,8 +122,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
: MemoryTag::kSampled;
size_t capacity = 0;
if (size_class != 0) {
TC_ASSERT_EQ(size_class,
state.pagemap().sizeclass(PageIdContainingTagged(obj)));
state.per_size_class_counts()[size_class].Add(allocation_estimate);

stack_trace.allocated_size = state.sizemap().class_to_size(size_class);
Expand Down Expand Up @@ -172,24 +150,10 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(
}
capacity = requested_size;
} else if ((span = state.page_allocator().New(
num_pages, {1, AccessDensityPrediction::kSparse}, tag)) ==
nullptr) {
num_pages, {1, AccessDensityPrediction::kSparse}, tag))) {
capacity = stack_trace.allocated_size;
return {obj, capacity};
} else {
capacity = stack_trace.allocated_size;
}

size_t span_size =
Length(state.sizemap().class_to_pages(size_class)).in_bytes();
size_t objects_per_span = stack_trace.allocated_size != 0
? span_size / stack_trace.allocated_size
: -1ul;

if (objects_per_span != 1) {
TC_ASSERT_GT(objects_per_span, 1);
stack_trace.proxy = obj;
obj = nullptr;
return {};
}
} else {
// Set stack_trace.allocated_size to the exact size for a page allocation.
Expand Down Expand Up @@ -255,12 +219,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation(

state.peak_heap_tracker().MaybeSaveSample();

if (obj != nullptr) {
// We are not maintaining precise statistics on malloc hit/miss rates at our
// cache tiers. We can deallocate into our ordinary cache.
TC_ASSERT_NE(size_class, 0);
FreeProxyObject(state, obj, size_class);
}
TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0);
return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc
: span->start_address(),
Expand All @@ -275,16 +233,15 @@ template <typename Policy>
static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy,
size_t requested_size, size_t weight,
Span* span) {
return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr,
span);
return SampleifyAllocation(state, policy, requested_size, weight, 0, span);
}

template <typename Policy>
static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy,
size_t requested_size, size_t weight,
size_t size_class, sized_ptr_t res) {
size_t size_class) {
return SampleifyAllocation(state, policy, requested_size, weight, size_class,
res.p, nullptr);
nullptr);
}

// Rewrite type so that the allocation type falls into one of the categories we
Expand Down Expand Up @@ -334,7 +291,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy,

TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0);

void* const proxy = sampled_allocation->sampled_stack.proxy;
const size_t weight = sampled_allocation->sampled_stack.weight;
const size_t requested_size =
sampled_allocation->sampled_stack.requested_size;
Expand Down Expand Up @@ -377,18 +333,14 @@ void MaybeUnsampleAllocation(Static& state, Policy policy,
sampled_allocation->sampled_stack.depth));
}

const size_t allocated_alignment = static_cast<size_t>(
sampled_allocation->sampled_stack.requested_alignment.value_or(
std::align_val_t{1}));
const std::optional<std::align_val_t> deallocated_alignment =
policy.has_explicit_alignment()
? std::make_optional<std::align_val_t>(policy.align())
: std::nullopt;

if ((size.has_value() || policy.allocation_type() == AllocationType::New)) {
const bool type_mismatch =
policy.allocation_type() !=
sampled_allocation->sampled_stack.allocation_type;
const std::optional<std::align_val_t> deallocated_alignment =
policy.has_explicit_alignment()
? std::make_optional<std::align_val_t>(policy.align())
: std::nullopt;
const bool alignment_mismatch =
deallocated_alignment !=
sampled_allocation->sampled_stack.requested_alignment;
Expand Down Expand Up @@ -439,23 +391,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy,
MallocHook::InvokeSampledDeleteHook(sampled_alloc);

state.deallocation_samples.ReportFree(sampled_alloc_handle);

if (proxy) {
const auto proxy_policy = policy.InSamePartitionAs(proxy);
size_t size_class;
if (AccessFromPointer(proxy) == AllocationAccess::kCold) {
size_class = state.sizemap().SizeClass(
proxy_policy.AccessAsCold().AlignAs(allocated_alignment),
allocated_size);
} else {
size_class = state.sizemap().SizeClass(
proxy_policy.AccessAsHot().AlignAs(allocated_alignment),
allocated_size);
}
TC_ASSERT_EQ(size_class,
state.pagemap().sizeclass(PageIdContainingTagged(proxy)));
FreeProxyObject(state, proxy, size_class);
}
}

} // namespace tcmalloc::tcmalloc_internal
Expand Down
41 changes: 0 additions & 41 deletions tcmalloc/central_freelist_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1076,47 +1076,6 @@ TEST_P(CentralFreeListTest, PassSpanDensityToPageheap) {
test_function(e.objects_per_span(), AccessDensityPrediction::kDense);
}

TEST_P(CentralFreeListTest, SpanFragmentation) {
#if ABSL_HAVE_HWADDRESS_SANITIZER
GTEST_SKIP()
<< "Skipping under HWASan, which uses the top bits of the pointer.";
#endif

// This test is primarily exercising Span itself to model how tcmalloc.cc uses
// it, but this gives us a self-contained (and sanitizable) implementation of
// the CentralFreeList.
TypeParam e(GetParam().size, GetParam().bytes, GetParam().num_to_move);
// Allocate one object from the CFL to allocate a span.
void* initial;
int got = e.central_freelist().RemoveRange(absl::MakeSpan(&initial, 1));
ASSERT_EQ(got, 1);

Span* const span = e.central_freelist().forwarder().MapObjectToSpan(initial);
const size_t object_size =
e.central_freelist().forwarder().class_to_size(TypeParam::kSizeClass);

ThreadManager fragmentation;
fragmentation.Start(1, [&](int) {
if (e.objects_per_span() != 1) {
benchmark::DoNotOptimize(span->Fragmentation(object_size));
}
});

ThreadManager cfl;
cfl.Start(1, [&](int) {
void* next;
int got = e.central_freelist().RemoveRange(absl::MakeSpan(&next, 1));
e.central_freelist().InsertRange(absl::MakeSpan(&next, got));
});

absl::SleepFor(absl::Milliseconds(50));

fragmentation.Stop();
cfl.Stop();

e.central_freelist().InsertRange(absl::MakeSpan(&initial, 1));
}

TEST_P(CentralFreeListTest, SpanLifetimeWithLongLivedSpans) {
#if ABSL_HAVE_HWADDRESS_SANITIZER
GTEST_SKIP()
Expand Down
11 changes: 0 additions & 11 deletions tcmalloc/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,17 +248,6 @@ enum class AllocationAccess {
kCold,
};

inline AllocationAccess AccessFromPointer(void* ptr) {
if (!kHasExpandedClasses) {
TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold);
return AllocationAccess::kHot;
}

return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold)
? AllocationAccess::kCold
: AllocationAccess::kHot;
}

inline MemoryTag MultiNormalTag(size_t partition) {
switch (partition) {
case 0:
Expand Down
4 changes: 2 additions & 2 deletions tcmalloc/experimental_pow2_size_class.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = {
{ 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00%
{ 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00%
{ 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00%
{262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00%
};
#elif TCMALLOC_PAGE_SHIFT == 12
Expand Down Expand Up @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = {
{ 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00%
{ 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00%
{ 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00%
{131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00%
{262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00%
};
#elif TCMALLOC_PAGE_SHIFT == 12
Expand Down
9 changes: 0 additions & 9 deletions tcmalloc/internal/logging.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,6 @@ struct StackTrace {
// memory block.
AllocHandle sampled_alloc_handle;

// For small sampled objects, we allocate a full span to hold the
// sampled object. However to avoid disturbing fragmentation
// profiles, in such cases we also allocate a small proxy object
// using the normal mechanism.
//
// proxy field is defined only for heap sample stack traces.
// For heap samples, proxy==NULL iff size > kMaxSize.
void* proxy;

uintptr_t requested_size;
std::optional<std::align_val_t> requested_alignment;
uintptr_t allocated_size; // size after sizeclass/page rounding
Expand Down
Loading