Skip to content

Clearly distinguish kPageSize (TCMalloc's internal page) from the HW page size. #347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Jul 13, 2026
Merged

Clearly distinguish kPageSize (TCMalloc's internal page) from the HW page size. #347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions tcmalloc/internal/mincore.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,14 @@ size_t MInCore::residence_impl(void* addr, size_t size,
return 0;
}
unsigned char res[kArrayLength];
const size_t kPageSize = GetPageSize();
const size_t kHardwarePageSize = GetPageSize();

uintptr_t uaddr = reinterpret_cast<uintptr_t>(addr);
// Round address down to get the start of the page containing the data.
uintptr_t basePage = uaddr & ~(kPageSize - 1);
uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1);
// Round end address up to get the end of the page containing the data.
uintptr_t endPage = (uaddr + size + kPageSize - 1) & ~(kPageSize - 1);
uintptr_t endPage =
(uaddr + size + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1);

uintptr_t remainingPages = endPage - basePage;

Expand All @@ -64,7 +65,7 @@ size_t MInCore::residence_impl(void* addr, size_t size,
// pages will contribute fewer than that. Easiest way to do this is to
// handle the special case where the entire object fits into a page,
// then handle the case where the object spans more than one page.
if (remainingPages == kPageSize) {
if (remainingPages == kHardwarePageSize) {
// Find out whether the first page is resident.
if (mincore->mincore(reinterpret_cast<void*>(basePage), remainingPages,
res) != 0) {
Expand All @@ -80,7 +81,8 @@ size_t MInCore::residence_impl(void* addr, size_t size,
// We're calling this outside the loop so that we can get info for the
// first page, deal with subsequent pages in the loop, and then handle
// the last page after the loop.
size_t scanLength = std::min(remainingPages, kPageSize * kArrayLength);
size_t scanLength =
std::min(remainingPages, kHardwarePageSize * kArrayLength);
if (mincore->mincore(reinterpret_cast<void*>(basePage), scanLength, res) !=
0) {
return 0;
Expand All @@ -89,27 +91,27 @@ size_t MInCore::residence_impl(void* addr, size_t size,
size_t totalResident = 0;

// Handle the first page.
size_t firstPageSize = kPageSize - (uaddr - basePage);
size_t firstPageSize = kHardwarePageSize - (uaddr - basePage);
if ((res[0] & 1) == 1) {
totalResident += firstPageSize;
}
basePage += kPageSize;
remainingPages -= kPageSize;
basePage += kHardwarePageSize;
remainingPages -= kHardwarePageSize;

int resIndex = 1;

// Handle all pages but the last page.
while (remainingPages > kPageSize) {
while (remainingPages > kHardwarePageSize) {
if ((res[resIndex] & 1) == 1) {
totalResident += kPageSize;
totalResident += kHardwarePageSize;
}
resIndex++;
basePage += kPageSize;
remainingPages -= kPageSize;
basePage += kHardwarePageSize;
remainingPages -= kHardwarePageSize;
// Refresh the array if necessary.
if (resIndex == kArrayLength) {
resIndex = 0;
scanLength = std::min(remainingPages, kPageSize * kArrayLength);
scanLength = std::min(remainingPages, kHardwarePageSize * kArrayLength);
if (mincore->mincore(reinterpret_cast<void*>(basePage), scanLength,
res) != 0) {
return 0;
Expand All @@ -118,7 +120,7 @@ size_t MInCore::residence_impl(void* addr, size_t size,
}

// Check final page
size_t lastPageSize = kPageSize - (endPage - uaddr - size);
size_t lastPageSize = kHardwarePageSize - (endPage - uaddr - size);
if ((res[resIndex] & 1) == 1) {
totalResident += lastPageSize;
}
Expand Down
6 changes: 3 additions & 3 deletions tcmalloc/internal/mincore_benchmark.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,16 @@ void BM_mincore(benchmark::State& state) {
TC_CHECK_LE(size, kMaxArraySize);
auto resident = std::make_unique<unsigned char[]>(kMaxArraySize);

const size_t kPageSize = tcmalloc_internal::GetPageSize();
const size_t kHardwarePageSize = tcmalloc_internal::GetPageSize();
// We want to scan the same amount of memory in all cases
const size_t regionSize = 1 * 1024 * 1024 * 1024;
for (auto s : state) {
uintptr_t memory = 0;
while (memory < regionSize) {
// Call mincore for the next section
int length = std::min(size * kPageSize, (regionSize - memory));
int length = std::min(size * kHardwarePageSize, (regionSize - memory));
::mincore(reinterpret_cast<void*>(memory), length, resident.get());
memory += length * kPageSize;
memory += length * kHardwarePageSize;
}
}
}
Expand Down
80 changes: 44 additions & 36 deletions tcmalloc/internal/mincore_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,17 @@ class MInCoreMock : public MInCoreInterface {

// Implementation of minCore that reports presence based on provided array.
int mincore(void* addr, size_t length, unsigned char* result) override {
const size_t kPageSize = GetPageSize();
const size_t kHardwarePageSize = GetPageSize();
uintptr_t uAddress = reinterpret_cast<uintptr_t>(addr);
// Check that we only pass page aligned addresses into mincore().
EXPECT_THAT(uAddress & (kPageSize - 1), Eq(0));
EXPECT_THAT(uAddress & (kHardwarePageSize - 1), Eq(0));

uintptr_t uEndAddress = uAddress + length;
int index = 0;
// Check for presence of the target pages in the map.
while (uAddress < uEndAddress) {
result[index] = (mapped_.find(uAddress) != mapped_.end() ? 1 : 0);
uAddress += kPageSize;
uAddress += kHardwarePageSize;
index++;
}
return 0;
Expand Down Expand Up @@ -89,14 +89,14 @@ using ::testing::Eq;

TEST(MInCoreTest, TestResidence) {
MInCoreTest mct;
const size_t kPageSize = GetPageSize();
const size_t kHardwarePageSize = GetPageSize();

// Set up a pattern with a few resident pages.
// page 0 not mapped
mct.addPage(kPageSize);
mct.addPage(kHardwarePageSize);
// page 2 not mapped
mct.addPage(3 * kPageSize);
mct.addPage(4 * kPageSize);
mct.addPage(3 * kHardwarePageSize);
mct.addPage(4 * kHardwarePageSize);

// An object of size zero should have a residence of zero.
EXPECT_THAT(mct.residence(320, 0), Eq(0));
Expand All @@ -107,44 +107,50 @@ TEST(MInCoreTest, TestResidence) {

// Check that an object entirely on the second page is
// reported as entirely mapped.
EXPECT_THAT(mct.residence(kPageSize + 320, 55), Eq(55));
EXPECT_THAT(mct.residence(kHardwarePageSize + 320, 55), Eq(55));

// An object of size zero should have a residence of zero.
EXPECT_THAT(mct.residence(kPageSize + 320, 0), Eq(0));
EXPECT_THAT(mct.residence(kHardwarePageSize + 320, 0), Eq(0));

// Check that an object over a mapped and unmapped page is half mapped.
EXPECT_THAT(mct.residence(kPageSize / 2, kPageSize), Eq(kPageSize / 2));
EXPECT_THAT(mct.residence(kHardwarePageSize / 2, kHardwarePageSize),
Eq(kHardwarePageSize / 2));

// Check that an object which spans two pages is reported as being mapped
// only on the page that's resident.
EXPECT_THAT(mct.residence(kPageSize / 2 * 3, kPageSize), Eq(kPageSize / 2));
EXPECT_THAT(mct.residence(kHardwarePageSize / 2 * 3, kHardwarePageSize),
Eq(kHardwarePageSize / 2));

// Check that an object that is on two mapped pages is reported as entirely
// resident.
EXPECT_THAT(mct.residence(kPageSize / 2 * 7, kPageSize), Eq(kPageSize));
EXPECT_THAT(mct.residence(kHardwarePageSize / 2 * 7, kHardwarePageSize),
Eq(kHardwarePageSize));

// Check that an object that is on one mapped page is reported as only
// resident on the mapped page.
EXPECT_THAT(mct.residence(kPageSize * 2, kPageSize + 1), Eq(1));
EXPECT_THAT(mct.residence(kHardwarePageSize * 2, kHardwarePageSize + 1),
Eq(1));

// Check that an object that is on one mapped page is reported as only
// resident on the mapped page.
EXPECT_THAT(mct.residence(kPageSize + 1, kPageSize + 1), Eq(kPageSize - 1));
EXPECT_THAT(mct.residence(kHardwarePageSize + 1, kHardwarePageSize + 1),
Eq(kHardwarePageSize - 1));

// Check that an object which spans beyond the mapped pages is reported
// as unmapped
EXPECT_THAT(mct.residence(kPageSize * 6, kPageSize), Eq(0));
EXPECT_THAT(mct.residence(kHardwarePageSize * 6, kHardwarePageSize), Eq(0));

// Check an object that spans three pages, two of them mapped.
EXPECT_THAT(mct.residence(kPageSize / 2 * 7 + 1, kPageSize * 2),
Eq(kPageSize * 3 / 2 - 1));
EXPECT_THAT(
mct.residence(kHardwarePageSize / 2 * 7 + 1, kHardwarePageSize * 2),
Eq(kHardwarePageSize * 3 / 2 - 1));
}

// Test whether we are correctly handling multiple calls to mincore.
TEST(MInCoreTest, TestLargeResidence) {
MInCoreTest mct;
uintptr_t uAddress = 0;
const size_t kPageSize = GetPageSize();
const size_t kHardwarePageSize = GetPageSize();
// Set up a pattern covering 6 * page size * MInCore::kArrayLength to
// allow us to test for situations where the region we're checking
// requires multiple calls to mincore().
Expand All @@ -153,47 +159,49 @@ TEST(MInCoreTest, TestLargeResidence) {
// with the pattern.
for (int i = 0; i < 2 * mct.chunkSize(); i++) {
mct.addPage(uAddress);
uAddress += 3 * kPageSize;
uAddress += 3 * kHardwarePageSize;
}

uintptr_t baseAddress = 0;
for (int size = kPageSize; size < 32 * 1024 * 1024; size += 2 * kPageSize) {
uintptr_t unit = kPageSize * 3;
for (int size = kHardwarePageSize; size < 32 * 1024 * 1024;
size += 2 * kHardwarePageSize) {
uintptr_t unit = kHardwarePageSize * 3;
EXPECT_THAT(mct.residence(baseAddress, size),
Eq(kPageSize * ((size + unit - 1) / unit)));
Eq(kHardwarePageSize * ((size + unit - 1) / unit)));
}
}

TEST(MInCoreTest, UnmappedMemory) {
const size_t kPageSize = GetPageSize();
const size_t kHardwarePageSize = GetPageSize();
const int kNumPages = 16;

// Overallocate kNumPages of memory, so we can munmap the page before and
// after it.
void* p = mmap(nullptr, (kNumPages + 2) * kPageSize, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
void* p = mmap(nullptr, (kNumPages + 2) * kHardwarePageSize,
PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
ASSERT_NE(p, MAP_FAILED) << errno;
ASSERT_EQ(munmap(p, kPageSize), 0);
void* q = reinterpret_cast<char*>(p) + kPageSize;
void* last = reinterpret_cast<char*>(p) + (kNumPages + 1) * kPageSize;
ASSERT_EQ(munmap(last, kPageSize), 0);
ASSERT_EQ(munmap(p, kHardwarePageSize), 0);
void* q = reinterpret_cast<char*>(p) + kHardwarePageSize;
void* last = reinterpret_cast<char*>(p) + (kNumPages + 1) * kHardwarePageSize;
ASSERT_EQ(munmap(last, kHardwarePageSize), 0);

memset(q, 0, kNumPages * kPageSize);
memset(q, 0, kNumPages * kHardwarePageSize);
::benchmark::DoNotOptimize(q);

EXPECT_EQ(0, MInCore::residence(nullptr, kPageSize));
EXPECT_EQ(0, MInCore::residence(p, kPageSize));
EXPECT_EQ(0, MInCore::residence(nullptr, kHardwarePageSize));
EXPECT_EQ(0, MInCore::residence(p, kHardwarePageSize));
for (int i = 0; i <= kNumPages; i++) {
EXPECT_EQ(i * kPageSize, MInCore::residence(q, i * kPageSize));
EXPECT_EQ(i * kHardwarePageSize,
MInCore::residence(q, i * kHardwarePageSize));
}

// Note we can only query regions that are entirely mapped, but we should also
// test the edge case of incomplete pages.
EXPECT_EQ((kNumPages - 1) * kPageSize,
EXPECT_EQ((kNumPages - 1) * kHardwarePageSize,
MInCore::residence(reinterpret_cast<char*>(q) + 7,
(kNumPages - 1) * kPageSize));
(kNumPages - 1) * kHardwarePageSize));

ASSERT_EQ(munmap(q, kNumPages * kPageSize), 0);
ASSERT_EQ(munmap(q, kNumPages * kHardwarePageSize), 0);
}

} // namespace
Expand Down
32 changes: 18 additions & 14 deletions tcmalloc/internal/pageflags.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ PageFlags::~PageFlags() {
}

size_t PageFlags::GetOffset(const uintptr_t vaddr) {
TC_ASSERT_EQ(vaddr % kPageSize, 0);
return vaddr / kPageSize * kPagemapEntrySize;
TC_ASSERT_EQ(vaddr % kHardwarePageSize, 0);
return vaddr / kHardwarePageSize * kPagemapEntrySize;
}

absl::StatusCode PageFlags::Seek(const uintptr_t vaddr) {
Expand Down Expand Up @@ -191,7 +191,7 @@ absl::StatusCode PageFlags::ReadMany(int64_t num_pages, PageStats& output) {
}
flags = last_head_read_;
}
MaybeAddToStats(output, flags, kPageSize);
MaybeAddToStats(output, flags, kHardwarePageSize);
}
num_pages -= batch_size;
}
Expand All @@ -207,7 +207,7 @@ std::optional<bool> PageFlags::IsHugepageBacked(const void* const addr) {
uintptr_t uaddr = reinterpret_cast<uintptr_t>(addr);
// Round address down to get the start of the first page that has any bytes
// corresponding to the span [addr, addr+size).
uintptr_t basePage = uaddr & ~(kPageSize - 1);
uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1);
// Seek into fd.
if (auto res = Seek(basePage); res != absl::StatusCode::kOk) return false;
// Read entry
Expand Down Expand Up @@ -237,13 +237,14 @@ std::optional<PageStats> PageFlags::Get(const void* const addr,
uintptr_t uaddr = reinterpret_cast<uintptr_t>(addr);
// Round address down to get the start of the first page that has any bytes
// corresponding to the span [addr, addr+size).
uintptr_t basePage = uaddr & ~(kPageSize - 1);
uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1);
// Round end address up to get the start of the first page that does not
// have any bytes corresponding to the span [addr, addr+size).
// The span is a subset of [basePage, endPage).
uintptr_t endPage = (uaddr + size + kPageSize - 1) & ~(kPageSize - 1);
uintptr_t endPage =
(uaddr + size + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1);

int64_t remainingPages = (endPage - basePage) / kPageSize;
int64_t remainingPages = (endPage - basePage) / kHardwarePageSize;

if (remainingPages == 1) {
if (auto res = MaybeReadOne(basePage, result_flags, is_huge);
Expand All @@ -268,7 +269,7 @@ std::optional<PageStats> PageFlags::Get(const void* const addr,
res != absl::StatusCode::kOk) {
return std::nullopt;
}
size_t firstPageSize = kPageSize - (uaddr - basePage);
size_t firstPageSize = kHardwarePageSize - (uaddr - basePage);
if (is_huge) {
// The object starts in the middle of a native page, but the entire page
// might be stale. So the situation looks like, simplifying to four native
Expand All @@ -278,11 +279,12 @@ std::optional<PageStats> PageFlags::Get(const void* const addr,
// [....|..XX|XXXX|XXXX]
// ^^^^^^^ some other stale object(s)
// ^^ firstPageSize
// ^^^^^^^^^^^^^^ `pages_represented` pages, each of kPageSize
// ^^^^^^^^^^^^^^ `pages_represented` pages, each of
// kHardwarePageSize
// The remainingPages <= 0 case covers the situation where the span ends
// before the hugepage.
const uint64_t base_page_offset = basePage & (kHugePageSize - 1);
const uint64_t base_page_index = base_page_offset / kPageSize;
const uint64_t base_page_index = base_page_offset / kHardwarePageSize;
const int64_t pages_represented = kPagesInHugePage - base_page_index;

remainingPages -= pages_represented;
Expand All @@ -299,8 +301,9 @@ std::optional<PageStats> PageFlags::Get(const void* const addr,

// pages_represented - 1 is the number of full pages represented (see
// diagram)
MaybeAddToStats(ret, result_flags,
firstPageSize + (pages_represented - 1) * kPageSize);
MaybeAddToStats(
ret, result_flags,
firstPageSize + (pages_represented - 1) * kHardwarePageSize);

// We've read one uint64_t about a single page, but it represents 512 small
// pages. So the next page that is of interest is one hugepage away -- seek
Expand All @@ -323,8 +326,9 @@ std::optional<PageStats> PageFlags::Get(const void* const addr,

// Check final page. It doesn't really matter if is_huge; we just want the
// statistics about the page that has the last byte of the object.
size_t lastPageSize = kPageSize - (endPage - uaddr - size);
if (auto res = MaybeReadOne(endPage - kPageSize, result_flags, is_huge);
size_t lastPageSize = kHardwarePageSize - (endPage - uaddr - size);
if (auto res =
MaybeReadOne(endPage - kHardwarePageSize, result_flags, is_huge);
res != absl::StatusCode::kOk) {
return std::nullopt;
}
Expand Down
4 changes: 2 additions & 2 deletions tcmalloc/internal/pageflags.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,13 +136,13 @@ class PageFlags final : public PageFlagsBase {
static constexpr int kPagemapEntrySize = 8;
static constexpr int kEntriesInBuf = kBufferLength / kPagemapEntrySize;

const size_t kPageSize = GetPageSize();
const size_t kHardwarePageSize = GetPageSize();
// You can technically not hard-code this but it would involve many more
// queries to figure out the size of every page. It's a lot easier to just
// assume any compound pages are 2 MB.
static constexpr int kHugePageSize = (2 << 20);
static constexpr uintptr_t kHugePageMask = ~(kHugePageSize - 1);
const size_t kPagesInHugePage = kHugePageSize / kPageSize;
const size_t kPagesInHugePage = kHugePageSize / kHardwarePageSize;

uint64_t buf_[kEntriesInBuf];
// Information about the previous head page. For any future-encountered tail
Expand Down
Loading
Loading