From 3cfc878d08c2ea89c7c7e4c81a806e7ed5c9ae35 Mon Sep 17 00:00:00 2001 From: Chris Kennelly Date: Mon, 13 Jul 2026 14:00:00 -0700 Subject: [PATCH] Clearly distinguish kPageSize (TCMalloc's internal page) from the HW page size. We use kPageSize for both, especially in our internal libraries, making this ambiguous. PiperOrigin-RevId: 947229295 --- tcmalloc/internal/mincore.cc | 30 +++--- tcmalloc/internal/mincore_benchmark.cc | 6 +- tcmalloc/internal/mincore_test.cc | 80 +++++++------- tcmalloc/internal/pageflags.cc | 32 +++--- tcmalloc/internal/pageflags.h | 4 +- tcmalloc/internal/pageflags_test.cc | 139 +++++++++++++------------ tcmalloc/internal/residency.cc | 15 +-- tcmalloc/internal/residency.h | 4 +- tcmalloc/internal/residency_test.cc | 65 ++++++------ 9 files changed, 199 insertions(+), 176 deletions(-) diff --git a/tcmalloc/internal/mincore.cc b/tcmalloc/internal/mincore.cc index b2dbcf9f2..2e66a8842 100644 --- a/tcmalloc/internal/mincore.cc +++ b/tcmalloc/internal/mincore.cc @@ -49,13 +49,14 @@ size_t MInCore::residence_impl(void* addr, size_t size, return 0; } unsigned char res[kArrayLength]; - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); uintptr_t uaddr = reinterpret_cast(addr); // Round address down to get the start of the page containing the data. - uintptr_t basePage = uaddr & ~(kPageSize - 1); + uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1); // Round end address up to get the end of the page containing the data. - uintptr_t endPage = (uaddr + size + kPageSize - 1) & ~(kPageSize - 1); + uintptr_t endPage = + (uaddr + size + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1); uintptr_t remainingPages = endPage - basePage; @@ -64,7 +65,7 @@ size_t MInCore::residence_impl(void* addr, size_t size, // pages will contribute fewer than that. Easiest way to do this is to // handle the special case where the entire object fits into a page, // then handle the case where the object spans more than one page. - if (remainingPages == kPageSize) { + if (remainingPages == kHardwarePageSize) { // Find out whether the first page is resident. if (mincore->mincore(reinterpret_cast(basePage), remainingPages, res) != 0) { @@ -80,7 +81,8 @@ size_t MInCore::residence_impl(void* addr, size_t size, // We're calling this outside the loop so that we can get info for the // first page, deal with subsequent pages in the loop, and then handle // the last page after the loop. - size_t scanLength = std::min(remainingPages, kPageSize * kArrayLength); + size_t scanLength = + std::min(remainingPages, kHardwarePageSize * kArrayLength); if (mincore->mincore(reinterpret_cast(basePage), scanLength, res) != 0) { return 0; @@ -89,27 +91,27 @@ size_t MInCore::residence_impl(void* addr, size_t size, size_t totalResident = 0; // Handle the first page. - size_t firstPageSize = kPageSize - (uaddr - basePage); + size_t firstPageSize = kHardwarePageSize - (uaddr - basePage); if ((res[0] & 1) == 1) { totalResident += firstPageSize; } - basePage += kPageSize; - remainingPages -= kPageSize; + basePage += kHardwarePageSize; + remainingPages -= kHardwarePageSize; int resIndex = 1; // Handle all pages but the last page. - while (remainingPages > kPageSize) { + while (remainingPages > kHardwarePageSize) { if ((res[resIndex] & 1) == 1) { - totalResident += kPageSize; + totalResident += kHardwarePageSize; } resIndex++; - basePage += kPageSize; - remainingPages -= kPageSize; + basePage += kHardwarePageSize; + remainingPages -= kHardwarePageSize; // Refresh the array if necessary. if (resIndex == kArrayLength) { resIndex = 0; - scanLength = std::min(remainingPages, kPageSize * kArrayLength); + scanLength = std::min(remainingPages, kHardwarePageSize * kArrayLength); if (mincore->mincore(reinterpret_cast(basePage), scanLength, res) != 0) { return 0; @@ -118,7 +120,7 @@ size_t MInCore::residence_impl(void* addr, size_t size, } // Check final page - size_t lastPageSize = kPageSize - (endPage - uaddr - size); + size_t lastPageSize = kHardwarePageSize - (endPage - uaddr - size); if ((res[resIndex] & 1) == 1) { totalResident += lastPageSize; } diff --git a/tcmalloc/internal/mincore_benchmark.cc b/tcmalloc/internal/mincore_benchmark.cc index 13b8c8029..30b3829b4 100644 --- a/tcmalloc/internal/mincore_benchmark.cc +++ b/tcmalloc/internal/mincore_benchmark.cc @@ -41,16 +41,16 @@ void BM_mincore(benchmark::State& state) { TC_CHECK_LE(size, kMaxArraySize); auto resident = std::make_unique(kMaxArraySize); - const size_t kPageSize = tcmalloc_internal::GetPageSize(); + const size_t kHardwarePageSize = tcmalloc_internal::GetPageSize(); // We want to scan the same amount of memory in all cases const size_t regionSize = 1 * 1024 * 1024 * 1024; for (auto s : state) { uintptr_t memory = 0; while (memory < regionSize) { // Call mincore for the next section - int length = std::min(size * kPageSize, (regionSize - memory)); + int length = std::min(size * kHardwarePageSize, (regionSize - memory)); ::mincore(reinterpret_cast(memory), length, resident.get()); - memory += length * kPageSize; + memory += length * kHardwarePageSize; } } } diff --git a/tcmalloc/internal/mincore_test.cc b/tcmalloc/internal/mincore_test.cc index 2c0778ae9..adc77f6c1 100644 --- a/tcmalloc/internal/mincore_test.cc +++ b/tcmalloc/internal/mincore_test.cc @@ -41,17 +41,17 @@ class MInCoreMock : public MInCoreInterface { // Implementation of minCore that reports presence based on provided array. int mincore(void* addr, size_t length, unsigned char* result) override { - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); uintptr_t uAddress = reinterpret_cast(addr); // Check that we only pass page aligned addresses into mincore(). - EXPECT_THAT(uAddress & (kPageSize - 1), Eq(0)); + EXPECT_THAT(uAddress & (kHardwarePageSize - 1), Eq(0)); uintptr_t uEndAddress = uAddress + length; int index = 0; // Check for presence of the target pages in the map. while (uAddress < uEndAddress) { result[index] = (mapped_.find(uAddress) != mapped_.end() ? 1 : 0); - uAddress += kPageSize; + uAddress += kHardwarePageSize; index++; } return 0; @@ -89,14 +89,14 @@ using ::testing::Eq; TEST(MInCoreTest, TestResidence) { MInCoreTest mct; - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); // Set up a pattern with a few resident pages. // page 0 not mapped - mct.addPage(kPageSize); + mct.addPage(kHardwarePageSize); // page 2 not mapped - mct.addPage(3 * kPageSize); - mct.addPage(4 * kPageSize); + mct.addPage(3 * kHardwarePageSize); + mct.addPage(4 * kHardwarePageSize); // An object of size zero should have a residence of zero. EXPECT_THAT(mct.residence(320, 0), Eq(0)); @@ -107,44 +107,50 @@ TEST(MInCoreTest, TestResidence) { // Check that an object entirely on the second page is // reported as entirely mapped. - EXPECT_THAT(mct.residence(kPageSize + 320, 55), Eq(55)); + EXPECT_THAT(mct.residence(kHardwarePageSize + 320, 55), Eq(55)); // An object of size zero should have a residence of zero. - EXPECT_THAT(mct.residence(kPageSize + 320, 0), Eq(0)); + EXPECT_THAT(mct.residence(kHardwarePageSize + 320, 0), Eq(0)); // Check that an object over a mapped and unmapped page is half mapped. - EXPECT_THAT(mct.residence(kPageSize / 2, kPageSize), Eq(kPageSize / 2)); + EXPECT_THAT(mct.residence(kHardwarePageSize / 2, kHardwarePageSize), + Eq(kHardwarePageSize / 2)); // Check that an object which spans two pages is reported as being mapped // only on the page that's resident. - EXPECT_THAT(mct.residence(kPageSize / 2 * 3, kPageSize), Eq(kPageSize / 2)); + EXPECT_THAT(mct.residence(kHardwarePageSize / 2 * 3, kHardwarePageSize), + Eq(kHardwarePageSize / 2)); // Check that an object that is on two mapped pages is reported as entirely // resident. - EXPECT_THAT(mct.residence(kPageSize / 2 * 7, kPageSize), Eq(kPageSize)); + EXPECT_THAT(mct.residence(kHardwarePageSize / 2 * 7, kHardwarePageSize), + Eq(kHardwarePageSize)); // Check that an object that is on one mapped page is reported as only // resident on the mapped page. - EXPECT_THAT(mct.residence(kPageSize * 2, kPageSize + 1), Eq(1)); + EXPECT_THAT(mct.residence(kHardwarePageSize * 2, kHardwarePageSize + 1), + Eq(1)); // Check that an object that is on one mapped page is reported as only // resident on the mapped page. - EXPECT_THAT(mct.residence(kPageSize + 1, kPageSize + 1), Eq(kPageSize - 1)); + EXPECT_THAT(mct.residence(kHardwarePageSize + 1, kHardwarePageSize + 1), + Eq(kHardwarePageSize - 1)); // Check that an object which spans beyond the mapped pages is reported // as unmapped - EXPECT_THAT(mct.residence(kPageSize * 6, kPageSize), Eq(0)); + EXPECT_THAT(mct.residence(kHardwarePageSize * 6, kHardwarePageSize), Eq(0)); // Check an object that spans three pages, two of them mapped. - EXPECT_THAT(mct.residence(kPageSize / 2 * 7 + 1, kPageSize * 2), - Eq(kPageSize * 3 / 2 - 1)); + EXPECT_THAT( + mct.residence(kHardwarePageSize / 2 * 7 + 1, kHardwarePageSize * 2), + Eq(kHardwarePageSize * 3 / 2 - 1)); } // Test whether we are correctly handling multiple calls to mincore. TEST(MInCoreTest, TestLargeResidence) { MInCoreTest mct; uintptr_t uAddress = 0; - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); // Set up a pattern covering 6 * page size * MInCore::kArrayLength to // allow us to test for situations where the region we're checking // requires multiple calls to mincore(). @@ -153,47 +159,49 @@ TEST(MInCoreTest, TestLargeResidence) { // with the pattern. for (int i = 0; i < 2 * mct.chunkSize(); i++) { mct.addPage(uAddress); - uAddress += 3 * kPageSize; + uAddress += 3 * kHardwarePageSize; } uintptr_t baseAddress = 0; - for (int size = kPageSize; size < 32 * 1024 * 1024; size += 2 * kPageSize) { - uintptr_t unit = kPageSize * 3; + for (int size = kHardwarePageSize; size < 32 * 1024 * 1024; + size += 2 * kHardwarePageSize) { + uintptr_t unit = kHardwarePageSize * 3; EXPECT_THAT(mct.residence(baseAddress, size), - Eq(kPageSize * ((size + unit - 1) / unit))); + Eq(kHardwarePageSize * ((size + unit - 1) / unit))); } } TEST(MInCoreTest, UnmappedMemory) { - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); const int kNumPages = 16; // Overallocate kNumPages of memory, so we can munmap the page before and // after it. - void* p = mmap(nullptr, (kNumPages + 2) * kPageSize, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + void* p = mmap(nullptr, (kNumPages + 2) * kHardwarePageSize, + PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); ASSERT_NE(p, MAP_FAILED) << errno; - ASSERT_EQ(munmap(p, kPageSize), 0); - void* q = reinterpret_cast(p) + kPageSize; - void* last = reinterpret_cast(p) + (kNumPages + 1) * kPageSize; - ASSERT_EQ(munmap(last, kPageSize), 0); + ASSERT_EQ(munmap(p, kHardwarePageSize), 0); + void* q = reinterpret_cast(p) + kHardwarePageSize; + void* last = reinterpret_cast(p) + (kNumPages + 1) * kHardwarePageSize; + ASSERT_EQ(munmap(last, kHardwarePageSize), 0); - memset(q, 0, kNumPages * kPageSize); + memset(q, 0, kNumPages * kHardwarePageSize); ::benchmark::DoNotOptimize(q); - EXPECT_EQ(0, MInCore::residence(nullptr, kPageSize)); - EXPECT_EQ(0, MInCore::residence(p, kPageSize)); + EXPECT_EQ(0, MInCore::residence(nullptr, kHardwarePageSize)); + EXPECT_EQ(0, MInCore::residence(p, kHardwarePageSize)); for (int i = 0; i <= kNumPages; i++) { - EXPECT_EQ(i * kPageSize, MInCore::residence(q, i * kPageSize)); + EXPECT_EQ(i * kHardwarePageSize, + MInCore::residence(q, i * kHardwarePageSize)); } // Note we can only query regions that are entirely mapped, but we should also // test the edge case of incomplete pages. - EXPECT_EQ((kNumPages - 1) * kPageSize, + EXPECT_EQ((kNumPages - 1) * kHardwarePageSize, MInCore::residence(reinterpret_cast(q) + 7, - (kNumPages - 1) * kPageSize)); + (kNumPages - 1) * kHardwarePageSize)); - ASSERT_EQ(munmap(q, kNumPages * kPageSize), 0); + ASSERT_EQ(munmap(q, kNumPages * kHardwarePageSize), 0); } } // namespace diff --git a/tcmalloc/internal/pageflags.cc b/tcmalloc/internal/pageflags.cc index d95dd1e36..f97e5011d 100644 --- a/tcmalloc/internal/pageflags.cc +++ b/tcmalloc/internal/pageflags.cc @@ -109,8 +109,8 @@ PageFlags::~PageFlags() { } size_t PageFlags::GetOffset(const uintptr_t vaddr) { - TC_ASSERT_EQ(vaddr % kPageSize, 0); - return vaddr / kPageSize * kPagemapEntrySize; + TC_ASSERT_EQ(vaddr % kHardwarePageSize, 0); + return vaddr / kHardwarePageSize * kPagemapEntrySize; } absl::StatusCode PageFlags::Seek(const uintptr_t vaddr) { @@ -191,7 +191,7 @@ absl::StatusCode PageFlags::ReadMany(int64_t num_pages, PageStats& output) { } flags = last_head_read_; } - MaybeAddToStats(output, flags, kPageSize); + MaybeAddToStats(output, flags, kHardwarePageSize); } num_pages -= batch_size; } @@ -207,7 +207,7 @@ std::optional PageFlags::IsHugepageBacked(const void* const addr) { uintptr_t uaddr = reinterpret_cast(addr); // Round address down to get the start of the first page that has any bytes // corresponding to the span [addr, addr+size). - uintptr_t basePage = uaddr & ~(kPageSize - 1); + uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1); // Seek into fd. if (auto res = Seek(basePage); res != absl::StatusCode::kOk) return false; // Read entry @@ -237,13 +237,14 @@ std::optional PageFlags::Get(const void* const addr, uintptr_t uaddr = reinterpret_cast(addr); // Round address down to get the start of the first page that has any bytes // corresponding to the span [addr, addr+size). - uintptr_t basePage = uaddr & ~(kPageSize - 1); + uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1); // Round end address up to get the start of the first page that does not // have any bytes corresponding to the span [addr, addr+size). // The span is a subset of [basePage, endPage). - uintptr_t endPage = (uaddr + size + kPageSize - 1) & ~(kPageSize - 1); + uintptr_t endPage = + (uaddr + size + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1); - int64_t remainingPages = (endPage - basePage) / kPageSize; + int64_t remainingPages = (endPage - basePage) / kHardwarePageSize; if (remainingPages == 1) { if (auto res = MaybeReadOne(basePage, result_flags, is_huge); @@ -268,7 +269,7 @@ std::optional PageFlags::Get(const void* const addr, res != absl::StatusCode::kOk) { return std::nullopt; } - size_t firstPageSize = kPageSize - (uaddr - basePage); + size_t firstPageSize = kHardwarePageSize - (uaddr - basePage); if (is_huge) { // The object starts in the middle of a native page, but the entire page // might be stale. So the situation looks like, simplifying to four native @@ -278,11 +279,12 @@ std::optional PageFlags::Get(const void* const addr, // [....|..XX|XXXX|XXXX] // ^^^^^^^ some other stale object(s) // ^^ firstPageSize - // ^^^^^^^^^^^^^^ `pages_represented` pages, each of kPageSize + // ^^^^^^^^^^^^^^ `pages_represented` pages, each of + // kHardwarePageSize // The remainingPages <= 0 case covers the situation where the span ends // before the hugepage. const uint64_t base_page_offset = basePage & (kHugePageSize - 1); - const uint64_t base_page_index = base_page_offset / kPageSize; + const uint64_t base_page_index = base_page_offset / kHardwarePageSize; const int64_t pages_represented = kPagesInHugePage - base_page_index; remainingPages -= pages_represented; @@ -299,8 +301,9 @@ std::optional PageFlags::Get(const void* const addr, // pages_represented - 1 is the number of full pages represented (see // diagram) - MaybeAddToStats(ret, result_flags, - firstPageSize + (pages_represented - 1) * kPageSize); + MaybeAddToStats( + ret, result_flags, + firstPageSize + (pages_represented - 1) * kHardwarePageSize); // We've read one uint64_t about a single page, but it represents 512 small // pages. So the next page that is of interest is one hugepage away -- seek @@ -323,8 +326,9 @@ std::optional PageFlags::Get(const void* const addr, // Check final page. It doesn't really matter if is_huge; we just want the // statistics about the page that has the last byte of the object. - size_t lastPageSize = kPageSize - (endPage - uaddr - size); - if (auto res = MaybeReadOne(endPage - kPageSize, result_flags, is_huge); + size_t lastPageSize = kHardwarePageSize - (endPage - uaddr - size); + if (auto res = + MaybeReadOne(endPage - kHardwarePageSize, result_flags, is_huge); res != absl::StatusCode::kOk) { return std::nullopt; } diff --git a/tcmalloc/internal/pageflags.h b/tcmalloc/internal/pageflags.h index c4a62aba1..6b18c41e7 100644 --- a/tcmalloc/internal/pageflags.h +++ b/tcmalloc/internal/pageflags.h @@ -136,13 +136,13 @@ class PageFlags final : public PageFlagsBase { static constexpr int kPagemapEntrySize = 8; static constexpr int kEntriesInBuf = kBufferLength / kPagemapEntrySize; - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); // You can technically not hard-code this but it would involve many more // queries to figure out the size of every page. It's a lot easier to just // assume any compound pages are 2 MB. static constexpr int kHugePageSize = (2 << 20); static constexpr uintptr_t kHugePageMask = ~(kHugePageSize - 1); - const size_t kPagesInHugePage = kHugePageSize / kPageSize; + const size_t kPagesInHugePage = kHugePageSize / kHardwarePageSize; uint64_t buf_[kEntriesInBuf]; // Information about the previous head page. For any future-encountered tail diff --git a/tcmalloc/internal/pageflags_test.cc b/tcmalloc/internal/pageflags_test.cc index e6e67a75c..98efc3687 100644 --- a/tcmalloc/internal/pageflags_test.cc +++ b/tcmalloc/internal/pageflags_test.cc @@ -147,21 +147,23 @@ TEST(PageFlagsTest, Stack) { TEST(PageFlagsTest, Alignment) { GTEST_SKIP() << "pageflags not commonly available"; - const size_t kPageSize = getpagesize(); - const int kNumPages = 6 * kHugePageSize / kPageSize; + const size_t kHardwarePageSize = getpagesize(); + const int kNumPages = 6 * kHugePageSize / kHardwarePageSize; for (auto mmap_hint : std::initializer_list{ nullptr, reinterpret_cast(0x00007BADDE000000), reinterpret_cast(0x00007BADDF001000)}) { void* p = mmap( - mmap_hint, kNumPages * kPageSize, PROT_READ | PROT_WRITE, + mmap_hint, kNumPages * kHardwarePageSize, PROT_READ | PROT_WRITE, (mmap_hint == nullptr ? 0 : MAP_FIXED) | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); ASSERT_NE(p, MAP_FAILED) << errno; - ASSERT_EQ(madvise(p, kPageSize * kNumPages, MADV_HUGEPAGE), 0) << errno; + ASSERT_EQ(madvise(p, kHardwarePageSize * kNumPages, MADV_HUGEPAGE), 0) + << errno; PageFlags s; - EXPECT_THAT(s.Get(p, kPageSize * kNumPages), Optional(PageStats{})) << p; - munmap(p, kNumPages * kPageSize); + EXPECT_THAT(s.Get(p, kHardwarePageSize * kNumPages), Optional(PageStats{})) + << p; + munmap(p, kNumPages * kHardwarePageSize); } } @@ -172,13 +174,13 @@ TEST(PageFlagsTest, Alignment) { // is page-aligned, this is a zero pointer (not to be confused with a null // pointer). void* GenerateAllStaleTest(absl::string_view filename, void* obj, size_t size) { - const size_t kPageSize = getpagesize(); + const size_t kHardwarePageSize = getpagesize(); uintptr_t ptr = reinterpret_cast(obj); uintptr_t pages_start = ptr & kHugePageMask; uintptr_t new_offset = ptr - pages_start; - off_t file_read_offset = pages_start / kPageSize * kPagemapEntrySize; + off_t file_read_offset = pages_start / kHardwarePageSize * kPagemapEntrySize; int read_fd = signal_safe_open("/proc/self/pageflags", O_RDONLY); CHECK_NE(read_fd, -1) << StrError(errno) @@ -208,16 +210,17 @@ void* GenerateAllStaleTest(absl::string_view filename, void* obj, size_t size) { TEST(PageFlagsTest, Stale) { GTEST_SKIP() << "pageflags not commonly available"; - constexpr size_t kPageSize = 4096; - constexpr int kNumPages = 6 * kHugePageSize / kPageSize; + constexpr size_t kHardwarePageSize = 4096; + constexpr int kNumPages = 6 * kHugePageSize / kHardwarePageSize; // This is hardcoded because we need to know number of pages in a hugepage. - ASSERT_EQ(getpagesize(), kPageSize); + ASSERT_EQ(getpagesize(), kHardwarePageSize); char* p = reinterpret_cast( - mmap(reinterpret_cast(0x00007BADDE001000), kNumPages * kPageSize, - PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); + mmap(reinterpret_cast(0x00007BADDE001000), + kNumPages * kHardwarePageSize, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); ASSERT_NE(p, MAP_FAILED) << errno; absl::BitGen rng; - for (int i = 0; i < kNumPages * kPageSize; ++i) { + for (int i = 0; i < kNumPages * kHardwarePageSize; ++i) { p[i] = absl::Uniform(rng, 0, 256); } @@ -232,7 +235,7 @@ TEST(PageFlagsTest, Stale) { ASSERT_EQ(madvise(p + 5 * kHugePageSize, kHugePageSize, MADV_HUGEPAGE), 0) << errno; PageFlags s; - ASSERT_THAT(s.Get(p, kPageSize * kNumPages), Optional(PageStats{})); + ASSERT_THAT(s.Get(p, kHardwarePageSize * kNumPages), Optional(PageStats{})); // This doesn't work within a short test timeout. But if you have your own // machine with appropriate patches, you can try it out! @@ -240,9 +243,9 @@ TEST(PageFlagsTest, Stale) { absl::Time start = absl::Now(); bool ok = false; do { - auto res = s.Get(p, kPageSize * kNumPages); + auto res = s.Get(p, kHardwarePageSize * kNumPages); ASSERT_TRUE(res.has_value()); - if (res->bytes_stale > kNumPages * kPageSize / 2) { + if (res->bytes_stale > kNumPages * kHardwarePageSize / 2) { LOG(INFO) << absl::StrFormat("Got %ld bytes stale, pointer is at %p", res->bytes_stale, p); ok = true; @@ -256,11 +259,11 @@ TEST(PageFlagsTest, Stale) { std::string fake_pageflags = absl::StrCat(testing::TempDir(), "/fake_pageflags"); void* fake_p = - GenerateAllStaleTest(fake_pageflags, p, kNumPages * kPageSize); + GenerateAllStaleTest(fake_pageflags, p, kNumPages * kHardwarePageSize); // fake_p is likely already aligned, but might as well make sure. This is // likely a zero pointer (not to be confused with nullptr). void* base_p = reinterpret_cast(reinterpret_cast(fake_p) & - ~(kPageSize - 1)); + ~(kHardwarePageSize - 1)); PageFlagsFriend mocks(fake_pageflags); constexpr uint64_t kSetScanSeconds = 63; mocks.SetCachedScanSeconds(kSetScanSeconds); @@ -271,48 +274,49 @@ TEST(PageFlagsTest, Stale) { // much overhead as adding a custom matcher. But if you add yet another // field here it's time to write one. uint64_t scan_seconds = kSetScanSeconds; - if (num_pages * kPageSize + offset == 0) scan_seconds = 0; + if (num_pages * kHardwarePageSize + offset == 0) scan_seconds = 0; // CAUTION: If you think this test is very flaky, it's possible it's // only passing when the machine you get scheduled on is out of // hugepages. - EXPECT_THAT(mocks.Get(base_p, num_pages * kPageSize + offset), - Optional(FieldsAre(num_pages * kPageSize + offset, 0, - scan_seconds))) + EXPECT_THAT(mocks.Get(base_p, num_pages * kHardwarePageSize + offset), + Optional(FieldsAre(num_pages * kHardwarePageSize + offset, + 0, scan_seconds))) << num_pages << "," << offset; - EXPECT_THAT( - mocks.Get((char*)fake_p - offset, num_pages * kPageSize + offset), - Optional( - FieldsAre(num_pages * kPageSize + offset, 0, scan_seconds))) + EXPECT_THAT(mocks.Get((char*)fake_p - offset, + num_pages * kHardwarePageSize + offset), + Optional(FieldsAre(num_pages * kHardwarePageSize + offset, + 0, scan_seconds))) << num_pages << "," << offset; - EXPECT_THAT(mocks.Get(fake_p, num_pages * kPageSize + offset), - Optional(FieldsAre(num_pages * kPageSize + offset, 0, - scan_seconds))) + EXPECT_THAT(mocks.Get(fake_p, num_pages * kHardwarePageSize + offset), + Optional(FieldsAre(num_pages * kHardwarePageSize + offset, + 0, scan_seconds))) << num_pages << "," << offset; - EXPECT_THAT( - mocks.Get((char*)fake_p + offset, num_pages * kPageSize + offset), - Optional( - FieldsAre(num_pages * kPageSize + offset, 0, scan_seconds))) + EXPECT_THAT(mocks.Get((char*)fake_p + offset, + num_pages * kHardwarePageSize + offset), + Optional(FieldsAre(num_pages * kHardwarePageSize + offset, + 0, scan_seconds))) << num_pages << "," << offset; scan_seconds = kSetScanSeconds; if (num_pages == 0) scan_seconds = 0; EXPECT_THAT( - mocks.Get((char*)kHugePageSize + offset, num_pages * kPageSize), - Optional(FieldsAre(num_pages * kPageSize, 0, scan_seconds))) + mocks.Get((char*)kHugePageSize + offset, + num_pages * kHardwarePageSize), + Optional(FieldsAre(num_pages * kHardwarePageSize, 0, scan_seconds))) << num_pages << "," << offset; } } EXPECT_THAT(mocks.Get(reinterpret_cast(2 * kHugePageSize + - 16 * kPageSize + 2), + 16 * kHardwarePageSize + 2), kHugePageSize * 3), Optional(FieldsAre(kHugePageSize * 3, 0, kSetScanSeconds))); } - ASSERT_EQ(munmap(p, kNumPages * kPageSize), 0) << errno; + ASSERT_EQ(munmap(p, kNumPages * kHardwarePageSize), 0) << errno; } TEST(PageFlagsTest, Locked) { @@ -323,21 +327,22 @@ TEST(PageFlagsTest, Locked) { GTEST_SKIP() << "Skipped under sanitizers."; #endif - constexpr size_t kPageSize = 4096; - constexpr int kNumPages = 6 * kHugePageSize / kPageSize; + constexpr size_t kHardwarePageSize = 4096; + constexpr int kNumPages = 6 * kHugePageSize / kHardwarePageSize; // This is hardcoded because we need to know number of pages in a hugepage. - ASSERT_EQ(getpagesize(), kPageSize); + ASSERT_EQ(getpagesize(), kHardwarePageSize); char* p = reinterpret_cast( - mmap(reinterpret_cast(0x00007BADDE000000), kNumPages * kPageSize, - PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); + mmap(reinterpret_cast(0x00007BADDE000000), + kNumPages * kHardwarePageSize, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); ASSERT_NE(p, MAP_FAILED) << errno; absl::BitGen rng; - for (int i = 0; i < kNumPages * kPageSize; ++i) { + for (int i = 0; i < kNumPages * kHardwarePageSize; ++i) { p[i] = absl::Uniform(rng, 0, 256); } PageFlags s; - ASSERT_THAT(s.Get(p, kPageSize * kNumPages), Optional(PageStats{})); + ASSERT_THAT(s.Get(p, kHardwarePageSize * kNumPages), Optional(PageStats{})); ASSERT_EQ(madvise(p, kHugePageSize, MADV_NOHUGEPAGE), 0) << errno; ASSERT_EQ(madvise(p + kHugePageSize, 3 * kHugePageSize, MADV_HUGEPAGE), 0) @@ -347,20 +352,20 @@ TEST(PageFlagsTest, Locked) { ASSERT_EQ(madvise(p + 5 * kHugePageSize, kHugePageSize, MADV_HUGEPAGE), 0) << errno; - ASSERT_THAT(s.Get(p, kPageSize * kNumPages), Optional(PageStats{})); + ASSERT_THAT(s.Get(p, kHardwarePageSize * kNumPages), Optional(PageStats{})); - ASSERT_EQ(mlock(p, kPageSize * kNumPages), 0) << errno; + ASSERT_EQ(mlock(p, kHardwarePageSize * kNumPages), 0) << errno; // Wait until the kernel has had time to propagate flags. absl::Time start = absl::Now(); do { - auto res = s.Get(p, kPageSize * kNumPages); + auto res = s.Get(p, kHardwarePageSize * kNumPages); ASSERT_TRUE(res.has_value()); - if (res->bytes_locked > kNumPages * kPageSize / 2) { + if (res->bytes_locked > kNumPages * kHardwarePageSize / 2) { LOG(INFO) << "Got " << res->bytes_locked << " bytes locked, pointer is at " << (uintptr_t)p; - if (res->bytes_locked == kNumPages * kPageSize) { + if (res->bytes_locked == kNumPages * kHardwarePageSize) { break; } } @@ -368,16 +373,16 @@ TEST(PageFlagsTest, Locked) { absl::SleepFor(absl::Milliseconds(100)); } while (absl::Now() - start < absl::Seconds(60)); - auto res = s.Get(p, kPageSize * kNumPages); + auto res = s.Get(p, kHardwarePageSize * kNumPages); ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->bytes_locked, kPageSize * kNumPages); + ASSERT_EQ(res->bytes_locked, kHardwarePageSize * kNumPages); - ASSERT_EQ(munmap(p, kNumPages * kPageSize), 0) << errno; + ASSERT_EQ(munmap(p, kNumPages * kHardwarePageSize), 0) << errno; } TEST(PageFlagsTest, OnlyTails) { - const size_t kPageSize = getpagesize(); - std::vector data(5 * kHugePageSize / kPageSize); + const size_t kHardwarePageSize = getpagesize(); + std::vector data(5 * kHugePageSize / kHardwarePageSize); for (auto& page : data) { page |= kPageTail; page |= kPageThp; @@ -408,8 +413,8 @@ TEST(PageFlagsTest, IsHugepageBackedBadFile) { // using MADV_COLLAPSE) to confirm the hugepage status using pageflags. TEST(PageFlagsTest, IsHugepageBacked) { const auto test_hugepage_status = [&](uint64_t flags, bool expected) { - const size_t kPageSize = getpagesize(); - const size_t kPagesPerHugepage = kHugePageSize / kPageSize; + const size_t kHardwarePageSize = getpagesize(); + const size_t kPagesPerHugepage = kHugePageSize / kHardwarePageSize; std::vector data(kPagesPerHugepage); for (auto& page : data) { @@ -440,16 +445,16 @@ TEST(PageFlagsTest, IsHugepageBacked) { } TEST(PageFlagsTest, TooManyTails) { - const size_t kPageSize = getpagesize(); - std::vector data(7 * kHugePageSize / kPageSize); + const size_t kHardwarePageSize = getpagesize(); + std::vector data(7 * kHugePageSize / kHardwarePageSize); for (auto& page : data) { page |= kPageTail; page |= kPageThp; } - data[kHugePageSize / kPageSize] = kPageHead | kPageThp; - data[2 * kHugePageSize / kPageSize] = kPageHead | kPageThp; - data[3 * kHugePageSize / kPageSize] = kPageHead | kPageThp; - data[5 * kHugePageSize / kPageSize] = kPageHead | kPageThp; + data[kHugePageSize / kHardwarePageSize] = kPageHead | kPageThp; + data[2 * kHugePageSize / kHardwarePageSize] = kPageHead | kPageThp; + data[3 * kHugePageSize / kHardwarePageSize] = kPageHead | kPageThp; + data[5 * kHugePageSize / kHardwarePageSize] = kPageHead | kPageThp; std::string file_path = absl::StrCat(testing::TempDir(), "/too-many-tails"); int write_fd = signal_safe_open(file_path.c_str(), O_CREAT | O_WRONLY, @@ -478,8 +483,8 @@ TEST(PageFlagsTest, TooManyTails) { } TEST(PageFlagsTest, NotThp) { - const size_t kPageSize = getpagesize(); - std::vector data(3 * kHugePageSize / kPageSize); + const size_t kHardwarePageSize = getpagesize(); + std::vector data(3 * kHugePageSize / kHardwarePageSize); for (auto& page : data) { page |= kPageHead; } @@ -625,8 +630,8 @@ TEST(PageFlagsTest, GetSinglePageBitmapsErrorCases) { { std::string fake_pageflags = absl::StrCat(testing::TempDir(), "/fake_pageflags_tail"); - const size_t kPageSize = getpagesize(); - const size_t kNativePagesInHugePage = kHugePageSize / kPageSize; + const size_t kHardwarePageSize = getpagesize(); + const size_t kNativePagesInHugePage = kHugePageSize / kHardwarePageSize; std::vector data(kNativePagesInHugePage, kPageTail); std::string content(reinterpret_cast(data.data()), diff --git a/tcmalloc/internal/residency.cc b/tcmalloc/internal/residency.cc index 7b92740e9..dbd316e48 100644 --- a/tcmalloc/internal/residency.cc +++ b/tcmalloc/internal/residency.cc @@ -87,7 +87,7 @@ ResidencyPageMap::~ResidencyPageMap() { } absl::StatusCode ResidencyPageMap::Seek(const uintptr_t vaddr) { - size_t offset = vaddr / kPageSize * kPagemapEntrySize; + size_t offset = vaddr / kHardwarePageSize * kPagemapEntrySize; // Note: lseek can't be interrupted. off_t status = ::lseek(fd_, offset, SEEK_SET); if (status != offset) { @@ -124,7 +124,7 @@ absl::StatusCode ResidencyPageMap::ReadMany(int64_t num_pages, return absl::StatusCode::kUnavailable; } for (int i = 0; i < batch_size; ++i) { - Update(buf_[i], kPageSize, info); + Update(buf_[i], kHardwarePageSize, info); } num_pages -= batch_size; } @@ -142,12 +142,13 @@ std::optional ResidencyPageMap::Get(const void* const addr, uintptr_t uaddr = reinterpret_cast(addr); // Round address down to get the start of the page containing the data. - uintptr_t basePage = uaddr & ~(kPageSize - 1); + uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1); // Round end address up to get the end of the page containing the data. // The data is in [basePage, endPage). - uintptr_t endPage = (uaddr + size + kPageSize - 1) & ~(kPageSize - 1); + uintptr_t endPage = + (uaddr + size + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1); - int64_t remainingPages = (endPage - basePage) / kPageSize; + int64_t remainingPages = (endPage - basePage) / kHardwarePageSize; if (auto res = Seek(basePage); res != absl::StatusCode::kOk) { return std::nullopt; @@ -168,7 +169,7 @@ std::optional ResidencyPageMap::Get(const void* const addr, if (!res.has_value()) return std::nullopt; // Handle the first page. - size_t firstPageSize = kPageSize - (uaddr - basePage); + size_t firstPageSize = kHardwarePageSize - (uaddr - basePage); Update(res.value(), firstPageSize, info); remainingPages--; @@ -179,7 +180,7 @@ std::optional ResidencyPageMap::Get(const void* const addr, } // Check final page - size_t lastPageSize = kPageSize - (endPage - uaddr - size); + size_t lastPageSize = kHardwarePageSize - (endPage - uaddr - size); res = ReadOne(); if (!res.has_value()) return std::nullopt; Update(res.value(), lastPageSize, info); diff --git a/tcmalloc/internal/residency.h b/tcmalloc/internal/residency.h index 440008405..02d595fec 100644 --- a/tcmalloc/internal/residency.h +++ b/tcmalloc/internal/residency.h @@ -119,10 +119,10 @@ class ResidencyPageMap : public Residency { static constexpr int kPagemapEntrySize = 8; static constexpr int kEntriesInBuf = kBufferLength / kPagemapEntrySize; - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); static constexpr uintptr_t kHugePageMask = ~(kHugePageSize - 1); - const size_t kNativePagesInHugePage = kHugePageSize / kPageSize; + const size_t kNativePagesInHugePage = kHugePageSize / kHardwarePageSize; uint64_t buf_[kEntriesInBuf]; const int fd_; diff --git a/tcmalloc/internal/residency_test.cc b/tcmalloc/internal/residency_test.cc index c93c2d906..83933d3fb 100644 --- a/tcmalloc/internal/residency_test.cc +++ b/tcmalloc/internal/residency_test.cc @@ -72,70 +72,73 @@ constexpr uint64_t kPageSwapped = (1ULL << 62); constexpr uint64_t kPagePresent = (1ULL << 63); TEST(ResidenceTest, ThisProcess) { - const size_t kPageSize = GetPageSize(); + const size_t kHardwarePageSize = GetPageSize(); const int kNumPages = 16; // Try both private and shared mappings to make sure we have the bit order of // /proc/pid/pageflags correct. for (const int flags : {MAP_ANONYMOUS | MAP_SHARED, MAP_ANONYMOUS | MAP_PRIVATE}) { - const size_t kHead = kPageSize * 10; - const size_t kTail = kPageSize * 10; + const size_t kHead = kHardwarePageSize * 10; + const size_t kTail = kHardwarePageSize * 10; ResidencyPageMap r; // Overallocate kNumPages of memory, so we can munmap the page before and // after it. - void* p = mmap(nullptr, kNumPages * kPageSize + kHead + kTail, + void* p = mmap(nullptr, kNumPages * kHardwarePageSize + kHead + kTail, PROT_READ | PROT_WRITE, flags, -1, 0); ASSERT_NE(p, MAP_FAILED) << errno; - EXPECT_THAT(r.Get(p, (kNumPages + 2) * kPageSize), + EXPECT_THAT(r.Get(p, (kNumPages + 2) * kHardwarePageSize), Optional(FieldsAre(0, 0))); - ASSERT_EQ(munmap(p, kPageSize), 0); + ASSERT_EQ(munmap(p, kHardwarePageSize), 0); void* q = reinterpret_cast(p) + kHead; - void* last = reinterpret_cast(p) + kNumPages * kPageSize + kHead; - ASSERT_EQ(munmap(last, kPageSize), 0); + void* last = + reinterpret_cast(p) + kNumPages * kHardwarePageSize + kHead; + ASSERT_EQ(munmap(last, kHardwarePageSize), 0); EXPECT_THAT(r.Get(p, kHead), Optional(FieldsAre(0, 0))); EXPECT_THAT(r.Get(last, kTail), Optional(FieldsAre(0, 0))); - memset(q, 0, kNumPages * kPageSize); - (void)mlock(q, kNumPages * kPageSize); + memset(q, 0, kNumPages * kHardwarePageSize); + (void)mlock(q, kNumPages * kHardwarePageSize); ::benchmark::DoNotOptimize(q); EXPECT_THAT(r.Get(p, kHead), Optional(FieldsAre(0, 0))); EXPECT_THAT(r.Get(last, kTail), Optional(FieldsAre(0, 0))); - EXPECT_THAT(r.Get(q, kPageSize), Optional(FieldsAre(kPageSize, 0))); + EXPECT_THAT(r.Get(q, kHardwarePageSize), + Optional(FieldsAre(kHardwarePageSize, 0))); - EXPECT_THAT(r.Get(q, (kNumPages + 2) * kPageSize), - Optional(FieldsAre(kPageSize * kNumPages, 0))); + EXPECT_THAT(r.Get(q, (kNumPages + 2) * kHardwarePageSize), + Optional(FieldsAre(kHardwarePageSize * kNumPages, 0))); - EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, kPageSize - 7), - Optional(FieldsAre(kPageSize - 7, 0))); + EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, kHardwarePageSize - 7), + Optional(FieldsAre(kHardwarePageSize - 7, 0))); - EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, kPageSize), - Optional(FieldsAre(kPageSize, 0))); + EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, kHardwarePageSize), + Optional(FieldsAre(kHardwarePageSize, 0))); - EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, 3 * kPageSize), - Optional(FieldsAre(kPageSize * 3, 0))); - - EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, kNumPages * kPageSize), - Optional(FieldsAre(kPageSize * kNumPages - 7, 0))); + EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, 3 * kHardwarePageSize), + Optional(FieldsAre(kHardwarePageSize * 3, 0))); EXPECT_THAT( - r.Get(reinterpret_cast(q) + 7, kNumPages * kPageSize - 7), - Optional(FieldsAre(kPageSize * kNumPages - 7, 0))); + r.Get(reinterpret_cast(q) + 7, kNumPages * kHardwarePageSize), + Optional(FieldsAre(kHardwarePageSize * kNumPages - 7, 0))); - EXPECT_THAT( - r.Get(reinterpret_cast(q) + 7, (kNumPages + 1) * kPageSize), - Optional(FieldsAre(kPageSize * kNumPages - 7, 0))); + EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, + kNumPages * kHardwarePageSize - 7), + Optional(FieldsAre(kHardwarePageSize * kNumPages - 7, 0))); - EXPECT_THAT( - r.Get(reinterpret_cast(q) + 7, (kNumPages + 1) * kPageSize - 7), - Optional(FieldsAre(kPageSize * kNumPages - 7, 0))); + EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, + (kNumPages + 1) * kHardwarePageSize), + Optional(FieldsAre(kHardwarePageSize * kNumPages - 7, 0))); + + EXPECT_THAT(r.Get(reinterpret_cast(q) + 7, + (kNumPages + 1) * kHardwarePageSize - 7), + Optional(FieldsAre(kHardwarePageSize * kNumPages - 7, 0))); - ASSERT_EQ(munmap(q, kNumPages * kPageSize), 0); + ASSERT_EQ(munmap(q, kNumPages * kHardwarePageSize), 0); } }