diff --git a/CMakeLists.txt b/CMakeLists.txt index 1dc1e782c..050703516 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) +option(MI_BUILD_SEGMENT_MAP_BENCH "Build the experimental segment-map contention benchmark (Linux only)" OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) @@ -783,6 +784,20 @@ if (MI_BUILD_TESTS) endif() endif() +if(MI_BUILD_SEGMENT_MAP_BENCH) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "The segment-map contention benchmark is only supported on Linux") + endif() + if(NOT MI_BUILD_STATIC OR MI_DEBUG_TSAN) + message(FATAL_ERROR "The segment-map contention benchmark requires MI_BUILD_STATIC=ON and MI_DEBUG_TSAN=OFF") + endif() + add_executable(mimalloc-bench-segment-map test/bench-segment-map.c) + target_compile_definitions(mimalloc-bench-segment-map PRIVATE ${mi_defines}) + target_compile_options(mimalloc-bench-segment-map PRIVATE ${mi_cflags}) + target_include_directories(mimalloc-bench-segment-map PRIVATE include) + target_link_libraries(mimalloc-bench-segment-map PRIVATE mimalloc-static ${mi_libraries}) +endif() + # ----------------------------------------------------------------------------- # Set override properties # ----------------------------------------------------------------------------- diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 618074b9b..464c0a43d 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -73,6 +73,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) #define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) #define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed)) +#define mi_atomic_exchange_acquire(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acquire)) #define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) #define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) #define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) @@ -84,6 +85,8 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed)) #define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel)) #define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel)) +#define mi_atomic_and_release(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(release)) +#define mi_atomic_or_release(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(release)) #define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel)) #define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) @@ -393,9 +396,8 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t; // Allows only one thread to execute at a time (without blocking anyone) #define mi_atomic_guard(guard) \ - uintptr_t _mi_guard_expected = 0; \ for(bool _mi_guard_once = true; \ - _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \ + _mi_guard_once && mi_atomic_load_relaxed(guard) == 0 && mi_atomic_exchange_acquire(guard,(uintptr_t)1) == 0; \ (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) ) diff --git a/src/segment-map.c b/src/segment-map.c index e3cbfc166..265f75792 100644 --- a/src/segment-map.c +++ b/src/segment-map.c @@ -85,11 +85,7 @@ void _mi_segment_map_allocated_at(const mi_segment_t* segment) { size_t bitidx; mi_segmap_part_t* part = mi_segment_map_index_of(segment, true /* alloc map if needed */, &index, &bitidx); if (part == NULL) return; // outside our address range.. - uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); - uintptr_t newmask; - do { - newmask = (mask | ((uintptr_t)1 << bitidx)); - } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask)); + mi_atomic_or_release(&part->map[index], ((uintptr_t)1 << bitidx)); } void _mi_segment_map_freed_at(const mi_segment_t* segment) { @@ -98,11 +94,7 @@ void _mi_segment_map_freed_at(const mi_segment_t* segment) { size_t bitidx; mi_segmap_part_t* part = mi_segment_map_index_of(segment, false /* don't alloc if not present */, &index, &bitidx); if (part == NULL) return; // outside our address range.. - uintptr_t mask = mi_atomic_load_relaxed(&part->map[index]); - uintptr_t newmask; - do { - newmask = (mask & ~((uintptr_t)1 << bitidx)); - } while (!mi_atomic_cas_weak_release(&part->map[index], &mask, newmask)); + mi_atomic_and_release(&part->map[index], ~((uintptr_t)1 << bitidx)); } // Determine the segment belonging to a pointer or NULL if it is not in a valid segment. diff --git a/test/bench-segment-map.c b/test/bench-segment-map.c new file mode 100644 index 000000000..e9179c4b9 --- /dev/null +++ b/test/bench-segment-map.c @@ -0,0 +1,146 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026 Microsoft Corporation +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* Measure contended segment-map updates without including OS allocation costs. + + Build: + cmake -S . -B out/segment-map -DCMAKE_BUILD_TYPE=Release \ + -DMI_NO_OPT_ARCH=ON -DMI_BUILD_SEGMENT_MAP_BENCH=ON + cmake --build out/segment-map --target mimalloc-bench-segment-map + + Run: + taskset -c 0-7 out/segment-map/mimalloc-bench-segment-map 8 5000000 + + Each thread repeatedly sets and clears a separate segment bit. Consecutive + synthetic segments ensure those bits share one atomic segment-map word. +*/ + +#define _GNU_SOURCE + +#include +#include + +#include +#include +#include +#include +#include +#include + +typedef struct mi_segment_map_bench_s { + int threads; + size_t iterations; + mi_segment_t* segments; + pthread_barrier_t barrier; +} mi_segment_map_bench_t; + +typedef struct mi_segment_map_worker_s { + mi_segment_map_bench_t* bench; + int id; +} mi_segment_map_worker_t; + +static double elapsed_seconds(struct timespec start, struct timespec end) { + return (double)(end.tv_sec - start.tv_sec) + + (double)(end.tv_nsec - start.tv_nsec) / 1000000000.0; +} + +static void* segment_map_worker(void* argument) { + mi_segment_map_worker_t* worker = (mi_segment_map_worker_t*)argument; + mi_segment_map_bench_t* bench = worker->bench; + mi_segment_t* segment = + (mi_segment_t*)((uint8_t*)bench->segments + (size_t)worker->id * MI_SEGMENT_SIZE); + + pthread_barrier_wait(&bench->barrier); + for (size_t i = 0; i < bench->iterations; i++) { + _mi_segment_map_allocated_at(segment); + _mi_segment_map_freed_at(segment); + } + return NULL; +} + +static void* reserve_segment_range(size_t size) { + for (uintptr_t address = (uintptr_t)1 << 40; + address < ((uintptr_t)32 << 40); + address += (uintptr_t)1 << 40) { + void* mapping = mmap((void*)address, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (mapping != MAP_FAILED) return mapping; + } + return NULL; +} + +int main(int argc, char** argv) { + if (argc != 3) { + fprintf(stderr, "usage: %s THREADS ITERATIONS\n", argv[0]); + return 2; + } + + mi_segment_map_bench_t bench; + bench.threads = atoi(argv[1]); + bench.iterations = (size_t)strtoull(argv[2], NULL, 10); + if (bench.threads <= 0 || bench.threads > (int)MI_INTPTR_BITS || + bench.iterations == 0) { + fprintf(stderr, "threads must be between 1 and %d; iterations must be positive\n", + MI_INTPTR_BITS); + return 2; + } + + const size_t range_size = (size_t)bench.threads * MI_SEGMENT_SIZE; + bench.segments = (mi_segment_t*)reserve_segment_range(range_size); + if (bench.segments == NULL) { + fprintf(stderr, "unable to reserve a tracked segment-aligned address range\n"); + return 1; + } + + for (int i = 0; i < bench.threads; i++) { + mi_segment_t* segment = + (mi_segment_t*)((uint8_t*)bench.segments + (size_t)i * MI_SEGMENT_SIZE); + segment->memid = _mi_memid_create(MI_MEM_OS); + _mi_segment_map_allocated_at(segment); + _mi_segment_map_freed_at(segment); + } + + pthread_t* threads = (pthread_t*)calloc((size_t)bench.threads, sizeof(*threads)); + mi_segment_map_worker_t* workers = + (mi_segment_map_worker_t*)calloc((size_t)bench.threads, sizeof(*workers)); + if (threads == NULL || workers == NULL) { + free(workers); + free(threads); + munmap(bench.segments, range_size); + return 1; + } + + pthread_barrier_init(&bench.barrier, NULL, (unsigned)bench.threads + 1); + for (int i = 0; i < bench.threads; i++) { + workers[i].bench = &bench; + workers[i].id = i; + if (pthread_create(&threads[i], NULL, segment_map_worker, &workers[i]) != 0) { + abort(); + } + } + + struct timespec start; + struct timespec end; + pthread_barrier_wait(&bench.barrier); + clock_gettime(CLOCK_MONOTONIC, &start); + for (int i = 0; i < bench.threads; i++) { + pthread_join(threads[i], NULL); + } + clock_gettime(CLOCK_MONOTONIC, &end); + + const double seconds = elapsed_seconds(start, end); + const double operations = + 2.0 * (double)bench.threads * (double)bench.iterations; + printf("elapsed: %.3f s\n", seconds); + printf("segment-map operation: %.3f ns\n", seconds * 1000000000.0 / operations); + + pthread_barrier_destroy(&bench.barrier); + free(workers); + free(threads); + munmap(bench.segments, range_size); + return 0; +}