Add custom mmap for posix systems (macOS performance fix).

builds/gnu/Makefile.am

@@ -56,6 +56,7 @@ src_libbitcoin_database_la_SOURCES = \
     ${srcdir}/../../src/locks/flush_lock.cpp \
     ${srcdir}/../../src/locks/interprocess_lock.cpp \
     ${srcdir}/../../src/memory/mman.cpp \
+    ${srcdir}/../../src/memory/mstage.cpp \
     ${srcdir}/../../src/memory/utilities.cpp \
     ${srcdir}/../../src/types/history.cpp \
     ${srcdir}/../../src/types/multisig_view.cpp \
@@ -94,6 +95,7 @@ include_bitcoin_database_impl_memory_HEADERS = \
     ${srcdir}/../../include/bitcoin/database/impl/memory/mmap.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/memory/mmap_dispatch.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/memory/mmap_private.ipp \
+    ${srcdir}/../../include/bitcoin/database/impl/memory/mmap_staging.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/memory/mmap_storage.ipp
 
 include_bitcoin_database_impl_primitivesdir = \
@@ -223,6 +225,7 @@ include_bitcoin_database_memory_HEADERS = \
     ${srcdir}/../../include/bitcoin/database/memory/mman.hpp \
     ${srcdir}/../../include/bitcoin/database/memory/mmap.hpp \
     ${srcdir}/../../include/bitcoin/database/memory/mmaps.hpp \
+    ${srcdir}/../../include/bitcoin/database/memory/mstage.hpp \
     ${srcdir}/../../include/bitcoin/database/memory/reader.hpp \
     ${srcdir}/../../include/bitcoin/database/memory/streamers.hpp \
     ${srcdir}/../../include/bitcoin/database/memory/utilities.hpp

include/bitcoin/database.hpp

@@ -35,6 +35,7 @@
 #include <bitcoin/database/memory/mman.hpp>
 #include <bitcoin/database/memory/mmap.hpp>
 #include <bitcoin/database/memory/mmaps.hpp>
+#include <bitcoin/database/memory/mstage.hpp>
 #include <bitcoin/database/memory/reader.hpp>
 #include <bitcoin/database/memory/streamers.hpp>
 #include <bitcoin/database/memory/utilities.hpp>

include/bitcoin/database/impl/memory/mmap.ipp

@@ -1,118 +1,134 @@
-/**
- * Copyright (c) 2011-2026 libbitcoin developers
- *
- * This file is part of libbitcoin.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef LIBBITCOIN_DATABASE_MEMORY_MMAP_IPP
-#define LIBBITCOIN_DATABASE_MEMORY_MMAP_IPP
-
-#include <algorithm>
-#include <filesystem>
-#include <shared_mutex>
-#include <bitcoin/database/define.hpp>
-#include <bitcoin/database/file/file.hpp>
-
-namespace libbitcoin {
-namespace database {
-
-// Constructors.
-// ----------------------------------------------------------------------------
-
-TEMPLATE
-CLASS::mmap(const path& filename, size_t minimum, size_t expansion,
-    bool random) NOEXCEPT
-    requires (is_one(columns))
-  : filenames_{ filename },
-    minimum_(to_rows(minimum)),
-    expansion_(expansion),
-    random_(random),
-    opened_{ file::invalid }
-{
-}
-
-TEMPLATE
-CLASS::mmap(const paths& filenames, size_t minimum, size_t expansion,
-    bool random) NOEXCEPT
-    requires (columns > one)
-  : filenames_(filenames),
-    minimum_(to_rows(minimum)),
-    expansion_(expansion),
-    random_(random),
-    opened_{}
-{
-    opened_.fill(file::invalid);
-}
-
-TEMPLATE
-CLASS::~mmap() NOEXCEPT
-{
-    BC_ASSERT(!loaded_.load());
-    BC_ASSERT(is_zero(logical_.load()));
-    BC_ASSERT(is_zero(capacity_.load()));
-    BC_ASSERT(std::ranges::all_of(memory_map_,
-        [](auto map) NOEXCEPT { return is_null(map); }));
-    BC_ASSERT(std::ranges::all_of(opened_,
-        [](auto opened) NOEXCEPT { return opened == file::invalid; }));
-}
-
-TEMPLATE
-bool CLASS::is_open() const NOEXCEPT
-{
-    std::shared_lock field_lock(field_mutex_);
-    return opened_.front() != file::invalid;
-}
-
-TEMPLATE
-bool CLASS::is_loaded() const NOEXCEPT
-{
-    return loaded_.load();
-}
-
-// protected
-// ----------------------------------------------------------------------------
-
-TEMPLATE
-size_t CLASS::to_capacity(size_t required) const NOEXCEPT
-{
-    // Covert required rows to capacity-padded rows.
-    using namespace system;
-    const auto growth = ceilinged_multiply(required, expansion_) / 100u;
-    return std::max(minimum_, ceilinged_add(required, growth));
-}
-
-// Write-write protected by remap_mutex.
-TEMPLATE
-void CLASS::set_first_code(const error::error_t& ec) NOEXCEPT
-{
-    if (!fault_.load())
-    {
-        fault_.store(true);
-
-        // error is atomic for public read exposure.
-        error_.store(ec);
-    }
-}
-
-TEMPLATE
-void CLASS::set_disk_space(size_t required) NOEXCEPT
-{
-    space_.store(required);
-}
-
-} // namespace database
-} // namespace libbitcoin
-
-#endif
+/**
+ * Copyright (c) 2011-2026 libbitcoin developers
+ *
+ * This file is part of libbitcoin.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef LIBBITCOIN_DATABASE_MEMORY_MMAP_IPP
+#define LIBBITCOIN_DATABASE_MEMORY_MMAP_IPP
+
+#include <algorithm>
+#include <filesystem>
+#include <shared_mutex>
+#include <bitcoin/database/define.hpp>
+#include <bitcoin/database/file/file.hpp>
+
+namespace libbitcoin {
+namespace database {
+
+// Constructors.
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+CLASS::mmap(const path& filename, size_t minimum, size_t expansion,
+    bool random, bool staged) NOEXCEPT
+    requires (is_one(columns))
+  : filenames_{ filename },
+    minimum_(to_rows(minimum)),
+    expansion_(expansion),
+    random_(random),
+    staged_(staged),
+    opened_{ file::invalid }
+{
+}
+
+TEMPLATE
+CLASS::mmap(const paths& filenames, size_t minimum, size_t expansion,
+    bool random, bool staged) NOEXCEPT
+    requires (columns > one)
+  : filenames_(filenames),
+    minimum_(to_rows(minimum)),
+    expansion_(expansion),
+    random_(random),
+    staged_(staged),
+    opened_{}
+{
+    opened_.fill(file::invalid);
+}
+
+TEMPLATE
+CLASS::~mmap() NOEXCEPT
+{
+#if defined(MANAGE_STAGING)
+    // Join a settler left running by an unload bypass (thread safety).
+    settler_stop_();
+#endif
+
+    BC_ASSERT(!loaded_.load());
+    BC_ASSERT(is_zero(logical_.load()));
+    BC_ASSERT(is_zero(capacity_.load()));
+    BC_ASSERT(std::ranges::all_of(memory_map_,
+        [](auto map) NOEXCEPT { return is_null(map); }));
+    BC_ASSERT(std::ranges::all_of(opened_,
+        [](auto opened) NOEXCEPT { return opened == file::invalid; }));
+#if defined(MANAGE_STAGING)
+    BC_ASSERT(std::ranges::all_of(reserved_,
+        [](auto reserved) NOEXCEPT { return is_zero(reserved); }));
+#endif
+}
+
+TEMPLATE
+bool CLASS::is_open() const NOEXCEPT
+{
+    std::shared_lock field_lock(field_mutex_);
+    return opened_.front() != file::invalid;
+}
+
+TEMPLATE
+bool CLASS::is_loaded() const NOEXCEPT
+{
+    return loaded_.load();
+}
+
+// protected
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+size_t CLASS::to_capacity(size_t required) const NOEXCEPT
+{
+    // Covert required rows to capacity-padded rows.
+    using namespace system;
+    const auto growth = ceilinged_multiply(required, expansion_) / 100u;
+    return std::max(minimum_, ceilinged_add(required, growth));
+}
+
+// Write-write protected by remap_mutex.
+TEMPLATE
+void CLASS::set_first_code(const error::error_t& ec) NOEXCEPT
+{
+    if (!fault_.load())
+    {
+        fault_.store(true);
+
+        // error is atomic for public read exposure.
+        error_.store(ec);
+
+#if defined(MANAGE_STAGING)
+        // A fault may stop draining, so it must release throttled callers.
+        signal_();
+#endif
+    }
+}
+
+TEMPLATE
+void CLASS::set_disk_space(size_t required) NOEXCEPT
+{
+    space_.store(required);
+}
+
+} // namespace database
+} // namespace libbitcoin
+
+#endif