Posix mmap optimization work in progress.

include/bitcoin/database/file/utilities.hpp

@@ -21,6 +21,7 @@
 
 #include <filesystem>
 #include <bitcoin/database/define.hpp>
+#include <bitcoin/database/memory/settings.hpp>
 
 namespace libbitcoin {
 namespace database {
@@ -71,9 +72,10 @@ BCD_API bool copy_directory(const path& from, const path& to) NOEXCEPT;
 BCD_API code copy_directory_ex(const path& from, const path& to) NOEXCEPT;
 
 /// File descriptor functions (for memory mapping).
-BCD_API int open(const path& filename, bool random=true) NOEXCEPT;
+BCD_API int open(const path& filename, bool random=true,
+    advice access=advice::normal) NOEXCEPT;
 BCD_API code open_ex(int& file_descriptor, const path& filename,
-    bool random=true) NOEXCEPT;
+    bool random=true, advice access=advice::normal) NOEXCEPT;
 BCD_API bool close(int file_descriptor) NOEXCEPT;
 BCD_API code close_ex(int file_descriptor) NOEXCEPT;
 BCD_API bool size(size_t& out, int file_descriptor) NOEXCEPT;

include/bitcoin/database/impl/memory/mmap.ipp

@@ -38,6 +38,7 @@ CLASS::mmap(const path& filename, const storage_settings& settings,
   : filenames_{ filename },
     minimum_(to_rows(settings.size)),
     expansion_(settings.rate),
+    access_(settings.access),
     random_(random),
     staged_(staged),
     opened_{ file::invalid }
@@ -51,6 +52,7 @@ CLASS::mmap(const paths& filenames, const storage_settings& settings,
   : filenames_(filenames),
     minimum_(to_rows(settings.size)),
     expansion_(settings.rate),
+    access_(settings.access),
     random_(random),
     staged_(staged),
     opened_{}
@@ -104,6 +106,92 @@ size_t CLASS::to_capacity(size_t required) const NOEXCEPT
     return std::max(minimum_, ceilinged_add(required, growth));
 }
 
+// Commitment growth target for the capacity slow paths. Unlike to_capacity
+// this never floors to the configured minimum: under lazy commitment that
+// floor would commit the full provisioning on first growth (the load failure
+// this design exists to prevent, moved from create to first touch). Growth is
+// chunked to bound slow path frequency, and clamped so that small tables do
+// not over-commit (the provisioned file requires no memory until committed).
+TEMPLATE
+size_t CLASS::to_growth(size_t required) const NOEXCEPT
+{
+#if defined(MANAGE_STAGING)
+    using namespace system;
+    const auto expand = ceilinged_multiply(required, expansion_) / 100u;
+    const auto expanded = ceilinged_add(required, expand);
+    const auto chunked = std::max(expanded,
+        ceilinged_add(capacity_.load(), to_rows(commit_chunk)));
+
+    return std::min(chunked, std::max(expanded, to_provision()));
+#else
+    // The classic mapping is file-backed, so growth is capacity.
+    return to_capacity(required);
+#endif
+}
+
+// Disk provisioning: the configured minimum is a file reservation, ensuring
+// that allocation within it cannot fail for space (disk full is detected at
+// provisioning, where it remains recoverable).
+TEMPLATE
+size_t CLASS::to_provision() const NOEXCEPT
+{
+    return std::max(logical_.load(), minimum_);
+}
+
+// Memory commitment follows use, not provisioning. Committing the configured
+// minimum would demand that much memory (Linux charges the commit) before any
+// row is written, failing load on any machine smaller than its store sizing.
+// Growth commits in chunks within the standing reservation (no remap), so the
+// cost is a syscall per chunk over the life of the store.
+TEMPLATE
+size_t CLASS::to_commitment() const NOEXCEPT
+{
+#if defined(MANAGE_STAGING)
+    const auto logical = logical_.load();
+    return std::min(to_provision(), std::max(logical, to_rows(commit_chunk)));
+#else
+    // The classic mapping is file-backed, so commitment is provisioning.
+    return to_provision();
+#endif
+}
+
+// The counter chain is the spine of the design (debug assertion only):
+//
+//     settled_ <= frontier_ <= logical_ <= capacity_ <= file_
+//
+// settled: rows durable and converted (staged) or drained (unstaged).
+// frontier: completed-write prefix bound (extent ring floor).
+// logical: rows claimed by writers (fast path CAS).
+// capacity: rows claimable (committed memory).
+// file: rows provisioned on disk (fallocate extent).
+//
+// Lower bounds are read first: the lock-free fast path can advance logical_
+// concurrently, so stale-low reads of lower bounds cannot falsify the chain,
+// while upper bounds only grow under locks held at every call site.
+TEMPLATE
+void CLASS::check_invariants_() const NOEXCEPT
+{
+#if !defined(NDEBUG)
+    if (!loaded_.load())
+        return;
+
+#if defined(MANAGE_STAGING)
+    if (staged_)
+    {
+        const auto settled = settled_.load();
+        const auto frontier = frontier_.load();
+        BC_ASSERT(settled <= frontier);
+        BC_ASSERT(frontier <= logical_.load());
+    }
+#endif
+
+    const auto logical = logical_.load();
+    const auto capacity = capacity_.load();
+    BC_ASSERT(logical <= capacity);
+    BC_ASSERT(capacity <= file_.load());
+#endif // NDEBUG
+}
+
 // Write-write protected by remap_mutex.
 TEMPLATE
 void CLASS::set_first_code(const error::error_t& ec) NOEXCEPT

include/bitcoin/database/impl/memory/mmap_dispatch.ipp

@@ -48,7 +48,7 @@ memory CLASS::get_filled(size_t offset, size_t size,
         const auto end = std::max(logical_.load(), offset + size);
         if (end > capacity_.load())
         {
-            const auto extended = to_capacity(end);
+            const auto extended = to_growth(end);
 
             // TODO: Could loop over a try lock here and log deadlock warning.
             std::unique_lock remap_lock(remap_mutex_);

include/bitcoin/database/impl/memory/mmap_private.ipp

@@ -67,10 +67,14 @@ bool CLASS::map_all_(std::index_sequence<Index...>) NOEXCEPT
     if (!(map_<Index>() && ...))
     {
         capacity_.store(zero);
+        file_.store(zero);
         return false;
     }
 
-    capacity_.store(std::max(logical_.load(), minimum_));
+    // The file is provisioned in full; capacity publishes committed rows only.
+    file_.store(to_provision());
+    capacity_.store(to_commitment());
+    check_invariants_();
     return true;
 }
 
@@ -82,6 +86,9 @@ bool CLASS::unmap_all_(std::index_sequence<Index...>) NOEXCEPT
     const auto success = (unmap_<Index>(capacity) && ...);
     capacity_.store(zero);
 
+    // Unmapping truncates the file to logical, which is then its provisioning.
+    file_.store(logical_.load());
+
 #if defined(MANAGE_STAGING)
     window_.store(zero);
     settled_.store(zero);
@@ -104,7 +111,11 @@ bool CLASS::remap_all_(size_t capacity, std::index_sequence<Index...>) NOEXCEPT
         return false;
     }
 
+    // Growth beyond the provisioned file extends it (resize_ is a no-op
+    // within), so the extent tracks the high water of provisioning.
+    file_.store(std::max(file_.load(), capacity));
     capacity_.store(capacity);
+    check_invariants_();
     return true;
 }
 
@@ -240,11 +251,11 @@ bool CLASS::map_() NOEXCEPT
 #if defined(MANAGE_STAGING)
     return stage_<Column>();
 #else
-    auto size = logical_.load();
-
     // Cannot map empty file, and want minimum capacity, so expand as required.
+    // The classic mapping is file-backed, so commitment is provisioning.
     // disk_full: space is set but no code is set with false return.
-    if ((size < minimum_) && !resize_<Column>(size = minimum_))
+    const auto size = to_provision();
+    if (!resize_<Column>(size))
         return false;
 
     memory_map_[Column] = system::pointer_cast<uint8_t>(
@@ -300,8 +311,14 @@ TEMPLATE
 template <size_t Column>
 bool CLASS::resize_(size_t size) NOEXCEPT
 {
+    // The file is provisioned ahead of commitment, so growth within the
+    // provisioned extent requires no disk operation (the space is reserved).
+    const auto extent = file_.load();
+    if (size <= extent)
+        return true;
+
     const auto target = to_width<Column>(size);
-    const auto capacity = to_width<Column>(capacity_.load());
+    const auto capacity = to_width<Column>(extent);
 
     // Disk full detection, any other failure is an abort.
 #if !defined(WITHOUT_FALLOCATE)
@@ -314,8 +331,8 @@ bool CLASS::resize_(size_t size) NOEXCEPT
         if (errno == ENOSPC)
         {
             using namespace system;
-            set_disk_space(ceilinged_multiply(floored_subtract(size,
-                capacity_.load()), stride));
+            set_disk_space(ceilinged_multiply(floored_subtract(size, extent),
+                stride));
             return false;
         }
 

include/bitcoin/database/impl/memory/mmap_staging.ipp

@@ -20,8 +20,13 @@
 #define LIBBITCOIN_DATABASE_MEMORY_MMAP_STAGING_IPP
 
 #include <algorithm>
+#include <array>
 #include <chrono>
 #include <fcntl.h>
+#if defined(STAGING_TELEMETRY)
+    #include <iostream>
+    #include <sstream>
+#endif
 #include <bitcoin/database/define.hpp>
 #include <bitcoin/database/memory/mstage.hpp>
 #include <bitcoin/database/memory/utilities.hpp>
@@ -186,6 +191,7 @@ void CLASS::maintain_() NOEXCEPT
     window_.store(pack_word<uint64_t>(head, size), release);
     frontier_.store(is_zero(size) ? logical_.load() :
         ring_.at(head).start.load(relaxed));
+    check_invariants_();
 }
 
 // Discard all extents, requires quiescent writers (locked). Any extent then
@@ -200,6 +206,7 @@ void CLASS::discard_() NOEXCEPT
 
     window_.store(zero, release);
     frontier_.store(logical_.load());
+    check_invariants_();
 }
 
 // Delay the caller while staging exceeds its memory bound (write throttle).
@@ -250,6 +257,7 @@ bool CLASS::settle_all_(size_t rows, std::index_sequence<Index...>) NOEXCEPT
 
     settled_.store(rows);
     signal_();
+    check_invariants_();
     return true;
 }
 
@@ -274,15 +282,20 @@ TEMPLATE
 template <size_t Column>
 bool CLASS::stage_() NOEXCEPT
 {
-    auto size = logical_.load();
-
-    // Cannot map empty file, and want minimum capacity, so expand as required.
+    // Provision the file in full (disk reserved, so allocation within cannot
+    // fail for space), but commit only what is in use; committing provisioned
+    // rows would charge that memory before anything is written.
     // disk_full: space is set but no code is set with false return.
-    if ((size < minimum_) && !resize_<Column>(size = minimum_))
+    const auto provision = to_provision();
+    if (!resize_<Column>(provision))
         return false;
 
-    // Reserve address space with generous multiple of capacity (costless).
-    const auto reserved = page_ceiling(to_width<Column>(to_reservation(size)));
+    const auto size = to_commitment();
+
+    // Reserve address space with generous multiple of capacity (costless), so
+    // that commitment growth never migrates the mapping (base is stable).
+    const auto reserved = page_ceiling(to_width<Column>(
+        to_reservation(provision)));
     const auto base = mmap_reserve(reserved);
 
     if (base == MAP_FAILED)
@@ -533,11 +546,22 @@ void CLASS::teardown_(const error::error_t& ec) NOEXCEPT
 TEMPLATE
 bool CLASS::advise_(uint8_t* map, size_t size) const NOEXCEPT
 {
-    const auto advice = random_ ? MADV_RANDOM : MADV_SEQUENTIAL;
+    // Advice is elective (normal is the kernel default) and configured from
+    // the read pattern (see database::advice); random_ is structural.
+    if (access_ == advice::normal)
+        return true;
+
+    // Order follows the advice enumeration.
+    static constexpr std::array<int, 3> advices
+    {
+        MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL
+    };
+
+    const auto behavior = advices.at(static_cast<uint8_t>(access_));
     for (size_t offset{}; offset < size; offset += advise_chunk)
     {
         const auto length = std::min(advise_chunk, size - offset);
-        if (::madvise(std::next(map, offset), length, advice) == fail)
+        if (::madvise(std::next(map, offset), length, behavior) == fail)
             return false;
     }
 
@@ -714,6 +738,34 @@ void CLASS::settler_run_() NOEXCEPT
         still = (top == mark) ? std::min(add1(still), idle_seconds) : zero;
         mark = top;
 
+#if defined(STAGING_TELEMETRY)
+        // The settler drains to the frontier while the throttle measures debt
+        // to logical, so a pinned frontier stops settling while debt grows
+        // (unbounded). lag exposes the pin, debt the throttle pressure. One
+        // string per line, as settler threads share the stream.
+        if (is_zero(++telemetry_ % telemetry_seconds))
+        {
+            using namespace system;
+            const auto settled = settled_.load();
+            const auto frontier = frontier_.load();
+            const auto [head_, size_] = unpack_word<uint64_t>(
+                window_.load(relaxed));
+
+            std::ostringstream line{};
+            line << "staging " << filenames_.front().filename().string()
+                << " logical=" << top
+                << " frontier=" << frontier
+                << " settled=" << settled
+                << " lag=" << floored_subtract(top, frontier)
+                << " debt=" << floored_subtract(top, settled)
+                << " window=" << size_
+                << " head=" << head_
+                << std::endl;
+
+            std::cerr << line.str() << std::flush;
+        }
+#endif
+
         auto bytes = backlog();
         if (is_zero(bytes))
             continue;