diff --git a/include/bitcoin/database/file/utilities.hpp b/include/bitcoin/database/file/utilities.hpp index c3ff3ed22..852bf1b51 100644 --- a/include/bitcoin/database/file/utilities.hpp +++ b/include/bitcoin/database/file/utilities.hpp @@ -21,6 +21,7 @@ #include #include +#include namespace libbitcoin { namespace database { @@ -71,9 +72,10 @@ BCD_API bool copy_directory(const path& from, const path& to) NOEXCEPT; BCD_API code copy_directory_ex(const path& from, const path& to) NOEXCEPT; /// File descriptor functions (for memory mapping). -BCD_API int open(const path& filename, bool random=true) NOEXCEPT; +BCD_API int open(const path& filename, bool random=true, + advice access=advice::normal) NOEXCEPT; BCD_API code open_ex(int& file_descriptor, const path& filename, - bool random=true) NOEXCEPT; + bool random=true, advice access=advice::normal) NOEXCEPT; BCD_API bool close(int file_descriptor) NOEXCEPT; BCD_API code close_ex(int file_descriptor) NOEXCEPT; BCD_API bool size(size_t& out, int file_descriptor) NOEXCEPT; diff --git a/include/bitcoin/database/impl/memory/mmap.ipp b/include/bitcoin/database/impl/memory/mmap.ipp index 1c77a4d80..3c248c06b 100644 --- a/include/bitcoin/database/impl/memory/mmap.ipp +++ b/include/bitcoin/database/impl/memory/mmap.ipp @@ -38,6 +38,7 @@ CLASS::mmap(const path& filename, const storage_settings& settings, : filenames_{ filename }, minimum_(to_rows(settings.size)), expansion_(settings.rate), + access_(settings.access), random_(random), staged_(staged), opened_{ file::invalid } @@ -51,6 +52,7 @@ CLASS::mmap(const paths& filenames, const storage_settings& settings, : filenames_(filenames), minimum_(to_rows(settings.size)), expansion_(settings.rate), + access_(settings.access), random_(random), staged_(staged), opened_{} @@ -104,6 +106,92 @@ size_t CLASS::to_capacity(size_t required) const NOEXCEPT return std::max(minimum_, ceilinged_add(required, growth)); } +// Commitment growth target for the capacity slow paths. Unlike to_capacity +// this never floors to the configured minimum: under lazy commitment that +// floor would commit the full provisioning on first growth (the load failure +// this design exists to prevent, moved from create to first touch). Growth is +// chunked to bound slow path frequency, and clamped so that small tables do +// not over-commit (the provisioned file requires no memory until committed). +TEMPLATE +size_t CLASS::to_growth(size_t required) const NOEXCEPT +{ +#if defined(MANAGE_STAGING) + using namespace system; + const auto expand = ceilinged_multiply(required, expansion_) / 100u; + const auto expanded = ceilinged_add(required, expand); + const auto chunked = std::max(expanded, + ceilinged_add(capacity_.load(), to_rows(commit_chunk))); + + return std::min(chunked, std::max(expanded, to_provision())); +#else + // The classic mapping is file-backed, so growth is capacity. + return to_capacity(required); +#endif +} + +// Disk provisioning: the configured minimum is a file reservation, ensuring +// that allocation within it cannot fail for space (disk full is detected at +// provisioning, where it remains recoverable). +TEMPLATE +size_t CLASS::to_provision() const NOEXCEPT +{ + return std::max(logical_.load(), minimum_); +} + +// Memory commitment follows use, not provisioning. Committing the configured +// minimum would demand that much memory (Linux charges the commit) before any +// row is written, failing load on any machine smaller than its store sizing. +// Growth commits in chunks within the standing reservation (no remap), so the +// cost is a syscall per chunk over the life of the store. +TEMPLATE +size_t CLASS::to_commitment() const NOEXCEPT +{ +#if defined(MANAGE_STAGING) + const auto logical = logical_.load(); + return std::min(to_provision(), std::max(logical, to_rows(commit_chunk))); +#else + // The classic mapping is file-backed, so commitment is provisioning. + return to_provision(); +#endif +} + +// The counter chain is the spine of the design (debug assertion only): +// +// settled_ <= frontier_ <= logical_ <= capacity_ <= file_ +// +// settled: rows durable and converted (staged) or drained (unstaged). +// frontier: completed-write prefix bound (extent ring floor). +// logical: rows claimed by writers (fast path CAS). +// capacity: rows claimable (committed memory). +// file: rows provisioned on disk (fallocate extent). +// +// Lower bounds are read first: the lock-free fast path can advance logical_ +// concurrently, so stale-low reads of lower bounds cannot falsify the chain, +// while upper bounds only grow under locks held at every call site. +TEMPLATE +void CLASS::check_invariants_() const NOEXCEPT +{ +#if !defined(NDEBUG) + if (!loaded_.load()) + return; + +#if defined(MANAGE_STAGING) + if (staged_) + { + const auto settled = settled_.load(); + const auto frontier = frontier_.load(); + BC_ASSERT(settled <= frontier); + BC_ASSERT(frontier <= logical_.load()); + } +#endif + + const auto logical = logical_.load(); + const auto capacity = capacity_.load(); + BC_ASSERT(logical <= capacity); + BC_ASSERT(capacity <= file_.load()); +#endif // NDEBUG +} + // Write-write protected by remap_mutex. TEMPLATE void CLASS::set_first_code(const error::error_t& ec) NOEXCEPT diff --git a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp index a38d6a3dd..94a202a4c 100644 --- a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp +++ b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp @@ -48,7 +48,7 @@ memory CLASS::get_filled(size_t offset, size_t size, const auto end = std::max(logical_.load(), offset + size); if (end > capacity_.load()) { - const auto extended = to_capacity(end); + const auto extended = to_growth(end); // TODO: Could loop over a try lock here and log deadlock warning. std::unique_lock remap_lock(remap_mutex_); diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 343c60c20..70be61bd7 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -67,10 +67,14 @@ bool CLASS::map_all_(std::index_sequence) NOEXCEPT if (!(map_() && ...)) { capacity_.store(zero); + file_.store(zero); return false; } - capacity_.store(std::max(logical_.load(), minimum_)); + // The file is provisioned in full; capacity publishes committed rows only. + file_.store(to_provision()); + capacity_.store(to_commitment()); + check_invariants_(); return true; } @@ -82,6 +86,9 @@ bool CLASS::unmap_all_(std::index_sequence) NOEXCEPT const auto success = (unmap_(capacity) && ...); capacity_.store(zero); + // Unmapping truncates the file to logical, which is then its provisioning. + file_.store(logical_.load()); + #if defined(MANAGE_STAGING) window_.store(zero); settled_.store(zero); @@ -104,7 +111,11 @@ bool CLASS::remap_all_(size_t capacity, std::index_sequence) NOEXCEPT return false; } + // Growth beyond the provisioned file extends it (resize_ is a no-op + // within), so the extent tracks the high water of provisioning. + file_.store(std::max(file_.load(), capacity)); capacity_.store(capacity); + check_invariants_(); return true; } @@ -240,11 +251,11 @@ bool CLASS::map_() NOEXCEPT #if defined(MANAGE_STAGING) return stage_(); #else - auto size = logical_.load(); - // Cannot map empty file, and want minimum capacity, so expand as required. + // The classic mapping is file-backed, so commitment is provisioning. // disk_full: space is set but no code is set with false return. - if ((size < minimum_) && !resize_(size = minimum_)) + const auto size = to_provision(); + if (!resize_(size)) return false; memory_map_[Column] = system::pointer_cast( @@ -300,8 +311,14 @@ TEMPLATE template bool CLASS::resize_(size_t size) NOEXCEPT { + // The file is provisioned ahead of commitment, so growth within the + // provisioned extent requires no disk operation (the space is reserved). + const auto extent = file_.load(); + if (size <= extent) + return true; + const auto target = to_width(size); - const auto capacity = to_width(capacity_.load()); + const auto capacity = to_width(extent); // Disk full detection, any other failure is an abort. #if !defined(WITHOUT_FALLOCATE) @@ -314,8 +331,8 @@ bool CLASS::resize_(size_t size) NOEXCEPT if (errno == ENOSPC) { using namespace system; - set_disk_space(ceilinged_multiply(floored_subtract(size, - capacity_.load()), stride)); + set_disk_space(ceilinged_multiply(floored_subtract(size, extent), + stride)); return false; } diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index a63d202d1..972166db3 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -20,8 +20,13 @@ #define LIBBITCOIN_DATABASE_MEMORY_MMAP_STAGING_IPP #include +#include #include #include +#if defined(STAGING_TELEMETRY) + #include + #include +#endif #include #include #include @@ -186,6 +191,7 @@ void CLASS::maintain_() NOEXCEPT window_.store(pack_word(head, size), release); frontier_.store(is_zero(size) ? logical_.load() : ring_.at(head).start.load(relaxed)); + check_invariants_(); } // Discard all extents, requires quiescent writers (locked). Any extent then @@ -200,6 +206,7 @@ void CLASS::discard_() NOEXCEPT window_.store(zero, release); frontier_.store(logical_.load()); + check_invariants_(); } // Delay the caller while staging exceeds its memory bound (write throttle). @@ -250,6 +257,7 @@ bool CLASS::settle_all_(size_t rows, std::index_sequence) NOEXCEPT settled_.store(rows); signal_(); + check_invariants_(); return true; } @@ -274,15 +282,20 @@ TEMPLATE template bool CLASS::stage_() NOEXCEPT { - auto size = logical_.load(); - - // Cannot map empty file, and want minimum capacity, so expand as required. + // Provision the file in full (disk reserved, so allocation within cannot + // fail for space), but commit only what is in use; committing provisioned + // rows would charge that memory before anything is written. // disk_full: space is set but no code is set with false return. - if ((size < minimum_) && !resize_(size = minimum_)) + const auto provision = to_provision(); + if (!resize_(provision)) return false; - // Reserve address space with generous multiple of capacity (costless). - const auto reserved = page_ceiling(to_width(to_reservation(size))); + const auto size = to_commitment(); + + // Reserve address space with generous multiple of capacity (costless), so + // that commitment growth never migrates the mapping (base is stable). + const auto reserved = page_ceiling(to_width( + to_reservation(provision))); const auto base = mmap_reserve(reserved); if (base == MAP_FAILED) @@ -533,11 +546,22 @@ void CLASS::teardown_(const error::error_t& ec) NOEXCEPT TEMPLATE bool CLASS::advise_(uint8_t* map, size_t size) const NOEXCEPT { - const auto advice = random_ ? MADV_RANDOM : MADV_SEQUENTIAL; + // Advice is elective (normal is the kernel default) and configured from + // the read pattern (see database::advice); random_ is structural. + if (access_ == advice::normal) + return true; + + // Order follows the advice enumeration. + static constexpr std::array advices + { + MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL + }; + + const auto behavior = advices.at(static_cast(access_)); for (size_t offset{}; offset < size; offset += advise_chunk) { const auto length = std::min(advise_chunk, size - offset); - if (::madvise(std::next(map, offset), length, advice) == fail) + if (::madvise(std::next(map, offset), length, behavior) == fail) return false; } @@ -714,6 +738,34 @@ void CLASS::settler_run_() NOEXCEPT still = (top == mark) ? std::min(add1(still), idle_seconds) : zero; mark = top; +#if defined(STAGING_TELEMETRY) + // The settler drains to the frontier while the throttle measures debt + // to logical, so a pinned frontier stops settling while debt grows + // (unbounded). lag exposes the pin, debt the throttle pressure. One + // string per line, as settler threads share the stream. + if (is_zero(++telemetry_ % telemetry_seconds)) + { + using namespace system; + const auto settled = settled_.load(); + const auto frontier = frontier_.load(); + const auto [head_, size_] = unpack_word( + window_.load(relaxed)); + + std::ostringstream line{}; + line << "staging " << filenames_.front().filename().string() + << " logical=" << top + << " frontier=" << frontier + << " settled=" << settled + << " lag=" << floored_subtract(top, frontier) + << " debt=" << floored_subtract(top, settled) + << " window=" << size_ + << " head=" << head_ + << std::endl; + + std::cerr << line.str() << std::flush; + } +#endif + auto bytes = backlog(); if (is_zero(bytes)) continue; diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index e0f0039f0..6c7d9b3b6 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -77,7 +77,7 @@ code CLASS::open() NOEXCEPT // Windows doesn't use madvise, instead infers map access from file open. for (size_t index{}; index < columns; ++index) if (const auto ec = file::open_ex(opened_.at(index), - filenames_.at(index), random_)) + filenames_.at(index), random_, access_)) return ec; // logical_ is the shared row count, derived from column 0's byte size. @@ -85,7 +85,9 @@ code CLASS::open() NOEXCEPT if (const auto ec = file::size_ex(bytes, opened_.front())) return ec; + // The file as opened is its own provisioning (extent tracks the file). logical_.store(logical_rows(bytes)); + file_.store(logical_rows(bytes)); return error::success; } @@ -238,6 +240,11 @@ code CLASS::flush() NOEXCEPT if (!loaded_.load()) return error::flush_unloaded; + // The suspend-writes contract implies remaining extents are complete + // or abandoned (as reload), so the ring is discarded to reconcile the + // frontier, which settling to the flushed top would otherwise pass. + discard_(); + if (!settle_all_(rows, sequence{})) return error::flush_failure; } @@ -402,6 +409,7 @@ bool CLASS::truncate(size_t count) NOEXCEPT #endif logical_.store(count); + check_invariants_(); return true; } @@ -418,7 +426,7 @@ bool CLASS::expand(size_t count) NOEXCEPT if (count > capacity_.load()) { - const auto extended = to_capacity(count); + const auto extended = to_growth(count); std::unique_lock remap_lock(remap_mutex_); if (!remap_all_(extended, sequence{})) @@ -446,7 +454,7 @@ bool CLASS::reserve(size_t count) NOEXCEPT const auto end = logical_.load() + count; if (end > capacity_.load()) { - const auto extended = to_capacity(end); + const auto extended = to_growth(end); std::unique_lock remap_lock(remap_mutex_); if (!remap_all_(extended, sequence{})) @@ -515,7 +523,7 @@ size_t CLASS::allocate(size_t count) NOEXCEPT continue; } - const auto extended = to_capacity(end); + const auto extended = to_growth(end); // TODO: Could loop over a try lock here and log deadlock warning. std::unique_lock remap_lock(remap_mutex_); diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index b8d286dfd..f8b89065d 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -198,6 +198,10 @@ class mmap } size_t to_capacity(size_t required) const NOEXCEPT; + size_t to_growth(size_t required) const NOEXCEPT; + size_t to_provision() const NOEXCEPT; + size_t to_commitment() const NOEXCEPT; + void check_invariants_() const NOEXCEPT; void set_first_code(const error::error_t& ec) NOEXCEPT; void set_disk_space(size_t required) NOEXCEPT; @@ -205,12 +209,16 @@ class mmap static constexpr size_t page_bound = to_bits(sizeof(uint64_t)); static constexpr size_t settle_chunk = system::power2(28u); static constexpr size_t advise_chunk = system::power2(30u); + static constexpr size_t commit_chunk = system::power2(28u); static constexpr size_t compress_factor = 32; static constexpr size_t throttle_factor = 8; static constexpr size_t active_factor = 32; static constexpr size_t urgent_factor = 4; static constexpr size_t idle_seconds = 60; static constexpr size_t headroom = 4; +#if defined(STAGING_TELEMETRY) + static constexpr size_t telemetry_seconds = 60; +#endif static constexpr auto fail = -1; static constexpr auto relaxed = std::memory_order_relaxed; static constexpr auto release = std::memory_order_release; @@ -293,6 +301,7 @@ class mmap const paths filenames_; const size_t minimum_; const size_t expansion_; + const advice access_; const bool random_; const bool staged_; @@ -300,6 +309,7 @@ class mmap std::atomic error_{ error::success }; std::atomic space_{ zero }; std::atomic capacity_{}; + std::atomic file_{}; std::atomic logical_{}; std::atomic_bool fault_{}; std::atomic_bool loaded_{}; @@ -327,6 +337,11 @@ class mmap std::atomic outstanding; }; +#if defined(STAGING_TELEMETRY) + // This is unshared (settler thread only). + size_t telemetry_{}; +#endif + // These are thread safe (atomic). std::atomic marks_{}; std::atomic settled_{}; diff --git a/include/bitcoin/database/memory/settings.hpp b/include/bitcoin/database/memory/settings.hpp index 64bf2b94e..679048072 100644 --- a/include/bitcoin/database/memory/settings.hpp +++ b/include/bitcoin/database/memory/settings.hpp @@ -24,6 +24,22 @@ namespace libbitcoin { namespace database { +/// Expected read pattern of mapped storage, guiding kernel page advice. This +/// is independent of the write pattern (structural): bodies are appended +/// sequentially but read randomly by validation, so advising the kernel from +/// the write pattern invites eviction of the read set under memory pressure. +enum class advice : uint8_t +{ + /// Let the operating system decide (mixed or unpredictable access). + normal, + + /// Random access (suppresses read ahead). + random, + + /// One pass access (allows the kernel to free pages behind). + sequential +}; + /// Storage tuning consumed by memory map construction: the base of table /// configuration, as network settings bases are derived by server services. /// Future staging tunables land here without constructor signature changes. @@ -34,6 +50,9 @@ struct storage_settings /// Body expansion rate (percentage). uint16_t rate{ 5 }; + + /// Page advice for mapped reads (see advice). + advice access{ advice::normal }; }; } // namespace database diff --git a/include/bitcoin/database/store.hpp b/include/bitcoin/database/store.hpp index 4ef68cebf..3b08b020a 100644 --- a/include/bitcoin/database/store.hpp +++ b/include/bitcoin/database/store.hpp @@ -235,8 +235,9 @@ class store static constexpr bool sequential = false; // Heads are minimally allocated with no expansion (heads size to their - // configured buckets at table creation). - static constexpr storage_settings head_settings{ 1, 0 }; + // configured buckets at table creation) and randomly probed (the advice + // preserves optimal classic mapped reads; staged heads are anonymous). + static constexpr storage_settings head_settings{ 1, 0, advice::random }; // Bodies are append-only, so stage writes in anonymous memory where the // staging backend is built (heads update in place and remain resident). diff --git a/src/file/utilities.cpp b/src/file/utilities.cpp index a1479b827..c4c1fefea 100644 --- a/src/file/utilities.cpp +++ b/src/file/utilities.cpp @@ -248,7 +248,15 @@ code copy_directory_ex(const path& from, const path& to) NOEXCEPT #define MSC_OR_NOAPPLE(parameter) #endif -int open(const path& filename, bool MSC_OR_NOAPPLE(random)) NOEXCEPT +// Page advice is applied at open only where posix_fadvise is available. +#if !defined(HAVE_MSC) && !defined(HAVE_APPLE) + #define POSIX_ONLY(parameter) parameter +#else + #define POSIX_ONLY(parameter) +#endif + +int open(const path& filename, bool MSC_OR_NOAPPLE(random), + advice POSIX_ONLY(access)) NOEXCEPT { const auto path = system::extended_path(filename); int file_descriptor{}; @@ -264,28 +272,37 @@ int open(const path& filename, bool MSC_OR_NOAPPLE(random)) NOEXCEPT file_descriptor = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR); #if !defined(HAVE_APPLE) - if (file_descriptor != -1) + // Advice is elective (normal is the kernel default) and configured from + // the read pattern (see database::advice); random is structural. + if ((file_descriptor != -1) && (access != advice::normal)) { - // _O_RANDOM equivalent, posix_fadvise returns error on failure. - const auto advice = random ? POSIX_FADV_RANDOM : POSIX_FADV_SEQUENTIAL; - const auto result = ::posix_fadvise(file_descriptor, 0, 0, advice); + // Order follows the advice enumeration. + static constexpr std::array advices + { + POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL + }; + + // posix_fadvise returns error on failure. + const auto result = ::posix_fadvise(file_descriptor, 0, 0, + advices.at(static_cast(access))); if (!is_zero(result)) { close(file_descriptor); file_descriptor = -1; errno = result; } - else + } + + if (file_descriptor != -1) + { + // _SH_DENYWR equivalent. + const struct flock lock{ F_WRLCK, SEEK_SET, 0, 0 }; + if (::fcntl(file_descriptor, F_SETLK, &lock) == -1) { - // _SH_DENYWR equivalent. - const struct flock lock{ F_WRLCK, SEEK_SET, 0, 0 }; - if (::fcntl(file_descriptor, F_SETLK, &lock) == -1) - { - const auto last = errno; - close(file_descriptor); - file_descriptor = -1; - errno = last; - } + const auto last = errno; + close(file_descriptor); + file_descriptor = -1; + errno = last; } } #endif // !HAVE_APPLE @@ -294,10 +311,11 @@ int open(const path& filename, bool MSC_OR_NOAPPLE(random)) NOEXCEPT return file_descriptor; } -code open_ex(int& file_descriptor, const path& filename, bool random) NOEXCEPT +code open_ex(int& file_descriptor, const path& filename, bool random, + advice access) NOEXCEPT { system::error::clear_errno(); - file_descriptor = open(filename, random); + file_descriptor = open(filename, random, access); return system::error::get_errno(); }