From 5b0f250bdaf1b84f64127f56ba51e3af3e3b6c88 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Wed, 19 Aug 2026 00:02:54 -0400 Subject: [PATCH 1/2] Write head fill to file at create and load heads released (lazy). --- .../database/impl/memory/mmap_staging.ipp | 150 +++++++++++++++++- .../database/impl/memory/mmap_storage.ipp | 27 ++++ .../database/impl/primitives/hashhead.ipp | 18 +-- .../database/memory/interfaces/storage.hpp | 4 + include/bitcoin/database/memory/mmap.hpp | 6 + include/bitcoin/database/memory/mstage.hpp | 3 + src/memory/mstage.cpp | 5 + 7 files changed, 196 insertions(+), 17 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index d6b178a36..6558274ce 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -404,6 +404,22 @@ bool CLASS::stage_() NOEXCEPT return true; } + // Managed heads load released: the file holds the content and pages + // restore to anonymous per segment on first write, so load contributes + // no residency (an eager head population is the store's largest + // memory transient, the full head set at open). + if (!staged_ && dirty_) + { + if (!lazy_install_()) + { + teardown_(error::mmap_failure); + return false; + } + + loaded_.store(true); + return true; + } + // Commit anonymous pages above the settle boundary page floor. const auto settled = page_floor(to_width(settled_.load())); @@ -451,6 +467,133 @@ bool CLASS::stage_() NOEXCEPT return true; } +// Install the managed head lazily over its current logical span: a released +// (read-only file) full-page prefix restored to anonymous per segment on +// first write, an anonymous populated tail, and clean page tracking sized to +// a replacement reservation. Caller holds exclusive remap (or is loading). +TEMPLATE +bool CLASS::lazy_install_() NOEXCEPT +{ + using namespace system; + const auto rows = logical_.load(); + const auto logical = to_width(rows); + + // Replace any standing reservation (sized as stage_). + if (!is_null(memory_map_[zero])) + mmap_unreserve(memory_map_[zero], reserved_[zero]); + + const auto reserved = page_ceiling(to_width( + to_reservation(to_provision()))); + const auto base = mmap_reserve(reserved); + if (base == MAP_FAILED) + { + set_first_code(error::mmap_failure); + return false; + } + + memory_map_[zero] = pointer_cast(base); + reserved_[zero] = reserved; + + // Rebuild page tracking for the new reservation (value-initialized). + const auto pages = ceilinged_divide(reserved, page_); + words_ = ceilinged_divide(pages, page_bound); + dirty_ = std::make_unique(words_); + intent_ = std::make_unique(words_); + released_ = std::make_unique(words_); + sweep_ = std::make_unique(words_); + writers_.store(zero); + + // Released file prefix (full pages below logical). + const auto floor = page_floor(logical); + if (is_nonzero(floor) && (mmap_settle(memory_map_[zero], floor, + opened_[zero], zero) == fail)) + { + set_first_code(error::mmap_failure); + return false; + } + + // Commit the remainder to the commitment target and populate the tail. + const auto target = std::max(page_ceiling(to_width( + capacity_.load())), page_ceiling(logical)); + if ((target > floor) && (mmap_commit(std::next(memory_map_[zero], floor), + target - floor, headroom_) == fail)) + { + set_first_code(error::mmap_failure); + return false; + } + + if ((logical > floor) && !pread_all(opened_[zero], + std::next(memory_map_[zero], floor), logical - floor, floor)) + { + set_first_code(error::fsync_failure); + return false; + } + + // Declare the prefix released and engage the restore protocol. + const auto flags = floor / page_; + for (size_t word{}; word < ceilinged_divide(flags, page_bound); ++word) + { + const auto first = word * page_bound; + released_[word].store((flags >= (first + page_bound)) ? + bit_all : unmask_right(flags - first)); + } + + engaged_.store(true); + + // Attribute the anonymous span for diagnostics (smaps decomposition). + mmap_name(std::next(memory_map_[zero], floor), reserved - floor, + filenames_.front().filename().string().c_str()); + + return true; +} + +// Head-creation fill: the fill is file content, written sequentially to the +// file (page cache, no mapping involvement) and installed released, so +// creation contributes no memory residency (an in-memory backfill of the +// head set is the store's largest allocation transient). +TEMPLATE +size_t CLASS::allocate_filled_(size_t count, uint8_t backfill) NOEXCEPT +{ + BC_ASSERT(!staged_ && dirty_); + std::unique_lock field_lock(field_mutex_); + std::unique_lock map_lock(remap_mutex_); + + using namespace system; + const auto start = logical_.load(); + if (!loaded_.load() || fault_.load() || is_add_overflow(start, count)) + return storage::eof; + + // Provision the file physically (disk full detected here). + const auto end = start + count; + if (!resize_(end)) + return storage::eof; + + // Write the fill. + const auto from = to_width(start); + const auto to = to_width(end); + std::vector chunk(std::min(to - from, release_chunk), backfill); + for (auto at = from; at < to; ) + { + const auto size = std::min(to - at, chunk.size()); + if (!pwrite_all(opened_[zero], chunk.data(), size, at)) + { + set_first_code(error::fsync_failure); + return storage::eof; + } + + at += size; + } + + logical_.store(end); + file_.store(std::max(file_.load(), end)); + capacity_.store(std::max(capacity_.load(), end)); + if (!lazy_install_()) + return storage::eof; + + check_invariants_(); + return start; +} + // Commit failure results in unmapped when final (the default); a non-final // refusal (in-reservation commit, replacement reservation, or replacement // commit) returns false with the standing mapping untouched, so the caller @@ -1198,8 +1341,11 @@ void CLASS::head_run_() NOEXCEPT // Quiet is assured here (hot scarcity skipped above, and idle // draining implies sixty still seconds). - if (engaged && !release_pages_()) - continue; + // Lazy head load engages restore on all platforms; the release + // sweep remains a darwin response (see head_release). + if constexpr (head_release) + if (engaged && !release_pages_()) + continue; } transferred = top; diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index d757cd0a3..649f03a11 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -636,6 +636,33 @@ size_t CLASS::allocate(size_t count) NOEXCEPT } } +// Backfilled allocation (head creation). A managed head writes the fill to +// its file and maps it released, so the fill is never memory-resident; a +// native mapping fills through memory (file-backed, kernel writeback). +TEMPLATE +size_t CLASS::allocate(size_t count, uint8_t backfill) NOEXCEPT +{ +#if defined(MANAGE_STAGING) + if (!staged_ && dirty_) + return allocate_filled_(count, backfill); +#endif + + const auto start = allocate(count); + if (start == storage::eof) + return start; + + const auto offset = to_width(start); + const auto size = to_width(count); + const auto ptr = get(offset); + if (!ptr) + return storage::eof; + + prepare(offset, size); + std::fill_n(ptr.data(), size, backfill); + mark(offset, size); + return start; +} + } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/impl/primitives/hashhead.ipp b/include/bitcoin/database/impl/primitives/hashhead.ipp index d546e399e..2ee09ac6d 100644 --- a/include/bitcoin/database/impl/primitives/hashhead.ipp +++ b/include/bitcoin/database/impl/primitives/hashhead.ipp @@ -57,24 +57,12 @@ bool CLASS::create() NOEXCEPT if (is_nonzero(file_.size())) return false; - const auto allocation = size(); - const auto start = file_.allocate(allocation); - - // Guards addition overflow in file_.get (start must be valid). - if (start == storage::eof) - return false; - - const auto ptr = file_.get(start); - if (!ptr) + // The fill is file content: a managed head writes it to the file and + // maps it released, so creation contributes no memory residency. + if (file_.allocate(size(), system::bit_all) == storage::eof) return false; BC_ASSERT_MSG(verify(), "unexpected head size"); - - // std::memset/fill_n have identical performance (on win32). - ////std::memset(ptr.data(), system::bit_all, allocation); - file_.prepare(start, allocation); - std::fill_n(ptr.data(), allocation, system::bit_all); - file_.mark(start, allocation); return set_body_count(zero); } diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index 541fd81b4..33fe183fb 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -100,6 +100,10 @@ class storage /// Increase logical by specified rows/bytes, return row of first (or eof). virtual size_t allocate(size_t count) NOEXCEPT = 0; + /// Allocate backfilled rows/bytes, avoiding memory residency of the fill + /// where the backend supports it (head creation path). + virtual size_t allocate(size_t count, uint8_t backfill) NOEXCEPT = 0; + /// Report element write completion of count rows/bytes at offset. virtual void complete(size_t offset, size_t count) NOEXCEPT = 0; diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index df161830f..03fc56f92 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -156,6 +156,10 @@ class mmap /// Increase logical by specified rows/bytes, return row of first (or eof). size_t allocate(size_t count) NOEXCEPT override; + /// Allocate backfilled rows/bytes; managed heads write the fill to the + /// file and map it released (no memory residency of the fill). + size_t allocate(size_t count, uint8_t backfill) NOEXCEPT override; + /// Report element write completion of count rows/bytes at offset. void complete(size_t offset, size_t count) NOEXCEPT override; @@ -327,6 +331,8 @@ class mmap // staging utilities, not thread safe (claim_ is lock-free thread safe). struct extent; + size_t allocate_filled_(size_t count, uint8_t backfill) NOEXCEPT; + bool lazy_install_() NOEXCEPT; size_t record_(size_t count) NOEXCEPT; bool claim_(extent& record, size_t count) NOEXCEPT; void maintain_() NOEXCEPT; diff --git a/include/bitcoin/database/memory/mstage.hpp b/include/bitcoin/database/memory/mstage.hpp index ac0269e97..7022897f5 100644 --- a/include/bitcoin/database/memory/mstage.hpp +++ b/include/bitcoin/database/memory/mstage.hpp @@ -39,6 +39,9 @@ /// Reserve inaccessible anonymous address space (MAP_FAILED on failure). void* mmap_reserve(size_t size) NOEXCEPT; +/// Release reserved address space (with any mappings installed within it). +int mmap_unreserve(void* address, size_t size) NOEXCEPT; + /// Commit reserved pages as readable/writable anonymous memory. int mmap_commit(void* address, size_t size, size_t headroom) NOEXCEPT; diff --git a/src/memory/mstage.cpp b/src/memory/mstage.cpp index 45e787f4d..876fbc796 100644 --- a/src/memory/mstage.cpp +++ b/src/memory/mstage.cpp @@ -48,6 +48,11 @@ void* mmap_reserve(size_t size) NOEXCEPT -1, 0); } +int mmap_unreserve(void* address, size_t size) NOEXCEPT +{ + return ::munmap(address, size); +} + #if defined(HAVE_APPLE) // Darwin admits every anonymous ask (exhaustion arrives at first touch), so From 6d5a866b3ed9cb3c8c8ca9399cfd79887237a0c0 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Wed, 19 Aug 2026 01:39:47 -0400 Subject: [PATCH 2/2] Add backfilled allocate overload to test storage mock. --- test/mocks/chunk_storage.hpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/mocks/chunk_storage.hpp b/test/mocks/chunk_storage.hpp index db87d72c0..7937201cc 100644 --- a/test/mocks/chunk_storage.hpp +++ b/test/mocks/chunk_storage.hpp @@ -240,6 +240,28 @@ class chunk_storages return link; } + size_t allocate(size_t count, uint8_t backfill) NOEXCEPT override + { + std::unique_lock field_lock(field_mutex_); + if (system::is_add_overflow(logical_, count)) + return chunk_storages::eof; + + const auto end = logical_ + count; + for (size_t column{}; column < columns; ++column) + if (to_capacity(end, column) > at(column).max_size()) + return chunk_storages::eof; + + std::unique_lock map_lock(map_mutex_); + const auto link = logical_; + + logical_ = end; + for (size_t column{}; column < columns; ++column) + if (to_capacity(logical_, column) > at(column).size()) + at(column).resize(to_capacity(logical_, column), backfill); + + return link; + } + void complete(size_t, size_t) NOEXCEPT override { }