diff --git a/include/bitcoin/database/impl/memory/mmap.ipp b/include/bitcoin/database/impl/memory/mmap.ipp index 057942357..a22ed8186 100644 --- a/include/bitcoin/database/impl/memory/mmap.ipp +++ b/include/bitcoin/database/impl/memory/mmap.ipp @@ -43,6 +43,7 @@ CLASS::mmap(const path& filename, const storage_settings& settings, expansion_(settings.rate), headroom_(system::possible_narrow_cast(settings.headroom)), access_(settings.access), + zeroed_(settings.zeroed), random_(random), staged_(staged), managed_(!staged && !head_shared), @@ -59,6 +60,7 @@ CLASS::mmap(const paths& filenames, const storage_settings& settings, expansion_(settings.rate), headroom_(system::possible_narrow_cast(settings.headroom)), access_(settings.access), + zeroed_(settings.zeroed), random_(random), staged_(staged), managed_(false), diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index 73a49d50f..401009fe0 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -352,7 +352,7 @@ bool CLASS::truncate(size_t count) NOEXCEPT #endif // Released rows are zeroed, so an allocation is zero until written. - if (loaded_.load()) + if (zeroed_ && loaded_.load()) zero_all_(count, logical, fresh, sequence{}); logical_.store(count); diff --git a/include/bitcoin/database/impl/primitives/headmap.ipp b/include/bitcoin/database/impl/primitives/headmap.ipp index 81370579e..2b6c159bc 100644 --- a/include/bitcoin/database/impl/primitives/headmap.ipp +++ b/include/bitcoin/database/impl/primitives/headmap.ipp @@ -140,8 +140,6 @@ memory CLASS::get_memory() const NOEXCEPT TEMPLATE Value CLASS::at(size_t index) const NOEXCEPT { - using namespace system; - // Buckets at or above the logical size are unallocated (count publishes). if (index >= count()) return {}; @@ -150,24 +148,7 @@ Value CLASS::at(size_t index) const NOEXCEPT if (!ptr) return {}; - const auto raw = ptr.data(); - if constexpr (aligned) - { - // Reads full padded word (masked by to_value). - const auto& head = *pointer_cast>(raw); - return to_value(head.load(load_order)); - } - else - { - const auto& head = cell_array(raw); - cell value{}; - - mutex_.lock_shared(); - cell_array(value) = head; - mutex_.unlock_shared(); - - return to_value(value); - } + return load(ptr.data()); } TEMPLATE @@ -183,6 +164,29 @@ Value CLASS::at(const memory& ptr, size_t index) const NOEXCEPT if (is_null(raw)) return {}; + return load(raw); +} + +TEMPLATE +Value CLASS::top() const NOEXCEPT +{ + static_assert(is_link); + + const auto size = count(); + if (is_zero(size)) + return {}; + + const auto ptr = file_.get(link_to_position(sub1(size))); + if (!ptr) + return {}; + + return load(ptr.data()); +} + +TEMPLATE +Value CLASS::load(memory::iterator raw) const NOEXCEPT +{ + using namespace system; if constexpr (aligned) { // Reads full padded word (masked by to_value). diff --git a/include/bitcoin/database/impl/query/confirmed.ipp b/include/bitcoin/database/impl/query/confirmed.ipp index 294a53544..1bb341575 100644 --- a/include/bitcoin/database/impl/query/confirmed.ipp +++ b/include/bitcoin/database/impl/query/confirmed.ipp @@ -85,7 +85,7 @@ TEMPLATE bool CLASS::is_candidate_extension(const header_link& link) const NOEXCEPT { return !is_associated(link) && is_candidate_header(link) && - (to_confirmed(get_top_confirmed()) == to_parent(link)); + (to_top_confirmed() == to_parent(link)); } TEMPLATE diff --git a/include/bitcoin/database/impl/query/initialize.ipp b/include/bitcoin/database/impl/query/initialize.ipp index 0355f01a4..b2fa4186b 100644 --- a/include/bitcoin/database/impl/query/initialize.ipp +++ b/include/bitcoin/database/impl/query/initialize.ipp @@ -37,14 +37,14 @@ const database::envelope& CLASS::envelope() const NOEXCEPT TEMPLATE hash_digest CLASS::get_top_confirmed_hash() const NOEXCEPT { - return get_header_key(to_confirmed(get_top_confirmed())); + return get_header_key(to_top_confirmed()); } // server/dumps TEMPLATE hash_digest CLASS::get_top_candidate_hash() const NOEXCEPT { - return get_header_key(to_candidate(get_top_candidate())); + return get_header_key(to_top_candidate()); } // node start diff --git a/include/bitcoin/database/impl/query/navigate/navigate_natural.ipp b/include/bitcoin/database/impl/query/navigate/navigate_natural.ipp index 5a6c4b3bc..83d6f9bfd 100644 --- a/include/bitcoin/database/impl/query/navigate/navigate_natural.ipp +++ b/include/bitcoin/database/impl/query/navigate/navigate_natural.ipp @@ -39,6 +39,18 @@ inline header_link CLASS::to_confirmed(size_t height) const NOEXCEPT return store_.confirmed.at(height); } +TEMPLATE +inline header_link CLASS::to_top_candidate() const NOEXCEPT +{ + return store_.candidate.top(); +} + +TEMPLATE +inline header_link CLASS::to_top_confirmed() const NOEXCEPT +{ + return store_.confirmed.top(); +} + TEMPLATE inline header_link CLASS::to_header(const hash_digest& key) const NOEXCEPT { diff --git a/include/bitcoin/database/impl/query/properties_block.ipp b/include/bitcoin/database/impl/query/properties_block.ipp index 34b4b9946..2e3120286 100644 --- a/include/bitcoin/database/impl/query/properties_block.ipp +++ b/include/bitcoin/database/impl/query/properties_block.ipp @@ -78,8 +78,7 @@ inline bool CLASS::is_associated(const header_link& link) const NOEXCEPT TEMPLATE uint32_t CLASS::get_top_timestamp(bool confirmed) const NOEXCEPT { - const auto top = confirmed ? to_confirmed(get_top_confirmed()) : - to_candidate(get_top_candidate()); + const auto top = confirmed ? to_top_confirmed() : to_top_candidate(); // returns zero if read fails. uint32_t timestamp{}; diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index f30a54e0a..901e24d2f 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -430,6 +430,7 @@ class mmap const size_t expansion_; const size_t headroom_; const advice access_; + const bool zeroed_; const bool random_; const bool staged_; const bool managed_; diff --git a/include/bitcoin/database/memory/settings.hpp b/include/bitcoin/database/memory/settings.hpp index ca152f7c0..48ca28375 100644 --- a/include/bitcoin/database/memory/settings.hpp +++ b/include/bitcoin/database/memory/settings.hpp @@ -53,6 +53,12 @@ struct storage_settings /// KB/t to 4.0 (one page per fault), bandwidth to ~250 MB/s, and the /// validation rate from 648 to 720 blk/min at the same heights. advice access{ advice::scattered }; + + /// Rows released by truncate are zeroed, so a reallocation reads zero + /// until written. Heads override to retain at construction: a cell once + /// within the count holds its last store until the next, so a reader + /// that took that count reads a valid (possibly just-popped) link. + bool zeroed{ true }; }; } // namespace database diff --git a/include/bitcoin/database/primitives/headmap.hpp b/include/bitcoin/database/primitives/headmap.hpp index d61180323..9b89cd814 100644 --- a/include/bitcoin/database/primitives/headmap.hpp +++ b/include/bitcoin/database/primitives/headmap.hpp @@ -99,6 +99,9 @@ class headmap /// Return value at index using accessor, default if not allocated. Value at(const memory& ptr, size_t index) const NOEXCEPT; + /// Return link at the top index, terminal if empty or unloaded. + Value top() const NOEXCEPT; + /// Append link at count into reserved capacity (single writer). bool push(const Link& link) NOEXCEPT; @@ -183,6 +186,8 @@ class headmap floored_divide(position, bucket_size)); } + Value load(memory::iterator raw) const NOEXCEPT; + // These are thread safe. storage& file_; mutable shared_mutex mutex_{}; diff --git a/include/bitcoin/database/query.hpp b/include/bitcoin/database/query.hpp index 8b321d8f8..1984d6d4b 100644 --- a/include/bitcoin/database/query.hpp +++ b/include/bitcoin/database/query.hpp @@ -329,6 +329,8 @@ class query /// search key (entry) inline header_link to_candidate(size_t height) const NOEXCEPT; inline header_link to_confirmed(size_t height) const NOEXCEPT; + inline header_link to_top_candidate() const NOEXCEPT; + inline header_link to_top_confirmed() const NOEXCEPT; inline header_link to_header(const hash_digest& key) const NOEXCEPT; inline tx_link to_tx(const hash_digest& key) const NOEXCEPT; inline tx_link to_witness_tx(const hash_digest& key) const NOEXCEPT; diff --git a/include/bitcoin/database/store.hpp b/include/bitcoin/database/store.hpp index adc40d8a2..cc9648165 100644 --- a/include/bitcoin/database/store.hpp +++ b/include/bitcoin/database/store.hpp @@ -321,7 +321,8 @@ class store { .size = std::max(one, minimum), .rate = table.rate, - .access = advice::random + .access = advice::random, + .zeroed = false }; } diff --git a/test/memory/mmap.cpp b/test/memory/mmap.cpp index 3eeecdf44..28edec77e 100644 --- a/test/memory/mmap.cpp +++ b/test/memory/mmap.cpp @@ -1129,6 +1129,44 @@ BOOST_AUTO_TEST_CASE(mmap__truncate__reallocate__zeroed) BOOST_REQUIRE(!instance.get_fault()); } +BOOST_AUTO_TEST_CASE(mmap__truncate__unzeroed_reallocate__retained) +{ + constexpr size_t initial = 12'000; + constexpr size_t retained = 5'000; + constexpr size_t appended = 4'000; + const std::string file = TEST_PATH; + BOOST_REQUIRE(test::create(file)); + + map instance(file, { .size = 1, .rate = 50, .zeroed = false }); + BOOST_REQUIRE(!instance.open()); + BOOST_REQUIRE(!instance.load()); + + const auto original = stage_vector(stage_repattern, initial, one); + auto memory = instance.get(instance.allocate(initial)); + BOOST_REQUIRE(memory); + std::copy_n(original.begin(), original.size(), memory.begin()); + + memory.reset(); + BOOST_REQUIRE(instance.truncate(retained)); + memory = instance.get_capacity(retained); + BOOST_REQUIRE(memory); + + const auto released = std::next(original.begin(), retained); + const auto released_end = std::next(released, appended); + BOOST_REQUIRE_EQUAL_COLLECTIONS(memory.begin(), std::next(memory.begin(), appended), released, released_end); + + memory.reset(); + BOOST_REQUIRE_EQUAL(instance.allocate(appended), retained); + memory = instance.get(retained); + BOOST_REQUIRE(memory); + BOOST_REQUIRE_EQUAL_COLLECTIONS(memory.begin(), std::next(memory.begin(), appended), released, released_end); + + memory.reset(); + BOOST_REQUIRE(!instance.unload()); + BOOST_REQUIRE(!instance.close()); + BOOST_REQUIRE(!instance.get_fault()); +} + BOOST_AUTO_TEST_CASE(mmap__staged__truncate_below_flush_reallocate__zeroed) { constexpr size_t initial = 12'000; diff --git a/test/primitives/headmap.cpp b/test/primitives/headmap.cpp index 669e8757c..7f565feaa 100644 --- a/test/primitives/headmap.cpp +++ b/test/primitives/headmap.cpp @@ -32,6 +32,23 @@ using slab_headmap = headmap; using value_link = linkage<3>; using value_headmap = headmap; +// Reports a stale size once, as a reader racing a truncate observes it. +class stale_storage + : public test::chunk_storage +{ +public: + using test::chunk_storage::chunk_storage; + + size_t size() const NOEXCEPT override + { + const auto stale = stale_; + stale_ = zero; + return is_zero(stale) ? test::chunk_storage::size() : stale; + } + + mutable size_t stale_{}; +}; + class little_slab { public: @@ -209,6 +226,75 @@ BOOST_AUTO_TEST_CASE(headmap__truncate__push__rewrites_bucket) BOOST_REQUIRE_EQUAL(instance.at(1), 0x5566_u16); } +BOOST_AUTO_TEST_CASE(headmap__top__empty__terminal) +{ + data_chunk head_file{}; + test::chunk_storage head_store{ head_file }; + test_headmap instance{ head_store }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.top().is_terminal()); +} + +BOOST_AUTO_TEST_CASE(headmap__top__single_genesis__zero) +{ + data_chunk head_file{}; + test::chunk_storage head_store{ head_file }; + test_headmap instance{ head_store }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.reserve(1u)); + BOOST_REQUIRE(instance.push(0x0000_u16)); + BOOST_REQUIRE_EQUAL(instance.top(), 0x0000_u16); +} + +BOOST_AUTO_TEST_CASE(headmap__top__many__last_pushed) +{ + data_chunk head_file{}; + test::chunk_storage head_store{ head_file }; + test_headmap instance{ head_store }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.reserve(3u)); + BOOST_REQUIRE(instance.push(0x1122_u16)); + BOOST_REQUIRE(instance.push(0x3344_u16)); + BOOST_REQUIRE(instance.push(0x5566_u16)); + BOOST_REQUIRE_EQUAL(instance.count(), 3u); + BOOST_REQUIRE_EQUAL(instance.top(), 0x5566_u16); +} + +BOOST_AUTO_TEST_CASE(headmap__top__truncated__expected) +{ + data_chunk head_file{}; + test::chunk_storage head_store{ head_file }; + test_headmap instance{ head_store }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.reserve(3u)); + BOOST_REQUIRE(instance.push(0x1122_u16)); + BOOST_REQUIRE(instance.push(0x3344_u16)); + BOOST_REQUIRE(instance.push(0x5566_u16)); + BOOST_REQUIRE(instance.truncate(2u)); + BOOST_REQUIRE_EQUAL(instance.top(), 0x3344_u16); + BOOST_REQUIRE(instance.truncate(1u)); + BOOST_REQUIRE_EQUAL(instance.top(), 0x1122_u16); + BOOST_REQUIRE(instance.truncate(0u)); + BOOST_REQUIRE(instance.top().is_terminal()); +} + +BOOST_AUTO_TEST_CASE(headmap__top__truncate_race__popped_link) +{ + data_chunk head_file{}; + stale_storage head_store{ head_file }; + test_headmap instance{ head_store }; + BOOST_REQUIRE(instance.create()); + BOOST_REQUIRE(instance.reserve(3u)); + BOOST_REQUIRE(instance.push(0x1122_u16)); + BOOST_REQUIRE(instance.push(0x3344_u16)); + BOOST_REQUIRE(instance.push(0x5566_u16)); + BOOST_REQUIRE(instance.truncate(2u)); + head_store.stale_ = 6u; + BOOST_REQUIRE_EQUAL(instance.top(), 0x5566_u16); + BOOST_REQUIRE_EQUAL(instance.top(), 0x3344_u16); + BOOST_REQUIRE_EQUAL(instance.count(), 2u); +} + BOOST_AUTO_TEST_CASE(headmap__unaligned__push_at_truncate__expected) { data_chunk head_file{}; diff --git a/test/query/navigate/navigate_natural.cpp b/test/query/navigate/navigate_natural.cpp index 67cbe905c..e590552b6 100644 --- a/test/query/navigate/navigate_natural.cpp +++ b/test/query/navigate/navigate_natural.cpp @@ -102,6 +102,58 @@ BOOST_AUTO_TEST_CASE(query_navigate__to_confirmed__always__expected) BOOST_REQUIRE_EQUAL(query.to_candidate(4), header_link::terminal); } +// to_top_candidate + +BOOST_AUTO_TEST_CASE(query_navigate__to_top_candidate__push_pop__expected) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1, test::context, {}, false, false)); + BOOST_REQUIRE(query.set(test::block2, test::context, {}, false, false)); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 0u); + BOOST_REQUIRE(query.push_candidate(1)); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 1u); + BOOST_REQUIRE(query.push_candidate(2)); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 2u); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 0u); + BOOST_REQUIRE(query.pop_candidate()); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 1u); + BOOST_REQUIRE(query.pop_candidate()); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 0u); + BOOST_REQUIRE(!query.pop_candidate()); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 0u); +} + +// to_top_confirmed + +BOOST_AUTO_TEST_CASE(query_navigate__to_top_confirmed__push_pop__expected) +{ + settings settings{}; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE(!store.create(test::events_handler)); + BOOST_REQUIRE(query.initialize(test::genesis)); + BOOST_REQUIRE(query.set(test::block1, test::context, {}, false, false)); + BOOST_REQUIRE(query.set(test::block2, test::context, {}, false, false)); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 0u); + BOOST_REQUIRE(query.push_confirmed(1, false)); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 1u); + BOOST_REQUIRE(query.push_confirmed(2, false)); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 2u); + BOOST_REQUIRE_EQUAL(query.to_top_candidate(), 0u); + BOOST_REQUIRE(query.pop_confirmed()); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 1u); + BOOST_REQUIRE(query.pop_confirmed()); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 0u); + BOOST_REQUIRE(!query.pop_confirmed()); + BOOST_REQUIRE_EQUAL(query.to_top_confirmed(), 0u); +} + // to_header BOOST_AUTO_TEST_CASE(query_navigate__to_header__always__expected)