Add top link accessors to height indexes.

include/bitcoin/database/impl/memory/mmap.ipp

@@ -43,6 +43,7 @@ CLASS::mmap(const path& filename, const storage_settings& settings,
     expansion_(settings.rate),
     headroom_(system::possible_narrow_cast<size_t>(settings.headroom)),
     access_(settings.access),
+    zeroed_(settings.zeroed),
     random_(random),
     staged_(staged),
     managed_(!staged && !head_shared),
@@ -59,6 +60,7 @@ CLASS::mmap(const paths& filenames, const storage_settings& settings,
     expansion_(settings.rate),
     headroom_(system::possible_narrow_cast<size_t>(settings.headroom)),
     access_(settings.access),
+    zeroed_(settings.zeroed),
     random_(random),
     staged_(staged),
     managed_(false),

include/bitcoin/database/impl/memory/mmap_storage.ipp

@@ -352,7 +352,7 @@ bool CLASS::truncate(size_t count) NOEXCEPT
 #endif
 
     // Released rows are zeroed, so an allocation is zero until written.
-    if (loaded_.load())
+    if (zeroed_ && loaded_.load())
         zero_all_(count, logical, fresh, sequence{});
 
     logical_.store(count);

include/bitcoin/database/impl/primitives/headmap.ipp

@@ -140,8 +140,6 @@ memory CLASS::get_memory() const NOEXCEPT
 TEMPLATE
 Value CLASS::at(size_t index) const NOEXCEPT
 {
-    using namespace system;
-
     // Buckets at or above the logical size are unallocated (count publishes).
     if (index >= count())
         return {};
@@ -150,24 +148,7 @@ Value CLASS::at(size_t index) const NOEXCEPT
     if (!ptr)
         return {};
 
-    const auto raw = ptr.data();
-    if constexpr (aligned)
-    {
-        // Reads full padded word (masked by to_value).
-        const auto& head = *pointer_cast<std::atomic<cell>>(raw);
-        return to_value(head.load(load_order));
-    }
-    else
-    {
-        const auto& head = cell_array(raw);
-        cell value{};
-
-        mutex_.lock_shared();
-        cell_array(value) = head;
-        mutex_.unlock_shared();
-
-        return to_value(value);
-    }
+    return load(ptr.data());
 }
 
 TEMPLATE
@@ -183,6 +164,29 @@ Value CLASS::at(const memory& ptr, size_t index) const NOEXCEPT
     if (is_null(raw))
         return {};
 
+    return load(raw);
+}
+
+TEMPLATE
+Value CLASS::top() const NOEXCEPT
+{
+    static_assert(is_link);
+
+    const auto size = count();
+    if (is_zero(size))
+        return {};
+
+    const auto ptr = file_.get(link_to_position(sub1(size)));
+    if (!ptr)
+        return {};
+
+    return load(ptr.data());
+}
+
+TEMPLATE
+Value CLASS::load(memory::iterator raw) const NOEXCEPT
+{
+    using namespace system;
     if constexpr (aligned)
     {
         // Reads full padded word (masked by to_value).

include/bitcoin/database/impl/query/confirmed.ipp

@@ -85,7 +85,7 @@ TEMPLATE
 bool CLASS::is_candidate_extension(const header_link& link) const NOEXCEPT
 {
     return !is_associated(link) && is_candidate_header(link) &&
-        (to_confirmed(get_top_confirmed()) == to_parent(link));
+        (to_top_confirmed() == to_parent(link));
 }
 
 TEMPLATE

include/bitcoin/database/impl/query/initialize.ipp

@@ -37,14 +37,14 @@ const database::envelope& CLASS::envelope() const NOEXCEPT
 TEMPLATE
 hash_digest CLASS::get_top_confirmed_hash() const NOEXCEPT
 {
-    return get_header_key(to_confirmed(get_top_confirmed()));
+    return get_header_key(to_top_confirmed());
 }
 
 // server/dumps
 TEMPLATE
 hash_digest CLASS::get_top_candidate_hash() const NOEXCEPT
 {
-    return get_header_key(to_candidate(get_top_candidate()));
+    return get_header_key(to_top_candidate());
 }
 
 // node start

include/bitcoin/database/impl/query/navigate/navigate_natural.ipp

@@ -39,6 +39,18 @@ inline header_link CLASS::to_confirmed(size_t height) const NOEXCEPT
     return store_.confirmed.at(height);
 }
 
+TEMPLATE
+inline header_link CLASS::to_top_candidate() const NOEXCEPT
+{
+    return store_.candidate.top();
+}
+
+TEMPLATE
+inline header_link CLASS::to_top_confirmed() const NOEXCEPT
+{
+    return store_.confirmed.top();
+}
+
 TEMPLATE
 inline header_link CLASS::to_header(const hash_digest& key) const NOEXCEPT
 {

include/bitcoin/database/impl/query/properties_block.ipp

@@ -78,8 +78,7 @@ inline bool CLASS::is_associated(const header_link& link) const NOEXCEPT
 TEMPLATE
 uint32_t CLASS::get_top_timestamp(bool confirmed) const NOEXCEPT
 {
-    const auto top = confirmed ? to_confirmed(get_top_confirmed()) :
-        to_candidate(get_top_candidate());
+    const auto top = confirmed ? to_top_confirmed() : to_top_candidate();
 
     // returns zero if read fails.
     uint32_t timestamp{};

include/bitcoin/database/memory/mmap.hpp

@@ -430,6 +430,7 @@ class mmap
     const size_t expansion_;
     const size_t headroom_;
     const advice access_;
+    const bool zeroed_;
     const bool random_;
     const bool staged_;
     const bool managed_;

include/bitcoin/database/memory/settings.hpp

@@ -53,6 +53,12 @@ struct storage_settings
     /// KB/t to 4.0 (one page per fault), bandwidth to ~250 MB/s, and the
     /// validation rate from 648 to 720 blk/min at the same heights.
     advice access{ advice::scattered };
+
+    /// Rows released by truncate are zeroed, so a reallocation reads zero
+    /// until written. Heads override to retain at construction: a cell once
+    /// within the count holds its last store until the next, so a reader
+    /// that took that count reads a valid (possibly just-popped) link.
+    bool zeroed{ true };
 };
 
 } // namespace database

include/bitcoin/database/primitives/headmap.hpp

@@ -99,6 +99,9 @@ class headmap
     /// Return value at index using accessor, default if not allocated.
     Value at(const memory& ptr, size_t index) const NOEXCEPT;
 
+    /// Return link at the top index, terminal if empty or unloaded.
+    Value top() const NOEXCEPT;
+
     /// Append link at count into reserved capacity (single writer).
     bool push(const Link& link) NOEXCEPT;
 
@@ -183,6 +186,8 @@ class headmap
             floored_divide(position, bucket_size));
     }
 
+    Value load(memory::iterator raw) const NOEXCEPT;
+
     // These are thread safe.
     storage& file_;
     mutable shared_mutex mutex_{};

include/bitcoin/database/query.hpp

@@ -329,6 +329,8 @@ class query
     /// search key (entry)
     inline header_link to_candidate(size_t height) const NOEXCEPT;
     inline header_link to_confirmed(size_t height) const NOEXCEPT;
+    inline header_link to_top_candidate() const NOEXCEPT;
+    inline header_link to_top_confirmed() const NOEXCEPT;
     inline header_link to_header(const hash_digest& key) const NOEXCEPT;
     inline tx_link to_tx(const hash_digest& key) const NOEXCEPT;
     inline tx_link to_witness_tx(const hash_digest& key) const NOEXCEPT;

include/bitcoin/database/store.hpp

@@ -321,7 +321,8 @@ class store
         {
             .size = std::max<uint64_t>(one, minimum),
             .rate = table.rate,
-            .access = advice::random
+            .access = advice::random,
+            .zeroed = false
         };
     }
 

test/memory/mmap.cpp

@@ -1129,6 +1129,44 @@ BOOST_AUTO_TEST_CASE(mmap__truncate__reallocate__zeroed)
     BOOST_REQUIRE(!instance.get_fault());
 }
 
+BOOST_AUTO_TEST_CASE(mmap__truncate__unzeroed_reallocate__retained)
+{
+    constexpr size_t initial = 12'000;
+    constexpr size_t retained = 5'000;
+    constexpr size_t appended = 4'000;
+    const std::string file = TEST_PATH;
+    BOOST_REQUIRE(test::create(file));
+
+    map instance(file, { .size = 1, .rate = 50, .zeroed = false });
+    BOOST_REQUIRE(!instance.open());
+    BOOST_REQUIRE(!instance.load());
+
+    const auto original = stage_vector(stage_repattern, initial, one);
+    auto memory = instance.get(instance.allocate(initial));
+    BOOST_REQUIRE(memory);
+    std::copy_n(original.begin(), original.size(), memory.begin());
+
+    memory.reset();
+    BOOST_REQUIRE(instance.truncate(retained));
+    memory = instance.get_capacity(retained);
+    BOOST_REQUIRE(memory);
+
+    const auto released = std::next(original.begin(), retained);
+    const auto released_end = std::next(released, appended);
+    BOOST_REQUIRE_EQUAL_COLLECTIONS(memory.begin(), std::next(memory.begin(), appended), released, released_end);
+
+    memory.reset();
+    BOOST_REQUIRE_EQUAL(instance.allocate(appended), retained);
+    memory = instance.get(retained);
+    BOOST_REQUIRE(memory);
+    BOOST_REQUIRE_EQUAL_COLLECTIONS(memory.begin(), std::next(memory.begin(), appended), released, released_end);
+
+    memory.reset();
+    BOOST_REQUIRE(!instance.unload());
+    BOOST_REQUIRE(!instance.close());
+    BOOST_REQUIRE(!instance.get_fault());
+}
+
 BOOST_AUTO_TEST_CASE(mmap__staged__truncate_below_flush_reallocate__zeroed)
 {
     constexpr size_t initial = 12'000;

test/primitives/headmap.cpp

@@ -32,6 +32,23 @@ using slab_headmap = headmap<slab_link, max_size_t>;
 using value_link = linkage<3>;
 using value_headmap = headmap<value_link, one, uint8_t>;
 
+// Reports a stale size once, as a reader racing a truncate observes it.
+class stale_storage
+  : public test::chunk_storage
+{
+public:
+    using test::chunk_storage::chunk_storage;
+
+    size_t size() const NOEXCEPT override
+    {
+        const auto stale = stale_;
+        stale_ = zero;
+        return is_zero(stale) ? test::chunk_storage::size() : stale;
+    }
+
+    mutable size_t stale_{};
+};
+
 class little_slab
 {
 public:
@@ -209,6 +226,75 @@ BOOST_AUTO_TEST_CASE(headmap__truncate__push__rewrites_bucket)
     BOOST_REQUIRE_EQUAL(instance.at(1), 0x5566_u16);
 }
 
+BOOST_AUTO_TEST_CASE(headmap__top__empty__terminal)
+{
+    data_chunk head_file{};
+    test::chunk_storage head_store{ head_file };
+    test_headmap instance{ head_store };
+    BOOST_REQUIRE(instance.create());
+    BOOST_REQUIRE(instance.top().is_terminal());
+}
+
+BOOST_AUTO_TEST_CASE(headmap__top__single_genesis__zero)
+{
+    data_chunk head_file{};
+    test::chunk_storage head_store{ head_file };
+    test_headmap instance{ head_store };
+    BOOST_REQUIRE(instance.create());
+    BOOST_REQUIRE(instance.reserve(1u));
+    BOOST_REQUIRE(instance.push(0x0000_u16));
+    BOOST_REQUIRE_EQUAL(instance.top(), 0x0000_u16);
+}
+
+BOOST_AUTO_TEST_CASE(headmap__top__many__last_pushed)
+{
+    data_chunk head_file{};
+    test::chunk_storage head_store{ head_file };
+    test_headmap instance{ head_store };
+    BOOST_REQUIRE(instance.create());
+    BOOST_REQUIRE(instance.reserve(3u));
+    BOOST_REQUIRE(instance.push(0x1122_u16));
+    BOOST_REQUIRE(instance.push(0x3344_u16));
+    BOOST_REQUIRE(instance.push(0x5566_u16));
+    BOOST_REQUIRE_EQUAL(instance.count(), 3u);
+    BOOST_REQUIRE_EQUAL(instance.top(), 0x5566_u16);
+}
+
+BOOST_AUTO_TEST_CASE(headmap__top__truncated__expected)
+{
+    data_chunk head_file{};
+    test::chunk_storage head_store{ head_file };
+    test_headmap instance{ head_store };
+    BOOST_REQUIRE(instance.create());
+    BOOST_REQUIRE(instance.reserve(3u));
+    BOOST_REQUIRE(instance.push(0x1122_u16));
+    BOOST_REQUIRE(instance.push(0x3344_u16));
+    BOOST_REQUIRE(instance.push(0x5566_u16));
+    BOOST_REQUIRE(instance.truncate(2u));
+    BOOST_REQUIRE_EQUAL(instance.top(), 0x3344_u16);
+    BOOST_REQUIRE(instance.truncate(1u));
+    BOOST_REQUIRE_EQUAL(instance.top(), 0x1122_u16);
+    BOOST_REQUIRE(instance.truncate(0u));
+    BOOST_REQUIRE(instance.top().is_terminal());
+}
+
+BOOST_AUTO_TEST_CASE(headmap__top__truncate_race__popped_link)
+{
+    data_chunk head_file{};
+    stale_storage head_store{ head_file };
+    test_headmap instance{ head_store };
+    BOOST_REQUIRE(instance.create());
+    BOOST_REQUIRE(instance.reserve(3u));
+    BOOST_REQUIRE(instance.push(0x1122_u16));
+    BOOST_REQUIRE(instance.push(0x3344_u16));
+    BOOST_REQUIRE(instance.push(0x5566_u16));
+    BOOST_REQUIRE(instance.truncate(2u));
+    head_store.stale_ = 6u;
+    BOOST_REQUIRE_EQUAL(instance.top(), 0x5566_u16);
+    BOOST_REQUIRE_EQUAL(instance.top(), 0x3344_u16);
+    BOOST_REQUIRE_EQUAL(instance.count(), 2u);
+}
+
 BOOST_AUTO_TEST_CASE(headmap__unaligned__push_at_truncate__expected)
 {
     data_chunk head_file{};