Optimize SP scanning using a new threshold index on the scan table.

builds/gnu/Makefile.am

@@ -112,6 +112,7 @@ include_bitcoin_database_impl_primitivesdir = \
 include_bitcoin_database_impl_primitives_HEADERS = \
     ${srcdir}/../../include/bitcoin/database/impl/primitives/arrayhead.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/primitives/arraymap.ipp \
+    ${srcdir}/../../include/bitcoin/database/impl/primitives/arraymaps.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/primitives/body.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/primitives/hashhead.ipp \
     ${srcdir}/../../include/bitcoin/database/impl/primitives/hashmap.ipp \
@@ -269,6 +270,7 @@ include_bitcoin_database_primitivesdir = \
 include_bitcoin_database_primitives_HEADERS = \
     ${srcdir}/../../include/bitcoin/database/primitives/arrayhead.hpp \
     ${srcdir}/../../include/bitcoin/database/primitives/arraymap.hpp \
+    ${srcdir}/../../include/bitcoin/database/primitives/arraymaps.hpp \
     ${srcdir}/../../include/bitcoin/database/primitives/body.hpp \
     ${srcdir}/../../include/bitcoin/database/primitives/column.hpp \
     ${srcdir}/../../include/bitcoin/database/primitives/hashhead.hpp \
@@ -327,7 +329,6 @@ include_bitcoin_database_tables_indexes_HEADERS = \
     ${srcdir}/../../include/bitcoin/database/tables/indexes/duplicate.hpp \
     ${srcdir}/../../include/bitcoin/database/tables/indexes/height.hpp \
     ${srcdir}/../../include/bitcoin/database/tables/indexes/scan.hpp \
-    ${srcdir}/../../include/bitcoin/database/tables/indexes/scan_start.hpp \
     ${srcdir}/../../include/bitcoin/database/tables/indexes/state.hpp \
     ${srcdir}/../../include/bitcoin/database/tables/indexes/strong_tx.hpp \
     ${srcdir}/../../include/bitcoin/database/tables/indexes/wtxid.hpp
@@ -501,7 +502,6 @@ test_libbitcoin_database_test_SOURCES = \
     ${srcdir}/../../test/tables/indexes/duplicate.cpp \
     ${srcdir}/../../test/tables/indexes/height.cpp \
     ${srcdir}/../../test/tables/indexes/scan.cpp \
-    ${srcdir}/../../test/tables/indexes/scan_start.cpp \
     ${srcdir}/../../test/tables/indexes/state.cpp \
     ${srcdir}/../../test/tables/indexes/strong_tx.cpp \
     ${srcdir}/../../test/tables/indexes/wtxid.cpp \

builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj

@@ -157,6 +157,7 @@
     <ClInclude Include="..\..\..\..\include\bitcoin\database\memory\utilities.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\arrayhead.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\arraymap.hpp" />
+    <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\arraymaps.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\body.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\column.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\hashhead.hpp" />
@@ -194,7 +195,6 @@
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\duplicate.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\height.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\scan.hpp" />
-    <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\scan_start.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\state.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\strong_tx.hpp" />
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\wtxid.hpp" />
@@ -247,6 +247,7 @@
     <None Include="..\..\..\..\include\bitcoin\database\impl\memory\mmap_storage.ipp" />
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\arrayhead.ipp" />
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\arraymap.ipp" />
+    <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\arraymaps.ipp" />
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\body.ipp" />
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\hashhead.ipp" />
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\hashmap.ipp" />

builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters

@@ -239,6 +239,9 @@
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\arraymap.hpp">
       <Filter>include\bitcoin\database\primitives</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\arraymaps.hpp">
+      <Filter>include\bitcoin\database\primitives</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\..\include\bitcoin\database\primitives\body.hpp">
       <Filter>include\bitcoin\database\primitives</Filter>
     </ClInclude>
@@ -350,9 +353,6 @@
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\scan.hpp">
       <Filter>include\bitcoin\database\tables\indexes</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\scan_start.hpp">
-      <Filter>include\bitcoin\database\tables\indexes</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\..\..\include\bitcoin\database\tables\indexes\state.hpp">
       <Filter>include\bitcoin\database\tables\indexes</Filter>
     </ClInclude>
@@ -510,6 +510,9 @@
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\arraymap.ipp">
       <Filter>include\bitcoin\database\impl\primitives</Filter>
     </None>
+    <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\arraymaps.ipp">
+      <Filter>include\bitcoin\database\impl\primitives</Filter>
+    </None>
     <None Include="..\..\..\..\include\bitcoin\database\impl\primitives\body.ipp">
       <Filter>include\bitcoin\database\impl\primitives</Filter>
     </None>

include/bitcoin/database.hpp

@@ -46,6 +46,7 @@
 #include <bitcoin/database/memory/interfaces/storage.hpp>
 #include <bitcoin/database/primitives/arrayhead.hpp>
 #include <bitcoin/database/primitives/arraymap.hpp>
+#include <bitcoin/database/primitives/arraymaps.hpp>
 #include <bitcoin/database/primitives/body.hpp>
 #include <bitcoin/database/primitives/column.hpp>
 #include <bitcoin/database/primitives/hashhead.hpp>
@@ -84,7 +85,6 @@
 #include <bitcoin/database/tables/indexes/duplicate.hpp>
 #include <bitcoin/database/tables/indexes/height.hpp>
 #include <bitcoin/database/tables/indexes/scan.hpp>
-#include <bitcoin/database/tables/indexes/scan_start.hpp>
 #include <bitcoin/database/tables/indexes/state.hpp>
 #include <bitcoin/database/tables/indexes/strong_tx.hpp>
 #include <bitcoin/database/tables/indexes/wtxid.hpp>

include/bitcoin/database/impl/primitives/arraymaps.ipp

@@ -0,0 +1,261 @@
+/**
+ * Copyright (c) 2011-2026 libbitcoin developers
+ *
+ * This file is part of libbitcoin.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_ARRAYMAPS_IPP
+#define LIBBITCOIN_DATABASE_PRIMITIVES_ARRAYMAPS_IPP
+
+#include <bitcoin/database/define.hpp>
+#include <bitcoin/database/memory/memory.hpp>
+
+namespace libbitcoin {
+namespace database {
+
+TEMPLATE
+CLASS::arraymaps(storage& header, storage& body,
+    const Link& buckets) NOEXCEPT
+  : head_(header, buckets),
+    body_(body)
+{
+}
+
+// not thread safe
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+bool CLASS::create() NOEXCEPT
+{
+    Link count{};
+    return head_.create() &&
+        head_.get_body_count(count) && body_.truncate(count);
+}
+
+TEMPLATE
+bool CLASS::close() NOEXCEPT
+{
+    return head_.set_body_count(body_.count());
+}
+
+TEMPLATE
+bool CLASS::backup(bool) NOEXCEPT
+{
+    return head_.set_body_count(body_.count());
+}
+
+TEMPLATE
+bool CLASS::restore() NOEXCEPT
+{
+    Link count{};
+    return head_.verify() &&
+        head_.get_body_count(count) && body_.truncate(count);
+}
+
+TEMPLATE
+bool CLASS::verify() const NOEXCEPT
+{
+    Link count{};
+    return head_.verify() &&
+        head_.get_body_count(count) && count == body_.count();
+}
+
+// sizing
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+bool CLASS::enabled() const NOEXCEPT
+{
+    return head_.enabled();
+}
+
+TEMPLATE
+size_t CLASS::buckets() const NOEXCEPT
+{
+    return head_.buckets();
+}
+
+TEMPLATE
+size_t CLASS::head_size() const NOEXCEPT
+{
+    return head_.size();
+}
+
+TEMPLATE
+size_t CLASS::body_size() const NOEXCEPT
+{
+    return body_.size();
+}
+
+TEMPLATE
+Link CLASS::count() const NOEXCEPT
+{
+    return body_.count();
+}
+
+TEMPLATE
+Link CLASS::allocate(const Link& count) NOEXCEPT
+{
+    return body_.allocate(count);
+}
+
+TEMPLATE
+bool CLASS::truncate(const Link& count) NOEXCEPT
+{
+    return body_.truncate(count);
+}
+
+TEMPLATE
+bool CLASS::drop() NOEXCEPT
+{
+    return head_.clear() && body_.truncate(0) && backup();
+}
+
+// Faults.
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+code CLASS::get_fault() const NOEXCEPT
+{
+    const auto ec = head_.get_fault();
+    return ec ? ec : body_.get_fault();
+}
+
+TEMPLATE
+size_t CLASS::get_space() const NOEXCEPT
+{
+    return system::ceilinged_add(head_.get_space(), body_.get_space());
+}
+
+TEMPLATE
+code CLASS::reload() NOEXCEPT
+{
+    return body_.reload();
+}
+
+// query interface (head)
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+Link CLASS::at(size_t key) const NOEXCEPT
+{
+    return head_.at(key);
+}
+
+TEMPLATE
+bool CLASS::push(const Link& link, size_t key) NOEXCEPT
+{
+    // Avoid setting at/above terminal sentinel into a bucket position.
+    if (key >= Link::terminal)
+        return false;
+
+    return head_.push(link, head_.index(key));
+}
+
+// query interface (columnar)
+// ----------------------------------------------------------------------------
+
+TEMPLATE
+memory CLASS::guard() const NOEXCEPT
+{
+    return get_memory<zero>();
+}
+
+TEMPLATE
+template <size_t Column>
+memory CLASS::get_memory() const NOEXCEPT
+{
+    return body_.template get<Column>();
+}
+
+// static
+TEMPLATE
+template <size_t Column, typename Element>
+bool CLASS::get(const memory& ptr, const Link& link, Element& element) NOEXCEPT
+{
+    static_assert(Element::size == width<Column>);
+    if (!ptr || link.is_terminal())
+        return false;
+
+    using namespace system;
+    const auto start = body::template link_to_position<Column>(link);
+    if (is_limited<ptrdiff_t>(start))
+        return false;
+
+    const auto size = ptr.size();
+    const auto position = possible_narrow_sign_cast<ptrdiff_t>(start);
+    if (position >= size)
+        return false;
+
+    const auto offset = ptr.offset(start);
+    if (is_null(offset))
+        return false;
+
+    iostream stream{ offset, size - position };
+    reader source{ stream };
+
+    BC_DEBUG_ONLY(source.set_limit(width<Column> * element.count());)
+    return element.from_data(source);
+}
+
+// TODO: gets are not optimized for shared remap guard.
+TEMPLATE
+template <size_t Column, typename Element>
+bool CLASS::get(const Link& link, Element& element) const NOEXCEPT
+{
+    return get<Column>(get_memory<Column>(), link, element);
+}
+
+// TODO: puts are optimized for shared remap guard (required).
+TEMPLATE
+template <size_t Column, typename Element>
+bool CLASS::put(const Link& link, const Element& element) NOEXCEPT
+{
+    const auto ptr = body_.template get_raw<Column>(link);
+    if (!put<Column>(ptr, element))
+        return false;
+
+    body_.complete(link, element.count());
+    return true;
+}
+
+TEMPLATE
+void CLASS::complete(const Link& link, const Link& count) NOEXCEPT
+{
+    body_.complete(link, count);
+}
+
+// protected (unguarded memory access)
+TEMPLATE
+template <size_t Column, typename Element>
+bool CLASS::put(memory::iterator it, const Element& element) NOEXCEPT
+{
+    static_assert(Element::size == width<Column>);
+    if (is_null(it))
+        return false;
+
+    using namespace system;
+    const auto bytes = width<Column> * element.count();
+    iostream stream{ it, possible_narrow_sign_cast<ptrdiff_t>(bytes) };
+    flipper sink{ stream };
+
+    BC_DEBUG_ONLY(sink.set_limit(width<Column> * element.count());)
+    return element.to_data(sink);
+}
+
+} // namespace database
+} // namespace libbitcoin
+
+#endif

include/bitcoin/database/impl/query/archive/chain_writer.ipp

@@ -315,12 +315,17 @@ code CLASS::set_code(header_link& out_fk, const header& header,
     if (parent_fk.is_terminal() != (previous == system::null_hash))
         return system::error::orphan_block;
 
+    // Bypass states, and completion states that are not required.
     using namespace system;
-    const auto checkpoint = ctx.height <=
-        store_.get_envelope().top_checkpoint.height();
-    const auto bypass = bit_or<uint8_t>(
-        milestone ? block_state::milestoned : block_state::no_state,
-        checkpoint ? block_state::checkpointed : block_state::no_state);
+    uint8_t flags{};
+    if (milestone)
+        flags = bit_or<uint8_t>(flags, block_state::milestoned);
+    if (ctx.height <= store_.get_envelope().top_checkpoint.height())
+        flags = bit_or<uint8_t>(flags, block_state::checkpointed);
+    if (!is_silent(ctx))
+        flags = bit_or<uint8_t>(flags, block_state::scanned);
+    if (!filter_enabled())
+        flags = bit_or<uint8_t>(flags, block_state::filtered);
 
     // ========================================================================
     const auto scope = get_transactor();
@@ -340,8 +345,8 @@ code CLASS::set_code(header_link& out_fk, const header& header,
         return error::header_put;
 
     // Clean single allocation failure (e.g. disk full).
-    if (is_nonzero(bypass) && !store_.state.set(to_state(out_fk),
-        table::state::merge{ {}, bypass }))
+    if (is_nonzero(flags) && !store_.state.set(to_state(out_fk),
+        table::state::merge{ {}, flags }))
         return error::header_state_put;
 
     return error::success;