Skip to content

Honor include_mempool in gettxout/getdescriptoractivity, update console dumps. #1065

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
evoskuil merged 6 commits into from
Oct 13, 2026
Merged

Honor include_mempool in gettxout/getdescriptoractivity, update console dumps. #1065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a86d86d
Add outs and wtxid to collisions scan.
evoskuil Oct 13, 2026
1b41641
Clean up information dump.
evoskuil Oct 13, 2026
20511e4
Localize scan text.
evoskuil Oct 13, 2026
6d53123
Implement native address unconfirmed.
evoskuil Oct 13, 2026
ab48194
Honor include_mempool in gettxout and getdescriptoractivity.
evoskuil Oct 13, 2026
af1b022
Group collisions length expectations.
evoskuil Oct 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion console/executor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ class executor
void dump_records() const;
void dump_buckets() const;
void dump_progress() const;
void dump_collisions() const;
void dump_load_factors() const;

// Warnings.
void warn_hardware(system::string_list& out) const;
Expand Down
2 changes: 1 addition & 1 deletion console/executor_commands.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ bool executor::do_information()
dump_body_sizes();
dump_records();
dump_buckets();
dump_collisions();
dump_load_factors();

// This one can take a few seconds on cold iron.
logger(BS_INFORMATION_PROGRESS_START);
Expand Down
41 changes: 22 additions & 19 deletions console/executor_dumps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,6 @@ void executor::dump_body_sizes() const
query_.output_body_size(),
query_.ins_body_size(),
query_.outs_body_size(),
query_.candidate_body_size(),
query_.confirmed_body_size(),
query_.ecdsa0_body_size(),
query_.ecdsa1_body_size(),
query_.schnorr0_body_size(),
Expand All @@ -166,7 +164,6 @@ void executor::dump_body_sizes() const
query_.strong_body_size(),
query_.scan_body_size(),
query_.scan_prefix_body_size(),
query_.state_body_size(),
query_.pool_body_size(),
query_.wtxid_body_size(),
query_.spends_body_size(),
Expand All @@ -181,8 +178,6 @@ void executor::dump_records() const
query_.tx_records(),
query_.ins_records(),
query_.outs_records(),
query_.candidate_records(),
query_.confirmed_records(),
query_.ecdsa0_records(),
query_.ecdsa1_records(),
query_.schnorr0_records(),
Expand All @@ -199,7 +194,6 @@ void executor::dump_records() const
query_.strong_records(),
query_.scan_records(),
query_.scan_prefix_records(),
query_.state_records(),
query_.pool_records(),
query_.wtxid_records(),
query_.spends_records(),
Expand All @@ -214,44 +208,53 @@ void executor::dump_buckets() const
query_.tx_buckets(),
query_.ins_buckets(),
query_.outs_buckets(),
query_.candidate_buckets(),
query_.confirmed_buckets(),
query_.prevout_buckets(),
query_.duplicate_buckets(),
query_.strong_buckets(),
query_.scan_buckets(),
query_.state_buckets(),
query_.pool_buckets(),
query_.wtxid_buckets(),
query_.filter_bk_buckets(),
query_.filter_tx_buckets()));
}

void executor::dump_collisions() const
void executor::dump_load_factors() const
{
const auto rate = [](size_t records, size_t buckets)
const auto load = [](size_t records, size_t buckets)
{
return is_zero(buckets) ? 0.0 : to_double(records) / buckets;
};

const auto header = rate(query_.header_records(), query_.header_buckets());
const auto tx = rate(query_.tx_records(), query_.tx_buckets());
const auto ins = rate(query_.ins_records(), query_.ins_buckets());
const auto strong = rate(query_.strong_records(), query_.strong_buckets());
const auto pool = rate(query_.pool_records(), query_.pool_buckets());
const auto wtxid = rate(query_.wtxid_records(), query_.wtxid_buckets());
const auto header = load(query_.header_records(), query_.header_buckets());
const auto tx = load(query_.tx_records(), query_.tx_buckets());
const auto ins = load(query_.ins_records(), query_.ins_buckets());
const auto strong = load(query_.strong_records(), query_.strong_buckets());
const auto pool = load(query_.pool_records(), query_.pool_buckets());
const auto wtxid = load(query_.wtxid_records(), query_.wtxid_buckets());

if (query_.address_enabled())
{
const auto outs = rate(query_.outs_records(), query_.outs_buckets());
logger(std::format(BS_INFORMATION_COLLISION_RATES_OUTS,
const auto outs = load(query_.outs_records(), query_.outs_buckets());
logger(std::format(BS_INFORMATION_LOAD_FACTORS_OUTS,
header, tx, ins, outs, strong, pool, wtxid));
return;
}

logger(std::format(BS_INFORMATION_COLLISION_RATES,
logger(std::format(BS_INFORMATION_LOAD_FACTORS,
header, tx, ins, strong, pool, wtxid));
}

void executor::dump_progress() const
{
using namespace system;
const auto wire = [](size_t size)
{
return size == max_size_t ? std::string{ BS_INFORMATION_INCOMPLETE } :
std::to_string(size);
};

logger(std::format(BS_INFORMATION_PROGRESS,
query_.get_fork(),
Expand All @@ -261,8 +264,8 @@ void executor::dump_progress() const
encode_hash(query_.get_top_candidate_hash()),
query_.get_top_associated(),
(query_.get_top_candidate() - query_.get_unassociated_count()),
query_.get_confirmed_size(),
query_.get_candidate_size()));
wire(query_.get_confirmed_size()),
wire(query_.get_candidate_size())));
}

} // namespace server
Expand Down
2 changes: 1 addition & 1 deletion console/executor_options.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ void executor::do_info() const
dump_body_sizes();
dump_records();
dump_buckets();
dump_collisions();
dump_load_factors();
}

// [m]enu
Expand Down
113 changes: 65 additions & 48 deletions console/executor_scans.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,16 @@ void executor::scan_flags() const
const auto link = query_.to_candidate(height);
if (!query_.get_context(ctx, link) || (ctx.height != height))
{
logger(std::format("Error: {}", error));
logger(std::format(BS_FLAGS_ERROR, error));
return;
}

if (ctx.flags != flags)
{
const binary prev{ flag_bits, to_big_endian(flags) };
const binary next{ flag_bits, to_big_endian(ctx.flags) };
logger(std::format("Forked from [{}] to [{}] at [{}:{}]",
prev.encoded(), next.encoded(),
logger(std::format(BS_FLAGS_FORKED, prev.encoded(),
next.encoded(),
encode_hash(query_.get_header_key(link)), height));
flags = ctx.flags;
}
Expand All @@ -70,8 +70,7 @@ void executor::scan_flags() const
logger(BS_OPERATION_CANCELED);

const auto span = duration_cast<milliseconds>(logger::now() - start);
logger(std::format("Scanned {} headers for rule forks in {} ms.", height,
span.count()));
logger(std::format(BS_FLAGS_SCANNED, height, span.count()));
}

// Link keys are narrower than hashes, other keys are hashed.
Expand All @@ -84,6 +83,24 @@ constexpr bool is_linked() NOEXCEPT
return false;
}

// Null points (coinbase) and null search keys (unwritten rows) are excluded.
template <typename Key>
constexpr bool is_nullable() NOEXCEPT
{
return is_same_type<Key, chain::point> || database::keys::is_search<Key>;
}

template <typename Key>
constexpr bool is_null(const Key& key) NOEXCEPT
{
if constexpr (is_same_type<Key, chain::point>)
return key.index() == chain::point::null_index;
else if constexpr (database::keys::is_search<Key>)
return key.value == null_hash;
else
return false;
}

static std::string to_grouped(size_t value)
{
auto text = std::to_string(value);
Expand Down Expand Up @@ -116,17 +133,16 @@ void executor::scan_buckets(const std::string& name, const Table& table,
const auto load = to_double(records) / buckets;
const auto expect = linked ? std::min(load, 1.0) : 1.0 - std::exp(-load);
const auto span = duration_cast<seconds>(logger::now() - start);
logger(std::format("{:<10} {:>15} {:>15} {:>9.6f} {:>8.6f} {:>8.6f} "
"{:>6}", name, to_grouped(bucket), to_grouped(records), load,
(to_double(filled) / std::max(bucket, one)), expect, span.count()));
logger(std::format(BS_BUCKETS_ROW, name, to_grouped(bucket),
to_grouped(records), load, (to_double(filled) / std::max(bucket, one)),
expect, span.count()));
}

// hashmap bucket fill rates, all heads.
void executor::scan_buckets() const
{
logger(BS_OPERATION_INTERRUPT);
logger("table buckets records load "
"fill expect secs");
logger(BS_BUCKETS_HEADER);

scan_buckets("header", store_.header, query_.header_records());
scan_buckets("tx", store_.tx, query_.tx_records());
Expand Down Expand Up @@ -156,25 +172,18 @@ void executor::scan_collisions(const std::string& name, size_t records,
constexpr auto rows = 20_size;
constexpr auto filtered = !filter_t::disabled;
constexpr auto empty = unmask_right<filter>(m);
constexpr auto points = is_same_type<key_t, chain::point>;
constexpr auto nullable = is_nullable<key_t>();
constexpr auto linked = is_linked<key_t>();

if (is_zero(buckets))
return;

const auto kind = linked ? "link" : "hash";
const auto kind = linked ? BS_COLLISIONS_LINK : BS_COLLISIONS_HASH;
if constexpr (filtered)
{
logger(std::format("\n{} ({} keyed, filter m {} k {} k_max {})",
name, kind, m, k, Table::filter_k_max));
logger(" keys load fp model "
"walk model secs");
}
logger(std::format(BS_COLLISIONS_FILTERED, name, kind, m, k,
Table::filter_k_max));
else
{
logger(std::format("\n{} ({} keyed, unfiltered)", name, kind));
logger(" keys load secs");
}
logger(std::format(BS_COLLISIONS_UNFILTERED, name, kind));

std_vector<uint8_t> lengths(buckets, zero);
std_vector<filter> screens(filtered ? buckets : zero, empty);
Expand Down Expand Up @@ -208,8 +217,8 @@ void executor::scan_collisions(const std::string& name, size_t records,
const auto cost = Table::expected_walk(k, middle);
fp_model += rate * window;
walk_model += cost * window;
logger(std::format("{:>15} {:>9.6f} {:>10.7f} {:>10.7f} "
"{:>10.7f} {:>10.7f} {:>6}", to_grouped(keyed), load,
logger(std::format(BS_COLLISIONS_FILTERED_ROW,
to_grouped(keyed), load,
(to_double(window_collisions) / window), rate,
(to_double(window_walk) / window), cost, secs.count()));

Expand All @@ -218,8 +227,8 @@ void executor::scan_collisions(const std::string& name, size_t records,
}
else
{
logger(std::format("{:>15} {:>9.6f} {:>6}", to_grouped(keyed),
load, secs.count()));
logger(std::format(BS_COLLISIONS_UNFILTERED_ROW,
to_grouped(keyed), load, secs.count()));
}

window_start = keyed;
Expand All @@ -231,13 +240,10 @@ void executor::scan_collisions(const std::string& name, size_t records,
progress();

const auto key = get_key(index);
if constexpr (points)
if (is_null(key))
{
if (key.index() == chain::point::null_index)
{
++nulls;
continue;
}
++nulls;
continue;
}

const auto bucket = keys::bucket(key, count);
Expand Down Expand Up @@ -289,7 +295,7 @@ void executor::scan_collisions(const std::string& name, size_t records,
return 0.0;
};

logger(" length buckets expect");
logger(BS_COLLISIONS_LENGTHS);
auto poisson = std::exp(-load);
auto squares = 0.0;
auto hits = 0.0;
Expand All @@ -307,13 +313,14 @@ void executor::scan_collisions(const std::string& name, size_t records,
squares += to_double(occurrences) * size * size;
hits += to_double(occurrences) * size * (size + 1.0) / 2.0;
longest = length;
logger(std::format("{:>15} {:>15} {:>15.0f}", length,
to_grouped(occurrences), (linked ? perfect(length) :
poisson * buckets)));
const auto expect = linked ? perfect(length) : poisson * buckets;
logger(std::format(BS_COLLISIONS_LENGTHS_ROW, length,
to_grouped(occurrences),
to_grouped(to_integer<size_t>(std::round(expect)))));
}

if constexpr (points)
logger(std::format("{:>15} null points excluded", to_grouped(nulls)));
if constexpr (nullable)
logger(std::format(BS_COLLISIONS_NULLS, to_grouped(nulls)));

const auto divisor = to_double(std::max(keyed, one));
const auto empty_load = is_zero(keyed);
Expand All @@ -329,17 +336,15 @@ void executor::scan_collisions(const std::string& name, size_t records,
((1.0 - part) * whole * (whole + 1.0) / 2.0 +
part * (whole + 1.0) * (whole + 2.0) / 2.0) / load;

distributions.push_back(std::format("{:<10} {:>15} {:>15} {:>9.6f} "
"{:>8.6f} {:>8.6f} {:>7.4f} {:>7.4f} {:>7.4f} {:>7.4f} {:>4}",
distributions.push_back(std::format(BS_COLLISIONS_DISTRIBUTION_ROW,
name, to_grouped(buckets), to_grouped(keyed), load, filled,
fill_expect, dispersion, dispersion_expect, walked, walk_expect,
longest));

if constexpr (filtered)
{
const auto optimal = Table::optimal_k(keyed, buckets);
filters.push_back(std::format("{:<10} {:>3} {:>3} {:>3} {:>10.7f} "
"{:>10.7f} {:>10.7f} {:>10.7f} {:>10.7f} {:>10.7f}", name, m, k,
filters.push_back(std::format(BS_COLLISIONS_FILTER_ROW, name, m, k,
optimal, (to_double(collisions) / divisor),
(fp_model / divisor), (to_double(walk) / divisor),
(walk_model / divisor), Table::false_positive(k, load),
Expand Down Expand Up @@ -389,15 +394,27 @@ void executor::scan_collisions()
ins_link::integer>(index));
});

logger("\nDistribution (measured, expected)...");
logger("table buckets keys load "
"fill expect disp expect walk expect max");
scan_collisions<table::outs>("outs", query_.outs_records(),
query_.outs_buckets(), store_.outs.filter_k(), distributions, filters,
[&](size_t index)
{
return table::outs::key{ query_.get_outs_key(possible_narrow_cast<
outs_link::integer>(index)) };
});

scan_collisions<table::wtxid>("wtxid", query_.wtxid_records(),
query_.wtxid_buckets(), store_.wtxid.filter_k(), distributions,
filters, [&](size_t index)
{
return table::wtxid::key{ query_.get_wtxid_key(
possible_narrow_cast<pool_link::integer>(index)) };
});

logger(BS_COLLISIONS_DISTRIBUTION);
for (const auto& row: distributions)
logger(row);

logger("\nFilter (measured vs model over growth, fp at load by k)...");
logger("table m k opt fp model walk "
"model fp(k) fp(opt)");
logger(BS_COLLISIONS_FILTER);
for (const auto& row: filters)
logger(row);
}
Expand Down
Loading
Loading