From d96a06fb1c5e31b10624c4dabed02e53d8cc0fcc Mon Sep 17 00:00:00 2001 From: echennells Date: Wed, 30 Sep 2026 15:52:08 +0000 Subject: [PATCH] Reclaim addresses of connect failures not attributable to the address. --- include/bitcoin/network/error.hpp | 3 + .../network/sessions/session_outbound.hpp | 19 +- src/channels/channel_peer.cpp | 3 + src/error.cpp | 33 +- src/sessions/session_outbound.cpp | 64 +- test/error.cpp | 41 +- test/net/acceptor_sam.cpp | 8 +- test/net/connector_socks.cpp | 4 +- test/net/socket.cpp | 8 +- test/sessions/session_outbound.cpp | 789 ++++++++++++++++++ 10 files changed, 936 insertions(+), 36 deletions(-) diff --git a/include/bitcoin/network/error.hpp b/include/bitcoin/network/error.hpp index 9842a5d21..bb4dd1f8f 100644 --- a/include/bitcoin/network/error.hpp +++ b/include/bitcoin/network/error.hpp @@ -85,6 +85,9 @@ enum error_t : uint8_t address_in_use, resolve_failed, connect_failed, + net_unreachable, + host_unreachable, + connection_refused, // heading read failures invalid_heading, diff --git a/include/bitcoin/network/sessions/session_outbound.hpp b/include/bitcoin/network/sessions/session_outbound.hpp index 7d0d75a87..f93d11f1b 100644 --- a/include/bitcoin/network/sessions/session_outbound.hpp +++ b/include/bitcoin/network/sessions/session_outbound.hpp @@ -69,15 +69,25 @@ class BCT_API session_outbound private: typedef race_quality race; + /// Unreachable members of a connect batch, restored only if every member + /// failed as unreachable (protected by strand). + struct batch + { + typedef std::shared_ptr ptr; + config::addresses unreachable{}; + bool attributable{}; + }; + void handle_started(const code& ec, const result_handler& handler) NOEXCEPT; void do_one(const code& ec, const config::address& peer, object_key key, - const race::ptr& racer, const connector::ptr& connector) NOEXCEPT; + const race::ptr& racer, const batch::ptr& members, + const connector::ptr& connector) NOEXCEPT; void handle_one(const code& ec, const socket::ptr& socket, - object_key key, const config::address& peer, - const race::ptr& racer) NOEXCEPT; + object_key key, const config::address& peer, const race::ptr& racer, + const batch::ptr& members) NOEXCEPT; void handle_connect(const code& ec, const socket::ptr& socket, - object_key key, size_t slot) NOEXCEPT; + object_key key, size_t slot, const batch::ptr& members) NOEXCEPT; void handle_channel_start(const code& ec, const channel::ptr& channel) NOEXCEPT; @@ -89,6 +99,7 @@ class BCT_API session_outbound /// Restore an address to the address pool. inline bool maybe_reclaim(const code& ec) const NOEXCEPT; + inline bool batch_reclaim(const code& ec) const NOEXCEPT; inline bool always_reclaim(const code& ec) const NOEXCEPT; void reclaim(const code& ec, const socket::ptr& socket) NOEXCEPT; void reclaim(const code& ec, const channel::ptr& channel) NOEXCEPT; diff --git a/src/channels/channel_peer.cpp b/src/channels/channel_peer.cpp index cbbe17d2a..222a7872a 100644 --- a/src/channels/channel_peer.cpp +++ b/src/channels/channel_peer.cpp @@ -344,6 +344,9 @@ void channel_peer::handle_send(const code& ec, size_t size, size_t index, ec != error::peer_disconnect && ec != error::operation_canceled && ec != error::connect_failed && + ec != error::net_unreachable && + ec != error::host_unreachable && + ec != error::connection_refused && index < registry::size) { LOGF("Send failure " << registry::commands().at(index) << " to [" diff --git a/src/error.cpp b/src/error.cpp index aa2e4ae52..88e383e8c 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -74,6 +74,9 @@ DEFINE_ERROR_T_MESSAGE_MAP(error) { address_in_use, "address already in use" }, { resolve_failed, "resolving hostname failed" }, { connect_failed, "unable to reach remote host" }, + { net_unreachable, "network unreachable" }, + { host_unreachable, "host unreachable" }, + { connection_refused, "connection refused" }, // heading read failures { invalid_heading, "invalid message heading" }, @@ -403,14 +406,20 @@ code errc_to_error_code(const boost_code& ec) NOEXCEPT // connect-connect case boost_errc_t::address_not_available: - case boost_errc_t::not_connected: - case boost_errc_t::connection_refused: - case boost_errc_t::broken_pipe: - case boost_errc_t::host_unreachable: case boost_errc_t::network_down: - case boost_errc_t::network_reset: case boost_errc_t::network_unreachable: case boost_errc_t::no_link: + return error::net_unreachable; + + case boost_errc_t::host_unreachable: + return error::host_unreachable; + + case boost_errc_t::connection_refused: + return error::connection_refused; + + case boost_errc_t::not_connected: + case boost_errc_t::broken_pipe: + case boost_errc_t::network_reset: case boost_errc_t::no_protocol_option: case boost_errc_t::no_such_file_or_directory: case boost_errc_t::not_a_socket: @@ -500,14 +509,20 @@ code asio_to_error_code(const boost_code& ec) NOEXCEPT case asio_basic_error_t::in_progress: return error::address_in_use; + case asio_basic_error_t::network_down: + case asio_basic_error_t::network_unreachable: + return error::net_unreachable; + + case asio_basic_error_t::host_unreachable: + return error::host_unreachable; + + case asio_basic_error_t::connection_refused: + return error::connection_refused; + case asio_basic_error_t::shut_down: case asio_basic_error_t::would_block: case asio_basic_error_t::broken_pipe: - case asio_basic_error_t::connection_refused: - case asio_basic_error_t::host_unreachable: - case asio_basic_error_t::network_down: case asio_basic_error_t::network_reset: - case asio_basic_error_t::network_unreachable: case asio_basic_error_t::no_protocol_option: case asio_basic_error_t::not_connected: case asio_basic_error_t::not_socket: diff --git a/src/sessions/session_outbound.cpp b/src/sessions/session_outbound.cpp index aa4985e10..d00dbbd06 100644 --- a/src/sessions/session_outbound.cpp +++ b/src/sessions/session_outbound.cpp @@ -177,19 +177,21 @@ void session_outbound::start_connect(const code&, size_t slot) NOEXCEPT // Bogus warning, this pointer is copied into std::bind(). BC_PUSH_WARNING(NO_UNUSED_LOCAL_SMART_PTR) const auto racer = std::make_shared(connectors->size()); + const auto members = std::make_shared(); BC_POP_WARNING() // Race to first success or last failure. - racer->start(BIND(handle_connect, _1, _2, key, slot)); + racer->start(BIND(handle_connect, _1, _2, key, slot, members)); // Attempt to connect with unique address for each connector of batch. for (const auto& connector: *connectors) - take(family(slot), BIND(do_one, _1, _2, key, racer, connector)); + take(family(slot), BIND(do_one, _1, _2, key, racer, members, + connector)); } // Attempt to connect the given peer and invoke handle_one. void session_outbound::do_one(const code& ec, const config::address& peer, - object_key key, const race::ptr& racer, + object_key key, const race::ptr& racer, const batch::ptr& members, const connector::ptr& connector) NOEXCEPT { BC_ASSERT(stranded()); @@ -210,19 +212,32 @@ void session_outbound::do_one(const code& ec, const config::address& peer, return; } - connector->connect(peer, BIND(handle_one, _1, _2, key, peer, racer)); + connector->connect(peer, BIND(handle_one, _1, _2, key, peer, racer, + members)); } // Handle each do_one connection attempt, stopping on first success. void session_outbound::handle_one(const code& ec, const socket::ptr& socket, - object_key key, const config::address& peer, - const race::ptr& racer) NOEXCEPT + object_key key, const config::address& peer, const race::ptr& racer, + const batch::ptr& members) NOEXCEPT { BC_ASSERT(stranded()); ////COUNT(events::outbound2, key); - if (ec == error::service_suspended) - restore(peer, BIND(handle_reclaim, _1)); + // A failed connect as unreachable is decided once its batch completes. + if (!socket && batch_reclaim(ec)) + { + members->unreachable.push_back(peer); + } + else + { + // Any other outcome attributes unreachable members to their address. + members->attributable = true; + + // Without a socket, reclaim below cannot restore the address. + if (ec == error::service_suspended || (!socket && maybe_reclaim(ec))) + restore(peer, BIND(handle_reclaim, _1)); + } // Winner in quality race is first to pass success. if (racer->finish(ec, socket)) @@ -238,7 +253,8 @@ void session_outbound::handle_one(const code& ec, const socket::ptr& socket, // Handle the singular batch result. void session_outbound::handle_connect(const code& ec, - const socket::ptr& socket, object_key key, size_t slot) NOEXCEPT + const socket::ptr& socket, object_key key, size_t slot, + const batch::ptr& members) NOEXCEPT { BC_ASSERT(stranded()); ////COUNT(events::outbound3, key); @@ -246,6 +262,17 @@ void session_outbound::handle_connect(const code& ec, // Unregister connectors, in case there was no winner. notify(key); + // A batch that failed only as unreachable implies a local outage, so + // restore its unreachable members (up to free pool capacity). + if (!members->attributable) + { + const auto pool = network_settings().outbound.host_pool_capacity; + const auto room = floored_subtract(pool, address_count()); + const auto count = std::min(room, members->unreachable.size()); + for (size_t index{}; index < count; ++index) + restore(members->unreachable.at(index), BIND(handle_reclaim, _1)); + } + // Guard restartable timer (shutdown delay). if (stopped()) { @@ -272,6 +299,9 @@ void session_outbound::handle_connect(const code& ec, if (ec) { if (ec == error::connect_failed || + ec == error::net_unreachable || + ec == error::host_unreachable || + ec == error::connection_refused || ec == error::operation_timeout || ec == error::service_suspended || ec == error::socks_failure) @@ -352,10 +382,22 @@ inline bool session_outbound::maybe_reclaim(const code& ec) const NOEXCEPT if (address_count() >= network_settings().outbound.host_pool_capacity) return false; - // Failures that might work later (timeouts can drain pool). + // Failures that might work later, not attributable to the address. An + // address that connected is not the cause of an unreachable failure. return ec == error::operation_timeout || ec == error::channel_timeout - || ec == error::peer_disconnect; + || ec == error::peer_disconnect + || ec == error::insufficient_buffer + || batch_reclaim(ec); +} + +inline bool session_outbound::batch_reclaim(const code& ec) const NOEXCEPT +{ + // Unreachable, from a local outage or a missing route to the address. + return ec == error::net_unreachable + || ec == error::host_unreachable + || ec == error::socks_net_unreachable + || ec == error::socks_host_unreachable; } inline bool session_outbound::always_reclaim(const code& ec) const NOEXCEPT diff --git a/test/error.cpp b/test/error.cpp index 0d380b673..76519463c 100644 --- a/test/error.cpp +++ b/test/error.cpp @@ -330,6 +330,33 @@ BOOST_AUTO_TEST_CASE(error_t__code__connect_failed__true_expected_message) BOOST_REQUIRE_EQUAL(ec.message(), "unable to reach remote host"); } +BOOST_AUTO_TEST_CASE(error_t__code__net_unreachable__true_expected_message) +{ + constexpr auto value = error::net_unreachable; + const auto ec = code(value); + BOOST_REQUIRE(ec); + BOOST_REQUIRE(ec == value); + BOOST_REQUIRE_EQUAL(ec.message(), "network unreachable"); +} + +BOOST_AUTO_TEST_CASE(error_t__code__host_unreachable__true_expected_message) +{ + constexpr auto value = error::host_unreachable; + const auto ec = code(value); + BOOST_REQUIRE(ec); + BOOST_REQUIRE(ec == value); + BOOST_REQUIRE_EQUAL(ec.message(), "host unreachable"); +} + +BOOST_AUTO_TEST_CASE(error_t__code__connection_refused__true_expected_message) +{ + constexpr auto value = error::connection_refused; + const auto ec = code(value); + BOOST_REQUIRE(ec); + BOOST_REQUIRE(ec == value); + BOOST_REQUIRE_EQUAL(ec.message(), "connection refused"); +} + // heading read failures BOOST_AUTO_TEST_CASE(error_t__code__invalid_heading__true_expected_message) @@ -2519,7 +2546,13 @@ BOOST_AUTO_TEST_CASE(errc_to_error_code__generic_conditions__expected) BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::no_buffer_space)), error::insufficient_buffer); BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::permission_denied)), error::not_allowed); BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::bad_address)), error::resolve_failed); - BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::connection_refused)), error::connect_failed); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::connection_refused)), error::connection_refused); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::network_unreachable)), error::net_unreachable); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::network_down)), error::net_unreachable); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::address_not_available)), error::net_unreachable); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::no_link)), error::net_unreachable); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::host_unreachable)), error::host_unreachable); + BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::broken_pipe)), error::connect_failed); BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::address_in_use)), error::address_in_use); BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::io_error)), error::bad_stream); BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::timed_out)), error::channel_timeout); @@ -2536,7 +2569,11 @@ BOOST_AUTO_TEST_CASE(asio_to_error_code__basic_codes__expected) BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::operation_aborted)), error::operation_canceled); BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::address_family_not_supported)), error::resolve_failed); BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::address_in_use)), error::address_in_use); - BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::connection_refused)), error::connect_failed); + BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::connection_refused)), error::connection_refused); + BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::network_unreachable)), error::net_unreachable); + BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::network_down)), error::net_unreachable); + BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::host_unreachable)), error::host_unreachable); + BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::shut_down)), error::connect_failed); BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::connection_reset)), error::peer_disconnect); BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::message_size)), error::bad_stream); BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::invalid_argument)), error::invalid_configuration); diff --git a/test/net/acceptor_sam.cpp b/test/net/acceptor_sam.cpp index 408184f8c..7a7a7117c 100644 --- a/test/net/acceptor_sam.cpp +++ b/test/net/acceptor_sam.cpp @@ -377,11 +377,11 @@ BOOST_FIXTURE_TEST_CASE(acceptor_sam__start__started__operation_failed, sam_setu BOOST_REQUIRE_EQUAL(restarted.get_future().get(), error::operation_failed); } -BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__no_bridge__connect_failed, sam_setup_fixture) +BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__no_bridge__connection_refused, sam_setup_fixture) { BOOST_REQUIRE_EQUAL(create(), error::success); listener.close(); - BOOST_REQUIRE_EQUAL(accept().get(), error::connect_failed); + BOOST_REQUIRE_EQUAL(accept().get(), error::connection_refused); } BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__hello_closed__peer_disconnect, sam_setup_fixture) @@ -477,7 +477,7 @@ BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__key_path_directory__file_save, sam BOOST_REQUIRE_EQUAL(result.get(), error::file_save); } -BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__forward_refused__connect_failed, sam_setup_fixture) +BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__forward_refused__connection_refused, sam_setup_fixture) { BOOST_REQUIRE_EQUAL(create(), error::success); auto result = accept(); @@ -489,7 +489,7 @@ BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__forward_refused__connect_failed, s listener.close(); write_line("SESSION STATUS RESULT=OK DESTINATION=" + destination); - BOOST_REQUIRE_EQUAL(result.get(), error::connect_failed); + BOOST_REQUIRE_EQUAL(result.get(), error::connection_refused); BOOST_REQUIRE(closed(bridge)); } diff --git a/test/net/connector_socks.cpp b/test/net/connector_socks.cpp index b3abf19cb..b5bcdd07d 100644 --- a/test/net/connector_socks.cpp +++ b/test/net/connector_socks.cpp @@ -386,11 +386,11 @@ BOOST_FIXTURE_TEST_CASE(connector_socks__connect__unproxied__direct_success, soc BOOST_REQUIRE_EQUAL(result.get(), error::success); } -BOOST_FIXTURE_TEST_CASE(connector_socks__connect__proxy_refused__connect_failed, socks_setup_fixture) +BOOST_FIXTURE_TEST_CASE(connector_socks__connect__proxy_refused__connection_refused, socks_setup_fixture) { acceptor.close(); create(); - BOOST_REQUIRE_EQUAL(connect(config::endpoint{ SOCKS_TARGET_ENDPOINT }).get(), error::connect_failed); + BOOST_REQUIRE_EQUAL(connect(config::endpoint{ SOCKS_TARGET_ENDPOINT }).get(), error::connection_refused); } BOOST_FIXTURE_TEST_CASE(connector_socks__connect__greeting_closed__peer_disconnect, socks_setup_fixture) diff --git a/test/net/socket.cpp b/test/net/socket.cpp index 680ff5dbb..1e1d99d95 100644 --- a/test/net/socket.cpp +++ b/test/net/socket.cpp @@ -369,7 +369,7 @@ BOOST_AUTO_TEST_CASE(socket__connect__bound_family_mismatch__resolve_failed) BOOST_REQUIRE(pool.join()); } -BOOST_AUTO_TEST_CASE(socket__connect__bound_unavailable__connect_failed) +BOOST_AUTO_TEST_CASE(socket__connect__bound_unavailable__net_unreachable) { using namespace std::chrono_literals; @@ -394,7 +394,7 @@ BOOST_AUTO_TEST_CASE(socket__connect__bound_unavailable__connect_failed) }); BOOST_REQUIRE(connect_future.wait_for(2s) == std::future_status::ready); - BOOST_REQUIRE_EQUAL(connect_future.get(), error::connect_failed); + BOOST_REQUIRE_EQUAL(connect_future.get(), error::net_unreachable); client->stop(); pool.stop(); @@ -788,12 +788,12 @@ BOOST_FIXTURE_TEST_CASE(socket__async_read_some__client_sent__base_expected, acc instance->stop(); } -BOOST_FIXTURE_TEST_CASE(socket__connect__refused__connect_failed, acceptor_fixture) +BOOST_FIXTURE_TEST_CASE(socket__connect__refused__connection_refused, acceptor_fixture) { const awaiter connected{}; const auto instance = connect(clear, { closed_endpoint }, connected); - BOOST_REQUIRE_EQUAL(connected.get(), error::connect_failed); + BOOST_REQUIRE_EQUAL(connected.get(), error::connection_refused); instance->stop(); } diff --git a/test/sessions/session_outbound.cpp b/test/sessions/session_outbound.cpp index 0a503655e..50608d5e0 100644 --- a/test/sessions/session_outbound.cpp +++ b/test/sessions/session_outbound.cpp @@ -337,6 +337,210 @@ class mock_net }; }; +template +class mock_connector_connect_code + : public connector +{ +public: + typedef std::shared_ptr ptr; + + using connector::connector; + + void start(const std::string&, uint16_t, const config::address&, + const config::endpoint&, socket_handler&& handler) NOEXCEPT override + { + boost::asio::post(strand_, [=]() NOEXCEPT + { + handler(Code, nullptr); + }); + } +}; + +template +class mock_net_restore + : public mock_net +{ +public: + using mock_net::mock_net; + + size_t address_count() const NOEXCEPT override + { + return one; + } + + void take(hosts::family, address_item_handler&& handler) NOEXCEPT override + { + const auto item = config::address{ "127.0.0.1:65154" }.to_address_item(0, service::node_none); + handler(error::success, system::to_shared(item)); + } + + void restore(const address_item_cptr&, result_handler&& handler) NOEXCEPT override + { + ++restores_; + handler(error::success); + } + + size_t restores() const NOEXCEPT + { + return restores_; + } + +private: + std::atomic restores_{}; +}; + +// Alternates the two codes across connects (one of each per pair). +template +class mock_connector_connect_codes + : public connector +{ +public: + typedef std::shared_ptr ptr; + + using connector::connector; + + void start(const std::string&, uint16_t, const config::address&, + const config::endpoint&, socket_handler&& handler) NOEXCEPT override + { + const auto code = is_odd(count_++) ? Second : First; + boost::asio::post(strand_, [=]() NOEXCEPT + { + handler(code, nullptr); + }); + } + +private: + static inline std::atomic count_{}; +}; + +// Alternates take between an address and none (one of each per pair). +template +class mock_net_restore_sparse + : public mock_net_restore +{ +public: + using mock_net_restore::mock_net_restore; + + void take(hosts::family family, address_item_handler&& handler) NOEXCEPT override + { + if (is_odd(takes_++)) + { + handler(error::address_not_found, {}); + return; + } + + mock_net_restore::take(family, std::move(handler)); + } + +private: + std::atomic takes_{}; +}; + +// Fails each connect by port parity (Even code on even ports, Odd on odd). +template +class mock_connector_connect_parity + : public connector +{ +public: + typedef std::shared_ptr ptr; + + using connector::connector; + + void start(const std::string&, uint16_t port, const config::address&, + const config::endpoint&, socket_handler&& handler) NOEXCEPT override + { + const auto code = is_odd(port) ? Odd : Even; + boost::asio::post(strand_, [=]() NOEXCEPT + { + handler(code, nullptr); + }); + } +}; + +// Connects on even ports (real socket), fails net_unreachable on odd. +class mock_connector_success_or_unreachable + : public connector +{ +public: + typedef std::shared_ptr ptr; + + using connector::connector; + + void start(const std::string&, uint16_t port, const config::address&, + const config::endpoint&, socket_handler&& handler) NOEXCEPT override + { + if (is_odd(port)) + { + boost::asio::post(strand_, [=]() NOEXCEPT + { + handler(error::net_unreachable, nullptr); + }); + + return; + } + + socket::parameters params{ .maximum_request = 42, + .maximum_buffer = settings::tcp_server{ "test" }.maximum_buffer }; + const auto socket = std::make_shared(log, service_, std::move(params)); + boost::asio::post(strand_, [=]() NOEXCEPT + { + handler(error::success, socket); + }); + } +}; + +// Takes a distinct port per draw and records the ports restored. +template +class mock_net_restore_keyed + : public mock_net +{ +public: + mock_net_restore_keyed(const settings& set, const logger& log, + size_t count=one) NOEXCEPT + : mock_net(set, log), count_(count) + { + } + + size_t address_count() const NOEXCEPT override + { + return count_; + } + + void take(hosts::family, address_item_handler&& handler) NOEXCEPT override + { + const auto port = std::to_string(40000u + takes_++); + const auto item = config::address{ "127.0.0.1:" + port }.to_address_item(0, service::node_none); + handler(error::success, system::to_shared(item)); + } + + void restore(const address_item_cptr& host, result_handler&& handler) NOEXCEPT override + { + { + std::lock_guard lock(mutex_); + restored_.push_back(host->port); + } + + handler(error::success); + } + + size_t takes() const NOEXCEPT + { + return takes_; + } + + std::vector restored() const NOEXCEPT + { + std::lock_guard lock(mutex_); + return restored_; + } + +private: + const size_t count_; + std::atomic takes_{}; + mutable std::mutex mutex_{}; + std::vector restored_{}; +}; + class mock_connector_stop_connect : public mock_connector_connect_success { @@ -727,6 +931,591 @@ BOOST_AUTO_TEST_CASE(session_outbound__start__handle_connect_stopped__first_chan BOOST_REQUIRE(session->stopped()); } +BOOST_AUTO_TEST_CASE(session_outbound__start__net_unreachable__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__host_unreachable__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__socks_net_unreachable__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__socks_host_unreachable__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__insufficient_buffer__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__batch_all_unreachable__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 2; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__batch_unreachable_and_refused__not_restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 2; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__batch_unreachable_and_connected__not_restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 2; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore_keyed net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_attached_handshake()); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE_GE(net.takes(), 2u); + + const auto restored = net.restored(); + BOOST_REQUIRE(std::none_of(restored.begin(), restored.end(), [](uint16_t port) NOEXCEPT + { + return is_odd(port); + })); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__batch_unreachable_and_no_address__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 2; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore_sparse> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__channel_timeout__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(net.restores())); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__peer_disconnect__restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(!is_zero(net.restores())); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__batch_unreachable_and_timeout__timeout_restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 2; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore_keyed> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + + const auto restored = net.restored(); + BOOST_REQUIRE(!restored.empty()); + BOOST_REQUIRE(std::all_of(restored.begin(), restored.end(), [](uint16_t port) NOEXCEPT + { + return is_odd(port); + })); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__batch_unreachable_near_full_pool__room_restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 2; + set.outbound.connect_batch_size = 2; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore_keyed> net(set, log, 1); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + + // Each batch draws a port pair (2n, 2n+1), with room to restore only one. + const auto restored = net.restored(); + BOOST_REQUIRE(!restored.empty()); + BOOST_REQUIRE(std::none_of(restored.begin(), restored.end(), [&](uint16_t port) NOEXCEPT + { + const auto pair = static_cast(port ^ 1u); + return std::find(restored.begin(), restored.end(), pair) != restored.end(); + })); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__net_unreachable_full_host_pool__not_restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 1; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(is_zero(restores)); +} + +BOOST_AUTO_TEST_CASE(session_outbound__start__connection_refused__not_restored) +{ + const logger log{}; + settings set(selection::mainnet); + set.outbound.host_pool_capacity = 10; + set.outbound.connect_batch_size = 1; + set.outbound.connections = 1; + set.outbound.connect_timeout_seconds = 10000; + set.retry_timeout_seconds = 0; + mock_net_restore> net(set, log); + auto session = std::make_shared(net, 1); + BOOST_REQUIRE(session->stopped()); + + std::promise started; + boost::asio::post(net.strand(), [=, &started]() NOEXCEPT + { + session->start([&](const code& ec) NOEXCEPT + { + started.set_value(ec); + }); + }); + + BOOST_REQUIRE_EQUAL(started.get_future().get(), error::success); + BOOST_REQUIRE(session->require_reconnect()); + const auto restores = net.restores(); + + std::promise stopped; + boost::asio::post(net.strand(), [=, &stopped]() NOEXCEPT + { + session->stop(); + stopped.set_value(true); + }); + + BOOST_REQUIRE(stopped.get_future().get()); + BOOST_REQUIRE(session->stopped()); + BOOST_REQUIRE(is_zero(restores)); +} + BOOST_AUTO_TEST_CASE(session_outbound__start__handle_one__first_channel_success) { const logger log{};