Reclaim addresses of connect failures not attributable to the address.

include/bitcoin/network/error.hpp

@@ -85,6 +85,9 @@ enum error_t : uint8_t
     address_in_use,
     resolve_failed,
     connect_failed,
+    net_unreachable,
+    host_unreachable,
+    connection_refused,
 
     // heading read failures
     invalid_heading,

include/bitcoin/network/sessions/session_outbound.hpp

@@ -69,15 +69,25 @@ class BCT_API session_outbound
 private:
     typedef race_quality<const code&, const socket::ptr&> race;
 
+    /// Unreachable members of a connect batch, restored only if every member
+    /// failed as unreachable (protected by strand).
+    struct batch
+    {
+        typedef std::shared_ptr<batch> ptr;
+        config::addresses unreachable{};
+        bool attributable{};
+    };
+
     void handle_started(const code& ec,
         const result_handler& handler) NOEXCEPT;
     void do_one(const code& ec, const config::address& peer, object_key key,
-        const race::ptr& racer, const connector::ptr& connector) NOEXCEPT;
+        const race::ptr& racer, const batch::ptr& members,
+        const connector::ptr& connector) NOEXCEPT;
     void handle_one(const code& ec, const socket::ptr& socket,
-        object_key key, const config::address& peer,
-        const race::ptr& racer) NOEXCEPT;
+        object_key key, const config::address& peer, const race::ptr& racer,
+        const batch::ptr& members) NOEXCEPT;
     void handle_connect(const code& ec, const socket::ptr& socket,
-        object_key key, size_t slot) NOEXCEPT;
+        object_key key, size_t slot, const batch::ptr& members) NOEXCEPT;
 
     void handle_channel_start(const code& ec,
         const channel::ptr& channel) NOEXCEPT;
@@ -89,6 +99,7 @@ class BCT_API session_outbound
 
     /// Restore an address to the address pool.
     inline bool maybe_reclaim(const code& ec) const NOEXCEPT;
+    inline bool batch_reclaim(const code& ec) const NOEXCEPT;
     inline bool always_reclaim(const code& ec) const NOEXCEPT;
     void reclaim(const code& ec, const socket::ptr& socket) NOEXCEPT;
     void reclaim(const code& ec, const channel::ptr& channel) NOEXCEPT;

src/channels/channel_peer.cpp

@@ -344,6 +344,9 @@ void channel_peer::handle_send(const code& ec, size_t size, size_t index,
         ec != error::peer_disconnect &&
         ec != error::operation_canceled &&
         ec != error::connect_failed &&
+        ec != error::net_unreachable &&
+        ec != error::host_unreachable &&
+        ec != error::connection_refused &&
         index < registry::size)
     {
         LOGF("Send failure " << registry::commands().at(index) << " to ["

src/error.cpp

@@ -74,6 +74,9 @@ DEFINE_ERROR_T_MESSAGE_MAP(error)
     { address_in_use, "address already in use" },
     { resolve_failed, "resolving hostname failed" },
     { connect_failed, "unable to reach remote host" },
+    { net_unreachable, "network unreachable" },
+    { host_unreachable, "host unreachable" },
+    { connection_refused, "connection refused" },
 
     // heading read failures
     { invalid_heading, "invalid message heading" },
@@ -403,14 +406,20 @@ code errc_to_error_code(const boost_code& ec) NOEXCEPT
 
             // connect-connect
             case boost_errc_t::address_not_available:
-            case boost_errc_t::not_connected:
-            case boost_errc_t::connection_refused:
-            case boost_errc_t::broken_pipe:
-            case boost_errc_t::host_unreachable:
             case boost_errc_t::network_down:
-            case boost_errc_t::network_reset:
             case boost_errc_t::network_unreachable:
             case boost_errc_t::no_link:
+                return error::net_unreachable;
+
+            case boost_errc_t::host_unreachable:
+                return error::host_unreachable;
+
+            case boost_errc_t::connection_refused:
+                return error::connection_refused;
+
+            case boost_errc_t::not_connected:
+            case boost_errc_t::broken_pipe:
+            case boost_errc_t::network_reset:
             case boost_errc_t::no_protocol_option:
             case boost_errc_t::no_such_file_or_directory:
             case boost_errc_t::not_a_socket:
@@ -500,14 +509,20 @@ code asio_to_error_code(const boost_code& ec) NOEXCEPT
             case asio_basic_error_t::in_progress:
                 return error::address_in_use;
 
+            case asio_basic_error_t::network_down:
+            case asio_basic_error_t::network_unreachable:
+                return error::net_unreachable;
+
+            case asio_basic_error_t::host_unreachable:
+                return error::host_unreachable;
+
+            case asio_basic_error_t::connection_refused:
+                return error::connection_refused;
+
             case asio_basic_error_t::shut_down:
             case asio_basic_error_t::would_block:
             case asio_basic_error_t::broken_pipe:
-            case asio_basic_error_t::connection_refused:
-            case asio_basic_error_t::host_unreachable:
-            case asio_basic_error_t::network_down:
             case asio_basic_error_t::network_reset:
-            case asio_basic_error_t::network_unreachable:
             case asio_basic_error_t::no_protocol_option:
             case asio_basic_error_t::not_connected:
             case asio_basic_error_t::not_socket:

src/sessions/session_outbound.cpp

@@ -177,19 +177,21 @@ void session_outbound::start_connect(const code&, size_t slot) NOEXCEPT
     // Bogus warning, this pointer is copied into std::bind().
     BC_PUSH_WARNING(NO_UNUSED_LOCAL_SMART_PTR)
     const auto racer = std::make_shared<race>(connectors->size());
+    const auto members = std::make_shared<batch>();
     BC_POP_WARNING()
 
     // Race to first success or last failure.
-    racer->start(BIND(handle_connect, _1, _2, key, slot));
+    racer->start(BIND(handle_connect, _1, _2, key, slot, members));
 
     // Attempt to connect with unique address for each connector of batch.
     for (const auto& connector: *connectors)
-        take(family(slot), BIND(do_one, _1, _2, key, racer, connector));
+        take(family(slot), BIND(do_one, _1, _2, key, racer, members,
+            connector));
 }
 
 // Attempt to connect the given peer and invoke handle_one.
 void session_outbound::do_one(const code& ec, const config::address& peer,
-    object_key key, const race::ptr& racer,
+    object_key key, const race::ptr& racer, const batch::ptr& members,
     const connector::ptr& connector) NOEXCEPT
 {
     BC_ASSERT(stranded());
@@ -210,19 +212,32 @@ void session_outbound::do_one(const code& ec, const config::address& peer,
         return;
     }
 
-    connector->connect(peer, BIND(handle_one, _1, _2, key, peer, racer));
+    connector->connect(peer, BIND(handle_one, _1, _2, key, peer, racer,
+        members));
 }
 
 // Handle each do_one connection attempt, stopping on first success.
 void session_outbound::handle_one(const code& ec, const socket::ptr& socket,
-    object_key key, const config::address& peer,
-    const race::ptr& racer) NOEXCEPT
+    object_key key, const config::address& peer, const race::ptr& racer,
+    const batch::ptr& members) NOEXCEPT
 {
     BC_ASSERT(stranded());
     ////COUNT(events::outbound2, key);
 
-    if (ec == error::service_suspended)
-        restore(peer, BIND(handle_reclaim, _1));
+    // A failed connect as unreachable is decided once its batch completes.
+    if (!socket && batch_reclaim(ec))
+    {
+        members->unreachable.push_back(peer);
+    }
+    else
+    {
+        // Any other outcome attributes unreachable members to their address.
+        members->attributable = true;
+
+        // Without a socket, reclaim below cannot restore the address.
+        if (ec == error::service_suspended || (!socket && maybe_reclaim(ec)))
+            restore(peer, BIND(handle_reclaim, _1));
+    }
 
     // Winner in quality race is first to pass success.
     if (racer->finish(ec, socket))
@@ -238,14 +253,26 @@ void session_outbound::handle_one(const code& ec, const socket::ptr& socket,
 
 // Handle the singular batch result.
 void session_outbound::handle_connect(const code& ec,
-    const socket::ptr& socket, object_key key, size_t slot) NOEXCEPT
+    const socket::ptr& socket, object_key key, size_t slot,
+    const batch::ptr& members) NOEXCEPT
 {
     BC_ASSERT(stranded());
     ////COUNT(events::outbound3, key);
 
     // Unregister connectors, in case there was no winner.
     notify(key);
 
+    // A batch that failed only as unreachable implies a local outage, so
+    // restore its unreachable members (up to free pool capacity).
+    if (!members->attributable)
+    {
+        const auto pool = network_settings().outbound.host_pool_capacity;
+        const auto room = floored_subtract(pool, address_count());
+        const auto count = std::min(room, members->unreachable.size());
+        for (size_t index{}; index < count; ++index)
+            restore(members->unreachable.at(index), BIND(handle_reclaim, _1));
+    }
+
     // Guard restartable timer (shutdown delay).
     if (stopped())
     {
@@ -272,6 +299,9 @@ void session_outbound::handle_connect(const code& ec,
     if (ec)
     {
         if (ec == error::connect_failed ||
+            ec == error::net_unreachable ||
+            ec == error::host_unreachable ||
+            ec == error::connection_refused ||
             ec == error::operation_timeout ||
             ec == error::service_suspended ||
             ec == error::socks_failure)
@@ -352,10 +382,22 @@ inline bool session_outbound::maybe_reclaim(const code& ec) const NOEXCEPT
     if (address_count() >= network_settings().outbound.host_pool_capacity)
         return false;
 
-    // Failures that might work later (timeouts can drain pool).
+    // Failures that might work later, not attributable to the address. An
+    // address that connected is not the cause of an unreachable failure.
     return ec == error::operation_timeout
         || ec == error::channel_timeout
-        || ec == error::peer_disconnect;
+        || ec == error::peer_disconnect
+        || ec == error::insufficient_buffer
+        || batch_reclaim(ec);
+}
+
+inline bool session_outbound::batch_reclaim(const code& ec) const NOEXCEPT
+{
+    // Unreachable, from a local outage or a missing route to the address.
+    return ec == error::net_unreachable
+        || ec == error::host_unreachable
+        || ec == error::socks_net_unreachable
+        || ec == error::socks_host_unreachable;
 }
 
 inline bool session_outbound::always_reclaim(const code& ec) const NOEXCEPT

test/error.cpp

@@ -330,6 +330,33 @@ BOOST_AUTO_TEST_CASE(error_t__code__connect_failed__true_expected_message)
     BOOST_REQUIRE_EQUAL(ec.message(), "unable to reach remote host");
 }
 
+BOOST_AUTO_TEST_CASE(error_t__code__net_unreachable__true_expected_message)
+{
+    constexpr auto value = error::net_unreachable;
+    const auto ec = code(value);
+    BOOST_REQUIRE(ec);
+    BOOST_REQUIRE(ec == value);
+    BOOST_REQUIRE_EQUAL(ec.message(), "network unreachable");
+}
+
+BOOST_AUTO_TEST_CASE(error_t__code__host_unreachable__true_expected_message)
+{
+    constexpr auto value = error::host_unreachable;
+    const auto ec = code(value);
+    BOOST_REQUIRE(ec);
+    BOOST_REQUIRE(ec == value);
+    BOOST_REQUIRE_EQUAL(ec.message(), "host unreachable");
+}
+
+BOOST_AUTO_TEST_CASE(error_t__code__connection_refused__true_expected_message)
+{
+    constexpr auto value = error::connection_refused;
+    const auto ec = code(value);
+    BOOST_REQUIRE(ec);
+    BOOST_REQUIRE(ec == value);
+    BOOST_REQUIRE_EQUAL(ec.message(), "connection refused");
+}
+
 // heading read failures
 
 BOOST_AUTO_TEST_CASE(error_t__code__invalid_heading__true_expected_message)
@@ -2519,7 +2546,13 @@ BOOST_AUTO_TEST_CASE(errc_to_error_code__generic_conditions__expected)
     BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::no_buffer_space)), error::insufficient_buffer);
     BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::permission_denied)), error::not_allowed);
     BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::bad_address)), error::resolve_failed);
-    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::connection_refused)), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::connection_refused)), error::connection_refused);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::network_unreachable)), error::net_unreachable);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::network_down)), error::net_unreachable);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::address_not_available)), error::net_unreachable);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::no_link)), error::net_unreachable);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::host_unreachable)), error::host_unreachable);
+    BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::broken_pipe)), error::connect_failed);
     BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::address_in_use)), error::address_in_use);
     BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::io_error)), error::bad_stream);
     BOOST_REQUIRE_EQUAL(error::errc_to_error_code(error::to_errc_code(error::boost_errc_t::timed_out)), error::channel_timeout);
@@ -2536,7 +2569,11 @@ BOOST_AUTO_TEST_CASE(asio_to_error_code__basic_codes__expected)
     BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::operation_aborted)), error::operation_canceled);
     BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::address_family_not_supported)), error::resolve_failed);
     BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::address_in_use)), error::address_in_use);
-    BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::connection_refused)), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::connection_refused)), error::connection_refused);
+    BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::network_unreachable)), error::net_unreachable);
+    BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::network_down)), error::net_unreachable);
+    BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::host_unreachable)), error::host_unreachable);
+    BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::shut_down)), error::connect_failed);
     BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::connection_reset)), error::peer_disconnect);
     BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::message_size)), error::bad_stream);
     BOOST_REQUIRE_EQUAL(error::asio_to_error_code(error::to_asio_basic_code(error::asio_basic_error_t::invalid_argument)), error::invalid_configuration);

test/net/acceptor_sam.cpp

@@ -377,11 +377,11 @@ BOOST_FIXTURE_TEST_CASE(acceptor_sam__start__started__operation_failed, sam_setu
     BOOST_REQUIRE_EQUAL(restarted.get_future().get(), error::operation_failed);
 }
 
-BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__no_bridge__connect_failed, sam_setup_fixture)
+BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__no_bridge__connection_refused, sam_setup_fixture)
 {
     BOOST_REQUIRE_EQUAL(create(), error::success);
     listener.close();
-    BOOST_REQUIRE_EQUAL(accept().get(), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(accept().get(), error::connection_refused);
 }
 
 BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__hello_closed__peer_disconnect, sam_setup_fixture)
@@ -477,7 +477,7 @@ BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__key_path_directory__file_save, sam
     BOOST_REQUIRE_EQUAL(result.get(), error::file_save);
 }
 
-BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__forward_refused__connect_failed, sam_setup_fixture)
+BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__forward_refused__connection_refused, sam_setup_fixture)
 {
     BOOST_REQUIRE_EQUAL(create(), error::success);
     auto result = accept();
@@ -489,7 +489,7 @@ BOOST_FIXTURE_TEST_CASE(acceptor_sam__accept__forward_refused__connect_failed, s
     listener.close();
     write_line("SESSION STATUS RESULT=OK DESTINATION=" + destination);
 
-    BOOST_REQUIRE_EQUAL(result.get(), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(result.get(), error::connection_refused);
     BOOST_REQUIRE(closed(bridge));
 }
 

test/net/connector_socks.cpp

@@ -386,11 +386,11 @@ BOOST_FIXTURE_TEST_CASE(connector_socks__connect__unproxied__direct_success, soc
     BOOST_REQUIRE_EQUAL(result.get(), error::success);
 }
 
-BOOST_FIXTURE_TEST_CASE(connector_socks__connect__proxy_refused__connect_failed, socks_setup_fixture)
+BOOST_FIXTURE_TEST_CASE(connector_socks__connect__proxy_refused__connection_refused, socks_setup_fixture)
 {
     acceptor.close();
     create();
-    BOOST_REQUIRE_EQUAL(connect(config::endpoint{ SOCKS_TARGET_ENDPOINT }).get(), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(connect(config::endpoint{ SOCKS_TARGET_ENDPOINT }).get(), error::connection_refused);
 }
 
 BOOST_FIXTURE_TEST_CASE(connector_socks__connect__greeting_closed__peer_disconnect, socks_setup_fixture)

test/net/socket.cpp

@@ -369,7 +369,7 @@ BOOST_AUTO_TEST_CASE(socket__connect__bound_family_mismatch__resolve_failed)
     BOOST_REQUIRE(pool.join());
 }
 
-BOOST_AUTO_TEST_CASE(socket__connect__bound_unavailable__connect_failed)
+BOOST_AUTO_TEST_CASE(socket__connect__bound_unavailable__net_unreachable)
 {
     using namespace std::chrono_literals;
 
@@ -394,7 +394,7 @@ BOOST_AUTO_TEST_CASE(socket__connect__bound_unavailable__connect_failed)
     });
 
     BOOST_REQUIRE(connect_future.wait_for(2s) == std::future_status::ready);
-    BOOST_REQUIRE_EQUAL(connect_future.get(), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(connect_future.get(), error::net_unreachable);
 
     client->stop();
     pool.stop();
@@ -788,12 +788,12 @@ BOOST_FIXTURE_TEST_CASE(socket__async_read_some__client_sent__base_expected, acc
     instance->stop();
 }
 
-BOOST_FIXTURE_TEST_CASE(socket__connect__refused__connect_failed, acceptor_fixture)
+BOOST_FIXTURE_TEST_CASE(socket__connect__refused__connection_refused, acceptor_fixture)
 {
     const awaiter<code> connected{};
     const auto instance = connect(clear, { closed_endpoint }, connected);
 
-    BOOST_REQUIRE_EQUAL(connected.get(), error::connect_failed);
+    BOOST_REQUIRE_EQUAL(connected.get(), error::connection_refused);
     instance->stop();
 }