Try out random DNS resolved endpoints in case of a failure.

Builds/CMake/RippledCore.cmake

@@ -380,6 +380,7 @@ target_sources (rippled PRIVATE
   src/ripple/app/misc/NegativeUNLVote.cpp
   src/ripple/app/misc/NetworkOPs.cpp
   src/ripple/app/misc/SHAMapStoreImp.cpp
+  src/ripple/app/misc/detail/impl/WorkSSL.cpp
   src/ripple/app/misc/impl/AccountTxPaging.cpp
   src/ripple/app/misc/impl/AmendmentTable.cpp
   src/ripple/app/misc/impl/LoadFeeTrack.cpp
@@ -659,6 +660,7 @@ target_sources (rippled PRIVATE
   src/test/app/DeliverMin_test.cpp
   src/test/app/DepositAuth_test.cpp
   src/test/app/Discrepancy_test.cpp
+  src/test/app/DNS_test.cpp
   src/test/app/Escrow_test.cpp
   src/test/app/FeeVote_test.cpp
   src/test/app/Flow_test.cpp

src/ripple/app/misc/ValidatorSite.h

@@ -27,10 +27,12 @@
 #include <ripple/basics/StringUtilities.h>
 #include <ripple/core/Config.h>
 #include <ripple/json/json_value.h>
+
 #include <boost/asio.hpp>
-#include <boost/optional.hpp>
+
 #include <memory>
 #include <mutex>
+#include <optional>
 
 namespace ripple {
 
@@ -71,6 +73,7 @@ class ValidatorSite
 private:
     using error_code = boost::system::error_code;
     using clock_type = std::chrono::system_clock;
+    using endpoint_type = boost::asio::ip::tcp::endpoint;
 
     struct Site
     {
@@ -106,7 +109,9 @@ class ValidatorSite
         unsigned short redirCount;
         std::chrono::minutes refreshInterval;
         clock_type::time_point nextRefresh;
-        boost::optional<Status> lastRefreshStatus;
+        std::optional<Status> lastRefreshStatus;
+        endpoint_type lastRequestEndpoint;
+        bool lastRequestSuccessful;
     };
 
     Application& app_;
@@ -135,7 +140,7 @@ class ValidatorSite
 public:
     ValidatorSite(
         Application& app,
-        boost::optional<beast::Journal> j = boost::none,
+        std::optional<beast::Journal> j = std::nullopt,
         std::chrono::seconds timeout = std::chrono::seconds{20});
     ~ValidatorSite();
 
@@ -206,6 +211,7 @@ class ValidatorSite
     void
     onSiteFetch(
         boost::system::error_code const& ec,
+        endpoint_type const& endpoint,
         detail::response_type&& res,
         std::size_t siteIdx);
 

src/ripple/app/misc/detail/WorkBase.h

@@ -21,13 +21,17 @@
 #define RIPPLE_APP_MISC_DETAIL_WORKBASE_H_INCLUDED
 
 #include <ripple/app/misc/detail/Work.h>
+#include <ripple/basics/random.h>
 #include <ripple/protocol/BuildInfo.h>
+
 #include <boost/asio.hpp>
 #include <boost/beast/core/multi_buffer.hpp>
 #include <boost/beast/http/empty_body.hpp>
 #include <boost/beast/http/read.hpp>
 #include <boost/beast/http/write.hpp>
 
+#include <vector>
+
 namespace ripple {
 
 namespace detail {
@@ -37,16 +41,16 @@ class WorkBase : public Work
 {
 protected:
     using error_code = boost::system::error_code;
+    using endpoint_type = boost::asio::ip::tcp::endpoint;
 
 public:
-    using callback_type =
-        std::function<void(error_code const&, response_type&&)>;
+    using callback_type = std::function<
+        void(error_code const&, endpoint_type const&, response_type&&)>;
 
 protected:
     using socket_type = boost::asio::ip::tcp::socket;
-    using endpoint_type = boost::asio::ip::tcp::endpoint;
     using resolver_type = boost::asio::ip::tcp::resolver;
-    using query_type = resolver_type::query;
+    using results_type = boost::asio::ip::tcp::resolver::results_type;
     using request_type =
         boost::beast::http::request<boost::beast::http::empty_body>;
 
@@ -60,14 +64,18 @@ class WorkBase : public Work
     socket_type socket_;
     request_type req_;
     response_type res_;
-    boost::beast::multi_buffer read_buf_;
+    boost::beast::multi_buffer readBuf_;
+    endpoint_type lastEndpoint_;
+    bool lastStatus_;
 
 public:
     WorkBase(
         std::string const& host,
         std::string const& path,
         std::string const& port,
         boost::asio::io_service& ios,
+        endpoint_type const& lastEndpoint,
+        bool lastStatus,
         callback_type cb);
     ~WorkBase();
 
@@ -87,7 +95,7 @@ class WorkBase : public Work
     fail(error_code const& ec);
 
     void
-    onResolve(error_code const& ec, resolver_type::iterator it);
+    onResolve(error_code const& ec, results_type results);
 
     void
     onStart();
@@ -111,6 +119,8 @@ WorkBase<Impl>::WorkBase(
     std::string const& path,
     std::string const& port,
     boost::asio::io_service& ios,
+    endpoint_type const& lastEndpoint,
+    bool lastStatus,
     callback_type cb)
     : host_(host)
     , path_(path)
@@ -120,6 +130,8 @@ WorkBase<Impl>::WorkBase(
     , strand_(ios)
     , resolver_(ios)
     , socket_(ios)
+    , lastEndpoint_{lastEndpoint}
+    , lastStatus_(lastStatus)
 {
 }
 
@@ -128,6 +140,7 @@ WorkBase<Impl>::~WorkBase()
 {
     if (cb_)
         cb_(make_error_code(boost::system::errc::not_a_socket),
+            lastEndpoint_,
             std::move(res_));
     close();
 }
@@ -141,7 +154,8 @@ WorkBase<Impl>::run()
             strand_.wrap(std::bind(&WorkBase::run, impl().shared_from_this())));
 
     resolver_.async_resolve(
-        query_type{host_, port_},
+        host_,
+        port_,
         strand_.wrap(std::bind(
             &WorkBase::onResolve,
             impl().shared_from_this(),
@@ -170,20 +184,54 @@ WorkBase<Impl>::fail(error_code const& ec)
 {
     if (cb_)
     {
-        cb_(ec, std::move(res_));
+        cb_(ec, lastEndpoint_, std::move(res_));
         cb_ = nullptr;
     }
 }
 
 template <class Impl>
 void
-WorkBase<Impl>::onResolve(error_code const& ec, resolver_type::iterator it)
+WorkBase<Impl>::onResolve(error_code const& ec, results_type results)
 {
     if (ec)
         return fail(ec);
 
+    // Use last endpoint if it is successfully connected
+    // and is in the list, otherwise pick a random endpoint
+    // from the list (excluding last endpoint). If there is
+    // only one endpoint and it is the last endpoint then
+    // use the last endpoint.
+    lastEndpoint_ = [&]() -> endpoint_type {
+        int foundIndex = 0;
+        auto const foundIt = std::find_if(
+            results.begin(), results.end(), [&](endpoint_type const& e) {
+                if (e == lastEndpoint_)
+                    return true;
+                foundIndex++;
+                return false;
+            });
+        if (foundIt != results.end() && lastStatus_)
+            return lastEndpoint_;
+        else if (results.size() == 1)
+            return *results.begin();
+        else if (foundIt == results.end())
+            return *std::next(results.begin(), rand_int(results.size() - 1));
+
+        // lastEndpoint_ is part of the collection
+        // Pick a random number from the n-1 valid choices, if we use
+        // this as an index, note the last element will never be chosen
+        // and the `lastEndpoint_` index may be chosen. So when the
+        // `lastEndpoint_` index is chosen, that is treated as if the
+        // last element was chosen.
+        auto randIndex =
+            (results.size() > 2) ? rand_int(results.size() - 2) : 0;
+        if (randIndex == foundIndex)
+            randIndex = results.size() - 1;
+        return *std::next(results.begin(), randIndex);
+    }();
+
     socket_.async_connect(
-        *it,
+        lastEndpoint_,
         strand_.wrap(std::bind(
             &Impl::onConnect,
             impl().shared_from_this(),
@@ -218,7 +266,7 @@ WorkBase<Impl>::onRequest(error_code const& ec)
 
     boost::beast::http::async_read(
         impl().stream(),
-        read_buf_,
+        readBuf_,
         res_,
         strand_.wrap(std::bind(
             &WorkBase::onResponse,
@@ -235,7 +283,7 @@ WorkBase<Impl>::onResponse(error_code const& ec)
 
     close();
     assert(cb_);
-    cb_(ec, std::move(res_));
+    cb_(ec, lastEndpoint_, std::move(res_));
     cb_ = nullptr;
 }
 

src/ripple/app/misc/detail/WorkPlain.h

@@ -38,6 +38,8 @@ class WorkPlain : public WorkBase<WorkPlain>,
         std::string const& path,
         std::string const& port,
         boost::asio::io_service& ios,
+        endpoint_type const& lastEndpoint,
+        bool lastStatus,
         callback_type cb);
     ~WorkPlain() = default;
 
@@ -59,8 +61,10 @@ WorkPlain::WorkPlain(
     std::string const& path,
     std::string const& port,
     boost::asio::io_service& ios,
+    endpoint_type const& lastEndpoint,
+    bool lastStatus,
     callback_type cb)
-    : WorkBase(host, path, port, ios, cb)
+    : WorkBase(host, path, port, ios, lastEndpoint, lastStatus, cb)
 {
 }
 

src/ripple/app/misc/detail/WorkSSL.h

@@ -53,6 +53,8 @@ class WorkSSL : public WorkBase<WorkSSL>,
         boost::asio::io_service& ios,
         beast::Journal j,
         Config const& config,
+        endpoint_type const& lastEndpoint,
+        bool lastStatus,
         callback_type cb);
     ~WorkSSL() = default;
 
@@ -70,48 +72,6 @@ class WorkSSL : public WorkBase<WorkSSL>,
     onHandshake(error_code const& ec);
 };
 
-//------------------------------------------------------------------------------
-
-WorkSSL::WorkSSL(
-    std::string const& host,
-    std::string const& path,
-    std::string const& port,
-    boost::asio::io_service& ios,
-    beast::Journal j,
-    Config const& config,
-    callback_type cb)
-    : WorkBase(host, path, port, ios, cb)
-    , context_(config, j, boost::asio::ssl::context::tlsv12_client)
-    , stream_(socket_, context_.context())
-{
-    auto ec = context_.preConnectVerify(stream_, host_);
-    if (ec)
-        Throw<std::runtime_error>(
-            boost::str(boost::format("preConnectVerify: %s") % ec.message()));
-}
-
-void
-WorkSSL::onConnect(error_code const& ec)
-{
-    auto err = ec ? ec : context_.postConnectVerify(stream_, host_);
-    if (err)
-        return fail(err);
-
-    stream_.async_handshake(
-        boost::asio::ssl::stream_base::client,
-        strand_.wrap(std::bind(
-            &WorkSSL::onHandshake, shared_from_this(), std::placeholders::_1)));
-}
-
-void
-WorkSSL::onHandshake(error_code const& ec)
-{
-    if (ec)
-        return fail(ec);
-
-    onStart();
-}
-
 }  // namespace detail
 
 }  // namespace ripple

src/ripple/app/misc/detail/impl/WorkSSL.cpp

@@ -0,0 +1,69 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2020 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <ripple/app/misc/detail/WorkSSL.h>
+
+namespace ripple {
+namespace detail {
+
+WorkSSL::WorkSSL(
+    std::string const& host,
+    std::string const& path,
+    std::string const& port,
+    boost::asio::io_service& ios,
+    beast::Journal j,
+    Config const& config,
+    endpoint_type const& lastEndpoint,
+    bool lastStatus,
+    callback_type cb)
+    : WorkBase(host, path, port, ios, lastEndpoint, lastStatus, cb)
+    , context_(config, j, boost::asio::ssl::context::tlsv12_client)
+    , stream_(socket_, context_.context())
+{
+    auto ec = context_.preConnectVerify(stream_, host_);
+    if (ec)
+        Throw<std::runtime_error>(
+            boost::str(boost::format("preConnectVerify: %s") % ec.message()));
+}
+
+void
+WorkSSL::onConnect(error_code const& ec)
+{
+    auto err = ec ? ec : context_.postConnectVerify(stream_, host_);
+    if (err)
+        return fail(err);
+
+    stream_.async_handshake(
+        boost::asio::ssl::stream_base::client,
+        strand_.wrap(std::bind(
+            &WorkSSL::onHandshake, shared_from_this(), std::placeholders::_1)));
+}
+
+void
+WorkSSL::onHandshake(error_code const& ec)
+{
+    if (ec)
+        return fail(ec);
+
+    onStart();
+}
+
+}  // namespace detail
+
+}  // namespace ripple