logging: fix delegating lock sink races

This fixes two different issues:
1) Previously there was no locking around log sink replacement,
   so it was possibles for a link sink to get removed by one
   thread while getting written to be another thread.
2) Even with locking, the base class destructor pattern would
   do the swap after the derived class was destroyed, leading to
   undefined behavior.

This was easy to reproduce in cx_limit_integration_test but is
an issue anywhere the log expectations are used or death test
stderr workaround for coverage which has been removed because
it no longer logs to a file.

Fixes #11841

Signed-off-by: Matt Klein <mklein@lyft.com>

source/common/common/logger.cc

@@ -19,18 +19,32 @@ namespace Logger {
 StandardLogger::StandardLogger(const std::string& name)
     : Logger(std::make_shared<spdlog::logger>(name, Registry::getSink())) {}
 
-SinkDelegate::SinkDelegate(DelegatingLogSinkSharedPtr log_sink)
-    : previous_delegate_(log_sink->delegate()), log_sink_(log_sink) {
-  log_sink->setDelegate(this);
-}
+SinkDelegate::SinkDelegate(DelegatingLogSinkSharedPtr log_sink) : log_sink_(log_sink) {}
 
 SinkDelegate::~SinkDelegate() {
-  assert(log_sink_->delegate() == this); // Ensures stacked allocation of delegates.
+  // The previous delegate should have never been set or should have been reset by now via
+  // restoreDelegate();
+  assert(previous_delegate_ == nullptr);
+}
+
+void SinkDelegate::setDelegate() {
+  // There should be no previous delegate before this call.
+  assert(previous_delegate_ == nullptr);
+  previous_delegate_ = log_sink_->delegate();
+  log_sink_->setDelegate(this);
+}
+
+void SinkDelegate::restoreDelegate() {
+  // Ensures stacked allocation of delegates.
+  assert(log_sink_->delegate() == this);
   log_sink_->setDelegate(previous_delegate_);
+  previous_delegate_ = nullptr;
 }
 
 StderrSinkDelegate::StderrSinkDelegate(DelegatingLogSinkSharedPtr log_sink)
-    : SinkDelegate(log_sink) {}
+    : SinkDelegate(log_sink) {
+  setDelegate();
+}
 
 void StderrSinkDelegate::log(absl::string_view msg) {
   Thread::OptionalLockGuard guard(lock_);
@@ -60,6 +74,9 @@ void DelegatingLogSink::log(const spdlog::details::log_msg& msg) {
   }
   lock.Release();
 
+  // Hold the sink mutex while performing the actual logging. This prevents the sink from being
+  // swapped during an individual log event.
+  absl::ReaderMutexLock sink_lock(&sink_mutex_);
   if (should_escape_) {
     sink_->log(escapeLogLine(msg_view));
   } else {

source/common/common/logger.h

@@ -104,10 +104,21 @@ class SinkDelegate : NonCopyable {
   virtual void flush() PURE;
 
 protected:
+  // Swap the current log sink delegate for this one. This should be called by the derived class
+  // constructor immediately before returning. This is required to match restoreDelegate(),
+  // otherwise it's possible for the previous delegate to get set in the base class constructor,
+  // the derived class constructor throws, and cleanup becomes broken.
+  void setDelegate();
+
+  // Swap the current log sink (this) for the previous one. This should be called by the derived
+  // class destructor in the body. This is critical as otherwise it's possible for a log message
+  // to get routed to a partially destructed sink.
+  void restoreDelegate();
+
   SinkDelegate* previousDelegate() { return previous_delegate_; }
 
 private:
-  SinkDelegate* previous_delegate_;
+  SinkDelegate* previous_delegate_{nullptr};
   DelegatingLogSinkSharedPtr log_sink_;
 };
 
@@ -141,7 +152,10 @@ class DelegatingLogSink : public spdlog::sinks::sink {
 
   // spdlog::sinks::sink
   void log(const spdlog::details::log_msg& msg) override;
-  void flush() override { sink_->flush(); }
+  void flush() override {
+    absl::ReaderMutexLock lock(&sink_mutex_);
+    sink_->flush();
+  }
   void set_pattern(const std::string& pattern) override {
     set_formatter(spdlog::details::make_unique<spdlog::pattern_formatter>(pattern));
   }
@@ -180,13 +194,20 @@ class DelegatingLogSink : public spdlog::sinks::sink {
 
   DelegatingLogSink() = default;
 
-  void setDelegate(SinkDelegate* sink) { sink_ = sink; }
-  SinkDelegate* delegate() { return sink_; }
+  void setDelegate(SinkDelegate* sink) {
+    absl::WriterMutexLock lock(&sink_mutex_);
+    sink_ = sink;
+  }
+  SinkDelegate* delegate() {
+    absl::ReaderMutexLock lock(&sink_mutex_);
+    return sink_;
+  }
 
-  SinkDelegate* sink_{nullptr};
+  SinkDelegate* sink_ ABSL_GUARDED_BY(sink_mutex_){nullptr};
+  absl::Mutex sink_mutex_;
   std::unique_ptr<StderrSinkDelegate> stderr_sink_; // Builtin sink to use as a last resort.
   std::unique_ptr<spdlog::formatter> formatter_ ABSL_GUARDED_BY(format_mutex_);
-  absl::Mutex format_mutex_; // direct absl reference to break build cycle.
+  absl::Mutex format_mutex_;
   bool should_escape_{false};
 };
 

source/common/common/logger_delegates.cc

@@ -13,7 +13,11 @@ namespace Logger {
 FileSinkDelegate::FileSinkDelegate(const std::string& log_path,
                                    AccessLog::AccessLogManager& log_manager,
                                    DelegatingLogSinkSharedPtr log_sink)
-    : SinkDelegate(log_sink), log_file_(log_manager.createAccessLog(log_path)) {}
+    : SinkDelegate(log_sink), log_file_(log_manager.createAccessLog(log_path)) {
+  setDelegate();
+}
+
+FileSinkDelegate::~FileSinkDelegate() { restoreDelegate(); }
 
 void FileSinkDelegate::log(absl::string_view msg) {
   // Log files have internal locking to ensure serial, non-interleaved

source/common/common/logger_delegates.h

@@ -14,16 +14,14 @@
 namespace Envoy {
 namespace Logger {
 
-class DelegatingLogSink;
-using DelegatingLogSinkSharedPtr = std::shared_ptr<DelegatingLogSink>;
-
 /**
  * SinkDelegate that writes log messages to a file.
  */
 class FileSinkDelegate : public SinkDelegate {
 public:
   FileSinkDelegate(const std::string& log_path, AccessLog::AccessLogManager& log_manager,
                    DelegatingLogSinkSharedPtr log_sink);
+  ~FileSinkDelegate();
 
   // SinkDelegate
   void log(absl::string_view msg) override;

test/common/buffer/owned_impl_test.cc

@@ -1034,7 +1034,6 @@ TEST_F(OwnedImplTest, ReadReserveAndCommit) {
 }
 
 TEST(OverflowDetectingUInt64, Arithmetic) {
-  Logger::StderrSinkDelegate stderr_sink(Logger::Registry::getSink()); // For coverage build.
   OverflowDetectingUInt64 length;
   length += 1;
   length -= 1;

test/common/common/assert_test.cc

@@ -7,7 +7,6 @@
 namespace Envoy {
 
 TEST(ReleaseAssertDeathTest, VariousLogs) {
-  Logger::StderrSinkDelegate stderr_sink(Logger::Registry::getSink()); // For coverage build.
   EXPECT_DEATH({ RELEASE_ASSERT(0, ""); }, ".*assert failure: 0.*");
   EXPECT_DEATH({ RELEASE_ASSERT(0, "With some logs"); },
                ".*assert failure: 0. Details: With some logs.*");

test/common/http/header_map_impl_test.cc

@@ -1005,14 +1005,14 @@ TEST(HeaderMapImplTest, TestAppendHeader) {
 TEST(TestHeaderMapImplDeathTest, TestHeaderLengthChecks) {
   HeaderString value;
   value.setCopy("some;");
-  EXPECT_DEATH_LOG_TO_STDERR(value.append(nullptr, std::numeric_limits<uint32_t>::max()),
-                             "Trying to allocate overly large headers.");
+  EXPECT_DEATH(value.append(nullptr, std::numeric_limits<uint32_t>::max()),
+               "Trying to allocate overly large headers.");
 
   std::string source("hello");
   HeaderString reference;
   reference.setReference(source);
-  EXPECT_DEATH_LOG_TO_STDERR(reference.append(nullptr, std::numeric_limits<uint32_t>::max()),
-                             "Trying to allocate overly large headers.");
+  EXPECT_DEATH(reference.append(nullptr, std::numeric_limits<uint32_t>::max()),
+               "Trying to allocate overly large headers.");
 }
 
 TEST(HeaderMapImplTest, PseudoHeaderOrder) {

test/common/http/http2/metadata_encoder_decoder_test.cc

@@ -333,7 +333,6 @@ using MetadataEncoderDecoderDeathTest = MetadataEncoderDecoderTest;
 
 // Crash if a caller tries to pack more frames than the encoder has data for.
 TEST_F(MetadataEncoderDecoderDeathTest, PackTooManyFrames) {
-  Logger::StderrSinkDelegate stderr_sink(Logger::Registry::getSink()); // For coverage build.
   MetadataMap metadata_map = {
       {"header_key1", std::string(5, 'a')},
       {"header_key2", std::string(5, 'b')},

test/common/network/address_impl_test.cc

@@ -448,9 +448,9 @@ TEST(AddressFromSockAddrDeathTest, IPv4) {
   EXPECT_EQ(1, inet_pton(AF_INET, "1.2.3.4", &sin.sin_addr));
   sin.sin_port = htons(6502);
 
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, 1), "ss_len");
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, sizeof(sockaddr_in) - 1), "ss_len");
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, sizeof(sockaddr_in) + 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, sizeof(sockaddr_in) - 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, sizeof(sockaddr_in) + 1), "ss_len");
 
   EXPECT_EQ("1.2.3.4:6502", addressFromSockAddr(ss, sizeof(sockaddr_in))->asString());
 
@@ -467,9 +467,9 @@ TEST(AddressFromSockAddrDeathTest, IPv6) {
   EXPECT_EQ(1, inet_pton(AF_INET6, "01:023::00Ef", &sin6.sin6_addr));
   sin6.sin6_port = htons(32000);
 
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, 1), "ss_len");
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, sizeof(sockaddr_in6) - 1), "ss_len");
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, sizeof(sockaddr_in6) + 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, sizeof(sockaddr_in6) - 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, sizeof(sockaddr_in6) + 1), "ss_len");
 
   EXPECT_EQ("[1:23::ef]:32000", addressFromSockAddr(ss, sizeof(sockaddr_in6))->asString());
 
@@ -490,9 +490,8 @@ TEST(AddressFromSockAddrDeathTest, Pipe) {
 
   StringUtil::strlcpy(sun.sun_path, "/some/path", sizeof sun.sun_path);
 
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, 1), "ss_len");
-  EXPECT_DEATH_LOG_TO_STDERR(addressFromSockAddr(ss, offsetof(struct sockaddr_un, sun_path)),
-                             "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, 1), "ss_len");
+  EXPECT_DEATH(addressFromSockAddr(ss, offsetof(struct sockaddr_un, sun_path)), "ss_len");
 
   socklen_t ss_len = offsetof(struct sockaddr_un, sun_path) + 1 + strlen(sun.sun_path);
   EXPECT_EQ("/some/path", addressFromSockAddr(ss, ss_len)->asString());

test/common/network/connection_impl_test.cc

@@ -85,7 +85,7 @@ TEST_P(ConnectionImplDeathTest, BadFd) {
   Event::DispatcherPtr dispatcher(api->allocateDispatcher("test_thread"));
   IoHandlePtr io_handle = std::make_unique<IoSocketHandleImpl>();
   StreamInfo::StreamInfoImpl stream_info(dispatcher->timeSource());
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       ConnectionImpl(*dispatcher,
                      std::make_unique<ConnectionSocketImpl>(std::move(io_handle), nullptr, nullptr),
                      Network::Test::createRawBufferSocket(), stream_info, false),

test/common/network/listener_impl_test.cc

@@ -59,7 +59,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ListenerImplDeathTest,
                          testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
                          TestUtility::ipTestParamsToString);
 TEST_P(ListenerImplDeathTest, ErrorCallback) {
-  EXPECT_DEATH_LOG_TO_STDERR(errorCallbackTest(GetParam()), ".*listener accept failure.*");
+  EXPECT_DEATH(errorCallbackTest(GetParam()), ".*listener accept failure.*");
 }
 
 class TestListenerImpl : public ListenerImpl {

test/common/network/udp_listener_impl_test.cc

@@ -390,7 +390,6 @@ TEST_P(UdpListenerImplTest, SendData) {
  * The send fails because the server_socket is created with bind=false.
  */
 TEST_P(UdpListenerImplTest, SendDataError) {
-  Logger::StderrSinkDelegate stderr_sink(Logger::Registry::getSink()); // For coverage build.
   const std::string payload("hello world");
   Buffer::InstancePtr buffer(new Buffer::OwnedImpl());
   buffer->add(payload);

test/common/signal/signals_test.cc

@@ -26,7 +26,7 @@ namespace Envoy {
 #ifndef ASANITIZED
 TEST(SignalsDeathTest, InvalidAddressDeathTest) {
   SignalAction actions;
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       []() -> void {
         // Oops!
         volatile int* nasty_ptr = reinterpret_cast<int*>(0x0);
@@ -44,7 +44,7 @@ TEST(SignalsDeathTest, RegisteredHandlerTest) {
   SignalAction::registerFatalErrorHandler(handler);
   SignalAction actions;
   // Make sure the fatal error log "HERE" registered above is logged on fatal error.
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       []() -> void {
         // Oops!
         volatile int* nasty_ptr = reinterpret_cast<int*>(0x0);
@@ -56,7 +56,7 @@ TEST(SignalsDeathTest, RegisteredHandlerTest) {
 
 TEST(SignalsDeathTest, BusDeathTest) {
   SignalAction actions;
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       []() -> void {
         // Bus error is tricky. There's one way that can work on POSIX systems
         // described below but it depends on mmaping a file. Just make it easy and
@@ -72,7 +72,7 @@ TEST(SignalsDeathTest, BusDeathTest) {
 
 TEST(SignalsDeathTest, BadMathDeathTest) {
   SignalAction actions;
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       []() -> void {
         // It turns out to be really hard to not have the optimizer get rid of a
         // division by zero. Just raise the signal for this test.
@@ -85,7 +85,7 @@ TEST(SignalsDeathTest, BadMathDeathTest) {
 // Unfortunately we don't have a reliable way to do this on other platforms
 TEST(SignalsDeathTest, IllegalInstructionDeathTest) {
   SignalAction actions;
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       []() -> void {
         // Intel defines the "ud2" opcode to be an invalid instruction:
         __asm__("ud2");
@@ -96,7 +96,7 @@ TEST(SignalsDeathTest, IllegalInstructionDeathTest) {
 
 TEST(SignalsDeathTest, AbortDeathTest) {
   SignalAction actions;
-  EXPECT_DEATH_LOG_TO_STDERR([]() -> void { abort(); }(), "backtrace.*Abort(ed)?");
+  EXPECT_DEATH([]() -> void { abort(); }(), "backtrace.*Abort(ed)?");
 }
 
 TEST(SignalsDeathTest, RestoredPreviousHandlerDeathTest) {
@@ -108,7 +108,7 @@ TEST(SignalsDeathTest, RestoredPreviousHandlerDeathTest) {
     // goes out of scope, NOT the default.
   }
   // Outer SignalAction should be active again:
-  EXPECT_DEATH_LOG_TO_STDERR([]() -> void { abort(); }(), "backtrace.*Abort(ed)?");
+  EXPECT_DEATH([]() -> void { abort(); }(), "backtrace.*Abort(ed)?");
 }
 
 #endif

test/common/singleton/manager_impl_test.cc

@@ -18,8 +18,7 @@ static void deathTestWorker() {
 }
 
 TEST(SingletonManagerImplDeathTest, NotRegistered) {
-  EXPECT_DEATH_LOG_TO_STDERR(deathTestWorker(),
-                             "invalid singleton name 'foo'. Make sure it is registered.");
+  EXPECT_DEATH(deathTestWorker(), "invalid singleton name 'foo'. Make sure it is registered.");
 }
 
 SINGLETON_MANAGER_REGISTRATION(test);

test/common/upstream/conn_pool_map_impl_test.cc

@@ -401,8 +401,8 @@ TEST_F(ConnPoolMapImplDeathTest, ReentryClearTripsAssert) {
   ON_CALL(*mock_pools_[0], addDrainedCallback(_))
       .WillByDefault(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); }));
 
-  EXPECT_DEATH_LOG_TO_STDERR(test_map->addDrainedCallback([&test_map] { test_map->clear(); }),
-                             ".*Details: A resource should only be entered once");
+  EXPECT_DEATH(test_map->addDrainedCallback([&test_map] { test_map->clear(); }),
+               ".*Details: A resource should only be entered once");
 }
 
 TEST_F(ConnPoolMapImplDeathTest, ReentryGetPoolTripsAssert) {
@@ -412,7 +412,7 @@ TEST_F(ConnPoolMapImplDeathTest, ReentryGetPoolTripsAssert) {
   ON_CALL(*mock_pools_[0], addDrainedCallback(_))
       .WillByDefault(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); }));
 
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       test_map->addDrainedCallback([&test_map, this] { test_map->getPool(2, getBasicFactory()); }),
       ".*Details: A resource should only be entered once");
 }
@@ -424,9 +424,8 @@ TEST_F(ConnPoolMapImplDeathTest, ReentryDrainConnectionsTripsAssert) {
   ON_CALL(*mock_pools_[0], addDrainedCallback(_))
       .WillByDefault(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); }));
 
-  EXPECT_DEATH_LOG_TO_STDERR(
-      test_map->addDrainedCallback([&test_map] { test_map->drainConnections(); }),
-      ".*Details: A resource should only be entered once");
+  EXPECT_DEATH(test_map->addDrainedCallback([&test_map] { test_map->drainConnections(); }),
+               ".*Details: A resource should only be entered once");
 }
 
 TEST_F(ConnPoolMapImplDeathTest, ReentryAddDrainedCallbackTripsAssert) {
@@ -436,9 +435,8 @@ TEST_F(ConnPoolMapImplDeathTest, ReentryAddDrainedCallbackTripsAssert) {
   ON_CALL(*mock_pools_[0], addDrainedCallback(_))
       .WillByDefault(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); }));
 
-  EXPECT_DEATH_LOG_TO_STDERR(
-      test_map->addDrainedCallback([&test_map] { test_map->addDrainedCallback([]() {}); }),
-      ".*Details: A resource should only be entered once");
+  EXPECT_DEATH(test_map->addDrainedCallback([&test_map] { test_map->addDrainedCallback([]() {}); }),
+               ".*Details: A resource should only be entered once");
 }
 #endif // !defined(NDEBUG)
 

test/exe/main_common_test.cc

@@ -161,7 +161,7 @@ TEST_P(MainCommonDeathTest, OutOfMemoryHandler) {
   // so disable handling that signal.
   signal(SIGABRT, SIG_DFL);
 #endif
-  EXPECT_DEATH_LOG_TO_STDERR(
+  EXPECT_DEATH(
       []() {
         // Allocating a fixed-size large array that results in OOM on gcc
         // results in a compile-time error on clang of "array size too big",

test/exe/terminate_handler_test.cc

@@ -8,7 +8,7 @@ namespace Envoy {
 
 TEST(TerminateHandlerDeathTest, HandlerInstalledTest) {
   TerminateHandler handler;
-  EXPECT_DEATH_LOG_TO_STDERR([]() -> void { std::terminate(); }(), ".*std::terminate called!.*");
+  EXPECT_DEATH([]() -> void { std::terminate(); }(), ".*std::terminate called!.*");
 }
 
 } // namespace Envoy