server: add --enable-core-dump flag (#15245)

This ensures Envoy can core dump when the dumpability bit might have
been unset (e.g.: running inside a container with fewer capabilities
than the ones Envoy itself has).

Fixes #15242.

Author: Raúl Gutiérrez Segalés

api/envoy/admin/v3/server_info.proto

@@ -58,7 +58,7 @@ message ServerInfo {
   config.core.v3.Node node = 7;
 }
 
-// [#next-free-field: 37]
+// [#next-free-field: 38]
 message CommandLineOptions {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.admin.v2alpha.CommandLineOptions";
@@ -189,4 +189,7 @@ message CommandLineOptions {
 
   // See :option:`--socket-mode` for details.
   uint32 socket_mode = 36;
+
+  // See :option:`--enable-core-dump` for details.
+  bool enable_core_dump = 37;
 }

api/envoy/admin/v4alpha/server_info.proto

@@ -8,7 +8,9 @@ load(
     _envoy_basic_cc_library = "envoy_basic_cc_library",
     _envoy_cc_extension = "envoy_cc_extension",
     _envoy_cc_library = "envoy_cc_library",
+    _envoy_cc_linux_library = "envoy_cc_linux_library",
     _envoy_cc_posix_library = "envoy_cc_posix_library",
+    _envoy_cc_posix_without_linux_library = "envoy_cc_posix_without_linux_library",
     _envoy_cc_win32_library = "envoy_cc_win32_library",
     _envoy_include_prefix = "envoy_include_prefix",
     _envoy_proto_library = "envoy_proto_library",
@@ -145,6 +147,16 @@ def envoy_cc_platform_dep(name):
         "//conditions:default": [name + "_posix"],
     })
 
+# Used to select a dependency that has different implementations on Linux vs rest of POSIX vs Windows.
+# The platform-specific implementations should be specified with envoy_cc_linux_library,
+# envoy_cc_posix_without_library and envoy_cc_win32_library respectively
+def envoy_cc_platform_specific_dep(name):
+    return select({
+        "@envoy//bazel:windows_x86_64": [name + "_win32"],
+        "@envoy//bazel:linux": [name + "_linux"],
+        "//conditions:default": [name + "_posix"],
+    })
+
 # Envoy proto descriptor targets should be specified with this function.
 # This is used for testing only.
 def envoy_proto_descriptor(name, out, srcs = [], external_deps = []):
@@ -202,7 +214,9 @@ envoy_cc_binary = _envoy_cc_binary
 envoy_basic_cc_library = _envoy_basic_cc_library
 envoy_cc_extension = _envoy_cc_extension
 envoy_cc_library = _envoy_cc_library
+envoy_cc_linux_library = _envoy_cc_linux_library
 envoy_cc_posix_library = _envoy_cc_posix_library
+envoy_cc_posix_without_linux_library = _envoy_cc_posix_without_linux_library
 envoy_cc_win32_library = _envoy_cc_win32_library
 envoy_include_prefix = _envoy_include_prefix
 envoy_proto_library = _envoy_proto_library

bazel/envoy_build_system.bzl

@@ -220,6 +220,38 @@ def envoy_cc_posix_library(name, srcs = [], hdrs = [], **kargs):
         **kargs
     )
 
+# Used to specify a library that only builds on POSIX excluding Linux
+def envoy_cc_posix_without_linux_library(name, srcs = [], hdrs = [], **kargs):
+    envoy_cc_library(
+        name = name + "_posix",
+        srcs = select({
+            "@envoy//bazel:windows_x86_64": [],
+            "@envoy//bazel:linux": [],
+            "//conditions:default": srcs,
+        }),
+        hdrs = select({
+            "@envoy//bazel:windows_x86_64": [],
+            "@envoy//bazel:linux": [],
+            "//conditions:default": hdrs,
+        }),
+        **kargs
+    )
+
+# Used to specify a library that only builds on Linux
+def envoy_cc_linux_library(name, srcs = [], hdrs = [], **kargs):
+    envoy_cc_library(
+        name = name + "_linux",
+        srcs = select({
+            "@envoy//bazel:linux": srcs,
+            "//conditions:default": [],
+        }),
+        hdrs = select({
+            "@envoy//bazel:linux": hdrs,
+            "//conditions:default": [],
+        }),
+        **kargs
+    )
+
 # Used to specify a library that only builds on Windows
 def envoy_cc_win32_library(name, srcs = [], hdrs = [], **kargs):
     envoy_cc_library(

bazel/envoy_library.bzl

@@ -351,3 +351,10 @@ following are the command line options that Envoy supports.
   * build mode - either ``RELEASE`` or ``DEBUG``,
 
   * TLS library - either ``BoringSSL`` or ``BoringSSL-FIPS``.
+
+.. option:: --enable-core-dump
+
+  *(optional)* This flag is intended for Linux-based systems and it's a no-op for all other platforms.
+  It enables core dumps by invoking `prctl <https://man7.org/linux/man-pages/man2/prctl.2.html>`_ using the
+  PR_SET_DUMPABLE option. This is useful for container environments when using capabilities, given that when
+  Envoy has more capabilities than its base environment core dumping will be disabled by the kernel.

docs/root/operations/cli.rst

@@ -125,6 +125,7 @@ New Features
 * route config: added :ref:`allow_post field <envoy_v3_api_field_config.route.v3.RouteAction.UpgradeConfig.ConnectConfig.allow_post>` for allowing POST payload as raw TCP.
 * route config: added :ref:`max_direct_response_body_size_bytes <envoy_v3_api_field_config.route.v3.RouteConfiguration.max_direct_response_body_size_bytes>` to set maximum :ref:`direct response body <envoy_v3_api_field_config.route.v3.DirectResponseAction.body>` size in bytes. If not specified the default remains 4096 bytes.
 * server: added *fips_mode* to :ref:`server compilation settings <server_compilation_settings_statistics>` related statistic.
+* server: added :option:`--enable-core-dump` flag to enable core dumps via prctl (Linux-based systems only).
 * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation <tunneling-tcp-over-http>` for details.
 * tcp_proxy: added a :ref:`use_post field <envoy_v3_api_field_extensions.filters.network.tcp_proxy.v3.TcpProxy.TunnelingConfig.use_post>` for using HTTP POST to proxy TCP streams.
 * tcp_proxy: added a :ref:`headers_to_add field <envoy_v3_api_field_extensions.filters.network.tcp_proxy.v3.TcpProxy.TunnelingConfig.headers_to_add>` for setting additional headers to the HTTP requests for TCP proxing.

docs/root/version_history/current.rst

@@ -234,6 +234,11 @@ class Options {
    */
   virtual bool mutexTracingEnabled() const PURE;
 
+  /**
+   * @return bool indicating whether core dumps have been enabled.
+   */
+  virtual bool coreDumpEnabled() const PURE;
+
   /**
    * @return bool indicating whether cpuset size should determine the number of worker threads.
    */

generated_api_shadow/envoy/admin/v3/server_info.proto

@@ -2,8 +2,9 @@ load(
     "//bazel:envoy_build_system.bzl",
     "envoy_cc_binary",
     "envoy_cc_library",
-    "envoy_cc_platform_dep",
-    "envoy_cc_posix_library",
+    "envoy_cc_linux_library",
+    "envoy_cc_platform_specific_dep",
+    "envoy_cc_posix_without_linux_library",
     "envoy_cc_win32_library",
     "envoy_package",
 )
@@ -142,7 +143,7 @@ envoy_cc_library(
 envoy_cc_library(
     name = "platform_impl_lib",
     deps = [":platform_header_lib"] +
-           envoy_cc_platform_dep("platform_impl_lib"),
+           envoy_cc_platform_specific_dep("platform_impl_lib"),
 )
 
 envoy_cc_library(
@@ -154,7 +155,7 @@ envoy_cc_library(
     ],
 )
 
-envoy_cc_posix_library(
+envoy_cc_posix_without_linux_library(
     name = "platform_impl_lib",
     srcs = ["posix/platform_impl.cc"],
     deps = [
@@ -164,6 +165,16 @@ envoy_cc_posix_library(
     ],
 )
 
+envoy_cc_linux_library(
+    name = "platform_impl_lib",
+    srcs = ["linux/platform_impl.cc"],
+    deps = [
+        ":platform_header_lib",
+        "//source/common/common:thread_lib",
+        "//source/common/filesystem:filesystem_lib",
+    ],
+)
+
 envoy_cc_win32_library(
     name = "platform_impl_lib",
     srcs = ["win32/platform_impl.cc"],

generated_api_shadow/envoy/admin/v4alpha/server_info.proto

@@ -0,0 +1,22 @@
+#if !defined(__linux__)
+#error "Linux platform file is part of non-Linux build."
+#endif
+
+#include <sys/prctl.h>
+
+#include "common/common/thread_impl.h"
+#include "common/filesystem/filesystem_impl.h"
+
+#include "exe/platform_impl.h"
+
+namespace Envoy {
+
+PlatformImpl::PlatformImpl()
+    : thread_factory_(std::make_unique<Thread::ThreadFactoryImplPosix>()),
+      file_system_(std::make_unique<Filesystem::InstanceImplPosix>()) {}
+
+PlatformImpl::~PlatformImpl() = default;
+
+bool PlatformImpl::enableCoreDump() { return prctl(PR_SET_DUMPABLE, 1) != -1; }
+
+} // namespace Envoy

include/envoy/server/options.h

@@ -45,16 +45,25 @@ Runtime::LoaderPtr ProdComponentFactory::createRuntime(Server::Instance& server,
 MainCommonBase::MainCommonBase(const Server::Options& options, Event::TimeSystem& time_system,
                                ListenerHooks& listener_hooks,
                                Server::ComponentFactory& component_factory,
+                               std::unique_ptr<PlatformImpl> platform_impl,
                                std::unique_ptr<Random::RandomGenerator>&& random_generator,
-                               Thread::ThreadFactory& thread_factory,
-                               Filesystem::Instance& file_system,
                                std::unique_ptr<ProcessContext> process_context)
-    : options_(options), component_factory_(component_factory), thread_factory_(thread_factory),
-      file_system_(file_system), stats_allocator_(symbol_table_) {
+    : platform_impl_(std::move(platform_impl)), options_(options),
+      component_factory_(component_factory), stats_allocator_(symbol_table_) {
   // Process the option to disable extensions as early as possible,
   // before we do any configuration loading.
   OptionsImpl::disableExtensions(options.disabledExtensions());
 
+  // Enable core dumps as early as possible.
+  if (options_.coreDumpEnabled()) {
+    const auto ret = platform_impl_->enableCoreDump();
+    if (ret) {
+      ENVOY_LOG_MISC(info, "core dump enabled");
+    } else {
+      ENVOY_LOG_MISC(warn, "failed to enable core dump");
+    }
+  }
+
   switch (options_.mode()) {
   case Server::Mode::InitOnly:
   case Server::Mode::Serve: {
@@ -79,7 +88,7 @@ MainCommonBase::MainCommonBase(const Server::Options& options, Event::TimeSystem
     server_ = std::make_unique<Server::InstanceImpl>(
         *init_manager_, options_, time_system, local_address, listener_hooks, *restarter_,
         *stats_store_, access_log_lock, component_factory, std::move(random_generator), *tls_,
-        thread_factory_, file_system_, std::move(process_context));
+        platform_impl_->threadFactory(), platform_impl_->fileSystem(), std::move(process_context));
 
     break;
   }
@@ -163,8 +172,8 @@ bool MainCommonBase::run() {
     return true;
   case Server::Mode::Validate: {
     auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion());
-    return Server::validateConfig(options_, local_address, component_factory_, thread_factory_,
-                                  file_system_);
+    return Server::validateConfig(options_, local_address, component_factory_,
+                                  platform_impl_->threadFactory(), platform_impl_->fileSystem());
   }
   case Server::Mode::InitOnly:
     PERF_DUMP();
@@ -188,14 +197,14 @@ void MainCommonBase::adminRequest(absl::string_view path_and_query, absl::string
 MainCommon::MainCommon(const std::vector<std::string>& args)
     : options_(args, &MainCommon::hotRestartVersion, spdlog::level::info),
       base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_,
-            std::make_unique<Random::RandomGeneratorImpl>(), platform_impl_.threadFactory(),
-            platform_impl_.fileSystem(), nullptr) {}
+            std::make_unique<PlatformImpl>(), std::make_unique<Random::RandomGeneratorImpl>(),
+            nullptr) {}
 
 MainCommon::MainCommon(int argc, const char* const* argv)
     : options_(argc, argv, &MainCommon::hotRestartVersion, spdlog::level::info),
       base_(options_, real_time_system_, default_listener_hooks_, prod_component_factory_,
-            std::make_unique<Random::RandomGeneratorImpl>(), platform_impl_.threadFactory(),
-            platform_impl_.fileSystem(), nullptr) {}
+            std::make_unique<PlatformImpl>(), std::make_unique<Random::RandomGeneratorImpl>(),
+            nullptr) {}
 
 std::string MainCommon::hotRestartVersion(bool hot_restart_enabled) {
 #ifdef ENVOY_HOT_RESTART

source/exe/BUILD

@@ -38,8 +38,8 @@ class MainCommonBase {
   // destructed.
   MainCommonBase(const Server::Options& options, Event::TimeSystem& time_system,
                  ListenerHooks& listener_hooks, Server::ComponentFactory& component_factory,
+                 std::unique_ptr<PlatformImpl> platform_impl,
                  std::unique_ptr<Random::RandomGenerator>&& random_generator,
-                 Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system,
                  std::unique_ptr<ProcessContext> process_context);
 
   bool run();
@@ -66,15 +66,14 @@ class MainCommonBase {
                     const AdminRequestFn& handler);
 
 protected:
+  std::unique_ptr<PlatformImpl> platform_impl_;
   ProcessWide process_wide_; // Process-wide state setup/teardown (excluding grpc).
   // We instantiate this class regardless of ENVOY_GOOGLE_GRPC, to avoid having
   // an ifdef in a header file exposed in a C++ library. It is too easy to have
   // the ifdef be inconsistent across build-system boundaries.
   Grpc::GoogleGrpcContext google_grpc_context_;
   const Envoy::Server::Options& options_;
   Server::ComponentFactory& component_factory_;
-  Thread::ThreadFactory& thread_factory_;
-  Filesystem::Instance& file_system_;
   Stats::SymbolTableImpl symbol_table_;
   Stats::AllocatorImpl stats_allocator_;
 
@@ -146,7 +145,6 @@ class MainCommon {
   Envoy::TerminateHandler log_on_terminate_;
 #endif
 
-  PlatformImpl platform_impl_;
   Envoy::OptionsImpl options_;
   Event::RealTimeSystem real_time_system_;
   DefaultListenerHooks default_listener_hooks_;

source/exe/linux/platform_impl.cc

@@ -8,9 +8,10 @@ namespace Envoy {
 class PlatformImpl {
 public:
   PlatformImpl();
-  ~PlatformImpl();
+  virtual ~PlatformImpl();
   Thread::ThreadFactory& threadFactory() { return *thread_factory_; }
   Filesystem::Instance& fileSystem() { return *file_system_; }
+  virtual bool enableCoreDump();
 
 private:
   Thread::ThreadFactoryPtr thread_factory_;

source/exe/main_common.cc

@@ -11,4 +11,6 @@ PlatformImpl::PlatformImpl()
 
 PlatformImpl::~PlatformImpl() = default;
 
+bool PlatformImpl::enableCoreDump() { return false; }
+
 } // namespace Envoy

source/exe/main_common.h

@@ -58,4 +58,6 @@ PlatformImpl::PlatformImpl()
 
 PlatformImpl::~PlatformImpl() { ::WSACleanup(); }
 
+bool PlatformImpl::enableCoreDump() { return false; }
+
 } // namespace Envoy

source/exe/platform_impl.h

@@ -156,6 +156,7 @@ OptionsImpl::OptionsImpl(std::vector<std::string> args,
 
   TCLAP::ValueArg<std::string> socket_mode("", "socket-mode", "Socket file permission", false,
                                            "600", "string", cmd);
+  TCLAP::SwitchArg enable_core_dump("", "enable-core-dump", "Enable core dumps", cmd, false);
 
   cmd.setExceptionHandling(false);
   try {
@@ -177,6 +178,7 @@ OptionsImpl::OptionsImpl(std::vector<std::string> args,
 
   hot_restart_disabled_ = disable_hot_restart.getValue();
   mutex_tracing_enabled_ = enable_mutex_tracing.getValue();
+  core_dump_enabled_ = enable_core_dump.getValue();
 
   cpuset_threads_ = cpuset_threads.getValue();