config: Scoped RDS (PR 1/4): Introduce {static,dynamic} config provid…

…er framework (envoyproxy#5243)

This PR introduces a framework for creating static, inline and dynamic (xDS) config providers.

The intent is to create a shared foundation that simplifies implementation of Envoy configuration, in particular xDS config handlers with shared ownership semantics such that the underlying subscription, config proto and config implementation (i.e., the resulting data structures and business logic) are shared across providers and Envoy worker threads.

This PR generalizes the RouteConfigProvider, RouteConfigProviderManager and associated *Impls currently used for implementing HTTP route configuration and the RDS API. Note that static route configuration and RDS are not being transitioned to the generalized framework as part of this PR (see note below for details).

NOTE: This is PR 1 (out of 4) to implement the scoped HTTP routing design introduced in envoyproxy#4704. The planned PR chain is as follows:
1. Introduce generalized config provider framework
2. Implement static and dynamic scoped routing configuration based on framework introduced in PR 1; the scoped routing business logic will _not_ be part of PR 2
3. Implement scoped routing business logic
4. Refactor RDS to use generalized config provider

*Risk Level*: Low (this code is not yet in use; it will be enabled in a follow up PR)
*Testing*: New tests added.
*Docs Changes*: N/A
*Release Notes*: N/A

Signed-off-by: Andres Guedez <aguedez@google.com>
Signed-off-by: Fred Douglas <fredlas@google.com>

include/envoy/config/BUILD

@@ -42,3 +42,23 @@ envoy_cc_library(
         "//source/common/protobuf",
     ],
 )
+
+envoy_cc_library(
+    name = "config_provider_interface",
+    hdrs = ["config_provider.h"],
+    external_deps = ["abseil_optional"],
+    deps = [
+        "//include/envoy/common:time_interface",
+        "//source/common/protobuf",
+    ],
+)
+
+envoy_cc_library(
+    name = "config_provider_manager_interface",
+    hdrs = ["config_provider_manager.h"],
+    deps = [
+        ":config_provider_interface",
+        "//include/envoy/server:filter_config_interface",
+        "//source/common/protobuf",
+    ],
+)

include/envoy/config/config_provider.h

@@ -0,0 +1,113 @@
+#pragma once
+
+#include <memory>
+
+#include "envoy/common/time.h"
+
+#include "common/protobuf/protobuf.h"
+
+#include "absl/types/optional.h"
+
+namespace Envoy {
+namespace Config {
+
+/**
+ * A provider for configuration obtained statically (via static resources in the bootstrap config),
+ * inline with a higher level resource or dynamically via xDS APIs.
+ *
+ * The ConfigProvider is an abstraction layer which higher level components such as the
+ * HttpConnectionManager, Listener, etc can leverage to interface with Envoy's configuration
+ * mechanisms. Implementations of this interface build upon lower level abstractions such as
+ * Envoy::Config::Subscription and Envoy::Config::SubscriptionCallbacks.
+ *
+ * The interface exposed below allows xDS providers to share the underlying config protos and
+ * resulting config implementations (i.e., the ConfigProvider::Config); this enables linear memory
+ * scaling based on the size of the configuration set, regardless of the number of threads/workers.
+ *
+ * Use config() to obtain a shared_ptr to the implementation of the config, and configProtoInfo() to
+ * obtain a reference to the underlying config proto and version (applicable only to dynamic config
+ * providers).
+ */
+class ConfigProvider {
+public:
+  /**
+   * The "implementation" of the configuration.
+   * Use config() to obtain a typed object that corresponds to the specific configuration
+   * represented by this abstract type.
+   */
+  class Config {
+  public:
+    virtual ~Config() = default;
+  };
+  using ConfigConstSharedPtr = std::shared_ptr<const Config>;
+
+  /**
+   * Stores the config proto as well as the associated version.
+   */
+  template <typename P> struct ConfigProtoInfo {
+    const P& config_proto_;
+
+    // Only populated by dynamic config providers.
+    std::string version_;
+  };
+
+  virtual ~ConfigProvider() = default;
+
+  /**
+   * Returns a ConfigProtoInfo associated with the provider.
+   * @return absl::optional<ConfigProtoInfo<P>> an optional ConfigProtoInfo; the value is set when a
+   * config is available.
+   */
+  template <typename P> absl::optional<ConfigProtoInfo<P>> configProtoInfo() const {
+    static_assert(std::is_base_of<Protobuf::Message, P>::value,
+                  "Proto type must derive from Protobuf::Message");
+
+    const auto* config_proto = dynamic_cast<const P*>(getConfigProto());
+    if (config_proto == nullptr) {
+      return absl::nullopt;
+    }
+    return ConfigProtoInfo<P>{*config_proto, getConfigVersion()};
+  }
+
+  /**
+   * Returns the Config corresponding to the provider.
+   * @return std::shared_ptr<const C> a shared pointer to the Config.
+   */
+  template <typename C> std::shared_ptr<const C> config() const {
+    static_assert(std::is_base_of<Config, C>::value,
+                  "Config type must derive from ConfigProvider::Config");
+
+    return std::dynamic_pointer_cast<const C>(getConfig());
+  }
+
+  /**
+   * Returns the timestamp associated with the last update to the Config.
+   * @return SystemTime the timestamp corresponding to the last config update.
+   */
+  virtual SystemTime lastUpdated() const PURE;
+
+protected:
+  /**
+   * Returns the config proto associated with the provider.
+   * @return Protobuf::Message* the config proto corresponding to the Config instantiated by the
+   *         provider.
+   */
+  virtual const Protobuf::Message* getConfigProto() const PURE;
+
+  /**
+   * Returns the config version associated with the provider.
+   * @return std::string the config version.
+   */
+  virtual std::string getConfigVersion() const PURE;
+
+  /**
+   * Returns the config implementation associated with the provider.
+   * @return ConfigConstSharedPtr the config as the base type.
+   */
+  virtual ConfigConstSharedPtr getConfig() const PURE;
+};
+
+using ConfigProviderPtr = std::unique_ptr<ConfigProvider>;
+
+} // namespace Config
+} // namespace Envoy

include/envoy/config/config_provider_manager.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/config/config_provider.h"
+#include "envoy/server/filter_config.h"
+
+#include "common/protobuf/protobuf.h"
+
+namespace Envoy {
+namespace Config {
+
+/**
+ * A ConfigProvider manager which instantiates static and dynamic (xDS) providers.
+ *
+ * ConfigProvider objects are owned by the caller of the
+ * createXdsConfigProvider()/createStaticConfigProvider() functions. The ConfigProviderManager holds
+ * raw pointers to those objects.
+ *
+ * Configuration implementations returned by ConfigProvider::config() are immutable, which allows
+ * them to share the underlying objects such as config protos and subscriptions (for dynamic
+ * providers) without synchronization related performance penalties. This enables linear memory
+ * growth based on the size of the configuration set, regardless of the number of threads/objects
+ * that must hold a reference/pointer to them.
+ */
+class ConfigProviderManager {
+public:
+  virtual ~ConfigProviderManager() = default;
+
+  /**
+   * Returns a dynamic ConfigProvider which receives configuration via an xDS API.
+   * A shared ownership model is used, such that the underlying subscription, config proto
+   * and Config are shared amongst all providers relying on the same config source.
+   * @param config_source_proto supplies the proto containing the xDS API configuration.
+   * @param factory_context is the context to use for the provider.
+   * @param stat_prefix supplies the prefix to use for statistics.
+   * @return ConfigProviderPtr a newly allocated dynamic config provider which shares underlying
+   *                           data structures with other dynamic providers configured with the same
+   *                           API source.
+   */
+  virtual ConfigProviderPtr
+  createXdsConfigProvider(const Protobuf::Message& config_source_proto,
+                          Server::Configuration::FactoryContext& factory_context,
+                          const std::string& stat_prefix) PURE;
+
+  /**
+   * Returns a ConfigProvider associated with a statically specified configuration.
+   * @param config_proto supplies the configuration proto.
+   * @param factory_context is the context to use for the provider.
+   * @return ConfigProviderPtr a newly allocated static config provider.
+   */
+  virtual ConfigProviderPtr
+  createStaticConfigProvider(const Protobuf::Message& config_proto,
+                             Server::Configuration::FactoryContext& factory_context) PURE;
+};
+
+} // namespace Config
+} // namespace Envoy

source/common/common/utility.h

@@ -500,6 +500,17 @@ struct StringViewHash {
   std::size_t operator()(const absl::string_view& k) const { return HashUtil::xxHash64(k); }
 };
 
+/**
+ * Hashing functor for use with enum class types.
+ * This is needed for GCC 5.X; newer versions of GCC, as well as clang7, provide native hashing
+ * specializations.
+ */
+struct EnumClassHash {
+  template <typename T> std::size_t operator()(T t) const {
+    return std::hash<std::size_t>()(static_cast<std::size_t>(t));
+  }
+};
+
 /**
  * Computes running standard-deviation using Welford's algorithm:
  * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm

source/common/config/BUILD

@@ -383,3 +383,20 @@ envoy_cc_library(
         "//source/common/singleton:const_singleton",
     ],
 )
+
+envoy_cc_library(
+    name = "config_provider_lib",
+    srcs = ["config_provider_impl.cc"],
+    hdrs = ["config_provider_impl.h"],
+    deps = [
+        ":utility_lib",
+        "//include/envoy/config:config_provider_interface",
+        "//include/envoy/config:config_provider_manager_interface",
+        "//include/envoy/init:init_interface",
+        "//include/envoy/server:admin_interface",
+        "//include/envoy/server:config_tracker_interface",
+        "//include/envoy/singleton:instance_interface",
+        "//include/envoy/thread_local:thread_local_interface",
+        "//source/common/protobuf",
+    ],
+)

source/common/config/config_provider_impl.cc

@@ -0,0 +1,121 @@
+#include "common/config/config_provider_impl.h"
+
+namespace Envoy {
+namespace Config {
+
+ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase(
+    Server::Configuration::FactoryContext& factory_context,
+    ConfigProviderManagerImplBase& config_provider_manager, ConfigProviderInstanceType type)
+    : last_updated_(factory_context.timeSource().systemTime()),
+      config_provider_manager_(config_provider_manager), type_(type) {
+  config_provider_manager_.bindImmutableConfigProvider(this);
+}
+
+ImmutableConfigProviderImplBase::~ImmutableConfigProviderImplBase() {
+  config_provider_manager_.unbindImmutableConfigProvider(this);
+}
+
+ConfigSubscriptionInstanceBase::~ConfigSubscriptionInstanceBase() {
+  runInitializeCallbackIfAny();
+  config_provider_manager_.unbindSubscription(manager_identifier_);
+}
+
+void ConfigSubscriptionInstanceBase::runInitializeCallbackIfAny() {
+  if (initialize_callback_) {
+    initialize_callback_();
+    initialize_callback_ = nullptr;
+  }
+}
+
+bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message& config_proto,
+                                                         const std::string& config_name,
+                                                         const std::string& version_info) {
+  const uint64_t new_hash = MessageUtil::hash(config_proto);
+  if (config_info_ && config_info_.value().last_config_hash_ == new_hash) {
+    return false;
+  }
+
+  config_info_ = {new_hash, version_info};
+  ENVOY_LOG(debug, "{}: loading new configuration: config_name={} hash={}", name_, config_name,
+            new_hash);
+
+  ASSERT(!mutable_config_providers_.empty());
+  ConfigProvider::ConfigConstSharedPtr new_config;
+  for (auto* provider : mutable_config_providers_) {
+    // All bound mutable config providers must be of the same type (see the ASSERT... in
+    // bindConfigProvider()).
+    // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config
+    // impl, which can then be passed to all providers.
+    if (new_config == nullptr) {
+      if ((new_config = provider->onConfigProtoUpdate(config_proto)) == nullptr) {
+        return false;
+      }
+    }
+    provider->onConfigUpdate(new_config);
+  }
+
+  return true;
+}
+
+void ConfigSubscriptionInstanceBase::bindConfigProvider(MutableConfigProviderImplBase* provider) {
+  // All config providers bound to a ConfigSubscriptionInstanceBase must be of the same concrete
+  // type; this is assumed by checkAndApplyConfig() and is verified by the assertion below.
+  // NOTE: an inlined statement ASSERT() triggers a potentially evaluated expression warning from
+  // clang due to `typeid(**mutable_config_providers_.begin())`. To avoid this, we use a lambda to
+  // separate the first mutable provider dereference from the typeid() statement.
+  ASSERT([&]() {
+    if (!mutable_config_providers_.empty()) {
+      const auto& first_provider = **mutable_config_providers_.begin();
+      return typeid(*provider) == typeid(first_provider);
+    }
+    return true;
+  }());
+  mutable_config_providers_.insert(provider);
+}
+
+ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin,
+                                                             const std::string& config_name) {
+  config_tracker_entry_ =
+      admin.getConfigTracker().add(config_name, [this] { return dumpConfigs(); });
+  // ConfigTracker keys must be unique. We are asserting that no one has stolen the key
+  // from us, since the returned entry will be nullptr if the key already exists.
+  RELEASE_ASSERT(config_tracker_entry_, "");
+}
+
+const ConfigProviderManagerImplBase::ConfigProviderSet&
+ConfigProviderManagerImplBase::immutableConfigProviders(ConfigProviderInstanceType type) const {
+  static ConfigProviderSet empty_set;
+  ConfigProviderMap::const_iterator it;
+  if ((it = immutable_config_providers_map_.find(type)) == immutable_config_providers_map_.end()) {
+    return empty_set;
+  }
+
+  return *it->second;
+}
+
+void ConfigProviderManagerImplBase::bindImmutableConfigProvider(
+    ImmutableConfigProviderImplBase* provider) {
+  ASSERT(provider->type() == ConfigProviderInstanceType::Static ||
+         provider->type() == ConfigProviderInstanceType::Inline);
+  ConfigProviderMap::iterator it;
+  if ((it = immutable_config_providers_map_.find(provider->type())) ==
+      immutable_config_providers_map_.end()) {
+    immutable_config_providers_map_.insert(std::make_pair(
+        provider->type(),
+        std::make_unique<ConfigProviderSet>(std::initializer_list<ConfigProvider*>({provider}))));
+  } else {
+    it->second->insert(provider);
+  }
+}
+
+void ConfigProviderManagerImplBase::unbindImmutableConfigProvider(
+    ImmutableConfigProviderImplBase* provider) {
+  ASSERT(provider->type() == ConfigProviderInstanceType::Static ||
+         provider->type() == ConfigProviderInstanceType::Inline);
+  auto it = immutable_config_providers_map_.find(provider->type());
+  ASSERT(it != immutable_config_providers_map_.end());
+  it->second->erase(provider);
+}
+
+} // namespace Config
+} // namespace Envoy