Sds api: implement SDS api

bazel/repository_locations.bzl

@@ -48,8 +48,9 @@ REPOSITORY_LOCATIONS = dict(
         remote = "https://github.com/google/libprotobuf-mutator",
     ),
     com_github_grpc_grpc = dict(
-        commit = "bec3b5ada2c5e5d782dff0b7b5018df646b65cb0",  # v1.12.0
-        remote = "https://github.com/grpc/grpc.git",
+        commit = "8e40a5b79a1e2f0535e71aab2bdc4844e5e5afd1",
+        # A forked grpc with local_credential supports.
+        remote = "https://github.com/qiwzhang/grpc.git",
     ),
     io_opentracing_cpp = dict(
         commit = "3b36b084a4d7fffc196eac83203cf24dfb8696b3",  # v1.4.2

include/envoy/secret/BUILD

@@ -8,11 +8,40 @@ load(
 
 envoy_package()
 
+envoy_cc_library(
+    name = "secret_callbacks_interface",
+    hdrs = ["secret_callbacks.h"],
+)
+
+envoy_cc_library(
+    name = "dynamic_secret_provider_interface",
+    hdrs = ["dynamic_secret_provider.h"],
+    deps = [
+        "secret_callbacks_interface",
+        "//include/envoy/ssl:tls_certificate_config_interface",
+    ],
+)
+
+envoy_cc_library(
+    name = "dynamic_secret_provider_factory_interface",
+    hdrs = ["dynamic_secret_provider_factory.h"],
+    deps = [
+        ":dynamic_secret_provider_interface",
+        "//include/envoy/event:dispatcher_interface",
+        "//include/envoy/local_info:local_info_interface",
+        "//include/envoy/runtime:runtime_interface",
+        "//include/envoy/stats:stats_interface",
+        "//include/envoy/upstream:cluster_manager_interface",
+        "@envoy_api//envoy/api/v2/core:config_source_cc",
+    ],
+)
+
 envoy_cc_library(
     name = "secret_manager_interface",
     hdrs = ["secret_manager.h"],
     deps = [
-        "//include/envoy/ssl:tls_certificate_config_interface",
+        ":dynamic_secret_provider_interface",
         "@envoy_api//envoy/api/v2/auth:cert_cc",
+        "@envoy_api//envoy/api/v2/core:config_source_cc",
     ],
 )

include/envoy/secret/dynamic_secret_provider.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/secret/secret_callbacks.h"
+#include "envoy/ssl/tls_certificate_config.h"
+
+namespace Envoy {
+namespace Secret {
+
+/**
+ * An interface to fetch dynamic secret.
+ *
+ * TODO(JimmyCYJ): Support other types of secrets.
+ */
+class DynamicTlsCertificateSecretProvider {
+public:
+  virtual ~DynamicTlsCertificateSecretProvider() {}
+
+  /**
+   * @return the TlsCertificate secret. Returns nullptr if the secret is not found.
+   */
+  virtual const Ssl::TlsCertificateConfig* secret() const PURE;
+  virtual void addUpdateCallback(SecretCallbacks& callback) PURE;
+  virtual void removeUpdateCallback(SecretCallbacks& callback) PURE;
+};
+
+typedef std::shared_ptr<DynamicTlsCertificateSecretProvider>
+    DynamicTlsCertificateSecretProviderSharedPtr;
+
+} // namespace Secret
+} // namespace Envoy

include/envoy/secret/dynamic_secret_provider_factory.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include "envoy/api/v2/core/config_source.pb.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/local_info/local_info.h"
+#include "envoy/runtime/runtime.h"
+#include "envoy/secret/dynamic_secret_provider.h"
+#include "envoy/stats/stats.h"
+#include "envoy/upstream/cluster_manager.h"
+
+namespace Envoy {
+namespace Secret {
+
+/**
+ * DynamicTlsCertificateSecretProviderFactoryContext passed to
+ * DynamicTlsCertificateSecretProviderFactory to access resources which are needed for creating
+ * dynamic tls certificate secret provider.
+ */
+class DynamicTlsCertificateSecretProviderFactoryContext {
+public:
+  virtual ~DynamicTlsCertificateSecretProviderFactoryContext() {}
+
+  /**
+   * @return information about the local environment the server is running in.
+   */
+  virtual const LocalInfo::LocalInfo& local_info() PURE;
+
+  /**
+   * @return Event::Dispatcher& the main thread's dispatcher.
+   */
+  virtual Event::Dispatcher& dispatcher() PURE;
+
+  /**
+   * @return RandomGenerator& the random generator for the server.
+   */
+  virtual Runtime::RandomGenerator& random() PURE;
+
+  /**
+   * @return the server-wide stats store.
+   */
+  virtual Stats::Store& stats() PURE;
+
+  /**
+   * @return Upstream::ClusterManager.
+   */
+  virtual Upstream::ClusterManager& cluster_manager() PURE;
+};
+
+/**
+ * Factory for creating dynamic TlsCertificate secret provider.
+ */
+class DynamicTlsCertificateSecretProviderFactory {
+public:
+  virtual ~DynamicTlsCertificateSecretProviderFactory() {}
+
+  /**
+   * Finds and returns a secret provider associated to SDS config. Create a new one
+   * if such provider does not exist.
+   *
+   * @param config_source a protobuf message object contains SDS config source.
+   * @param config_name a name that uniquely refers to the SDS config source.
+   * @return the dynamic tls certificate secret provider.
+   */
+  virtual DynamicTlsCertificateSecretProviderSharedPtr
+  findOrCreate(const envoy::api::v2::core::ConfigSource& sds_config,
+               std::string sds_config_name) PURE;
+};
+
+} // namespace Secret
+} // namespace Envoy

include/envoy/secret/secret_callbacks.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "envoy/common/pure.h"
+
+namespace Envoy {
+namespace Secret {
+
+/**
+ * Callbacks invoked by a secret manager.
+ */
+class SecretCallbacks {
+public:
+  virtual ~SecretCallbacks() {}
+
+  virtual void onAddOrUpdateSecret() PURE;
+};
+
+} // namespace Secret
+} // namespace Envoy

include/envoy/secret/secret_manager.h

@@ -3,15 +3,14 @@
 #include <string>
 
 #include "envoy/api/v2/auth/cert.pb.h"
+#include "envoy/secret/dynamic_secret_provider.h"
 #include "envoy/ssl/tls_certificate_config.h"
 
 namespace Envoy {
 namespace Secret {
 
 /**
- * A manager for static secrets.
- *
- * TODO(jaebong) Support dynamic secrets.
+ * A manager for static and dynamic secrets.
  */
 class SecretManager {
 public:
@@ -21,13 +20,37 @@ class SecretManager {
    * @param secret a protobuf message of envoy::api::v2::auth::Secret.
    * @throw an EnvoyException if the secret is invalid or not supported.
    */
-  virtual void addOrUpdateSecret(const envoy::api::v2::auth::Secret& secret) PURE;
+  virtual void addStaticSecret(const envoy::api::v2::auth::Secret& secret) PURE;
 
   /**
    * @param name a name of the Ssl::TlsCertificateConfig.
    * @return the TlsCertificate secret. Returns nullptr if the secret is not found.
    */
-  virtual const Ssl::TlsCertificateConfig* findTlsCertificate(const std::string& name) const PURE;
+  virtual const Ssl::TlsCertificateConfig*
+  findStaticTlsCertificate(const std::string& name) const PURE;
+
+  /**
+   * Finds and returns a secret provider associated to SDS config. Return nullptr
+   * if such provider does not exist.
+   *
+   * @param config_source a protobuf message object contains SDS config source.
+   * @param config_name a name that uniquely refers to the SDS config source.
+   * @return the dynamic tls certificate secret provider.
+   */
+  virtual DynamicTlsCertificateSecretProviderSharedPtr
+  findDynamicTlsCertificateSecretProvider(const envoy::api::v2::core::ConfigSource& config_source,
+                                          const std::string& config_name) PURE;
+
+  /**
+   * Add new dynamic tls certificate secret provider into secret manager.
+   *
+   * @param config_source a protobuf message object contains SDS config source.
+   * @param config_name a name that uniquely refers to the SDS config source.
+   * @param provider the dynamic tls certificate secret provider to be added into secret manager.
+   */
+  virtual void setDynamicTlsCertificateSecretProvider(
+      const envoy::api::v2::core::ConfigSource& config_source, const std::string& config_name,
+      DynamicTlsCertificateSecretProviderSharedPtr provider) PURE;
 };
 
 } // namespace Secret

include/envoy/server/BUILD

@@ -175,9 +175,12 @@ envoy_cc_library(
     name = "transport_socket_config_interface",
     hdrs = ["transport_socket_config.h"],
     deps = [
+        "//include/envoy/init:init_interface",
         "//include/envoy/network:transport_socket_interface",
+        "//include/envoy/secret:dynamic_secret_provider_factory_interface",
         "//include/envoy/secret:secret_manager_interface",
         "//include/envoy/ssl:context_manager_interface",
+        "//include/envoy/upstream:cluster_manager_interface",
         "//source/common/protobuf",
     ],
 )

include/envoy/server/transport_socket_config.h

@@ -2,9 +2,12 @@
 
 #include <string>
 
+#include "envoy/init/init.h"
 #include "envoy/network/transport_socket.h"
+#include "envoy/secret/dynamic_secret_provider_factory.h"
 #include "envoy/secret/secret_manager.h"
 #include "envoy/ssl/context_manager.h"
+#include "envoy/upstream/cluster_manager.h"
 
 #include "common/protobuf/protobuf.h"
 
@@ -29,10 +32,26 @@ class TransportSocketFactoryContext {
    */
   virtual Stats::Scope& statsScope() const PURE;
 
+  /**
+   * @return the instance of init manager.
+   */
+  virtual Init::Manager& initManager() PURE;
+
   /**
    * Return the instance of secret manager.
    */
   virtual Secret::SecretManager& secretManager() PURE;
+
+  /**
+   * @return the instance of ClusterManager.
+   */
+  virtual Upstream::ClusterManager& clusterManager() PURE;
+
+  /**
+   * @return the factory of dynamic tls certificate secret provider.
+   */
+  virtual Secret::DynamicTlsCertificateSecretProviderFactory&
+  dynamicTlsCertificateSecretProviderFactory() PURE;
 };
 
 class TransportSocketConfigFactory {

include/envoy/ssl/context_config.h

@@ -5,6 +5,7 @@
 #include <vector>
 
 #include "envoy/common/pure.h"
+#include "envoy/secret/dynamic_secret_provider.h"
 
 namespace Envoy {
 namespace Ssl {
@@ -111,6 +112,17 @@ class ContextConfig {
    * @return The maximum TLS protocol version to negotiate.
    */
   virtual unsigned maxProtocolVersion() const PURE;
+
+  /**
+   * @return true if the config is valid. Only when SDS dynamic secret is needed, but has not been
+   * downloaded yet, the config is invalid.
+   */
+  virtual bool isValid() const PURE;
+
+  /**
+   * @return the DynamicSecretProvider object.
+   */
+  virtual Secret::DynamicTlsCertificateSecretProvider* getDynamicSecretProvider() const PURE;
 };
 
 class ClientContextConfig : public virtual ContextConfig {
@@ -127,6 +139,8 @@ class ClientContextConfig : public virtual ContextConfig {
   virtual bool allowRenegotiation() const PURE;
 };
 
+typedef std::unique_ptr<ClientContextConfig> ClientContextConfigPtr;
+
 class ServerContextConfig : public virtual ContextConfig {
 public:
   struct SessionTicketKey {
@@ -148,5 +162,7 @@ class ServerContextConfig : public virtual ContextConfig {
   virtual const std::vector<SessionTicketKey>& sessionTicketKeys() const PURE;
 };
 
+typedef std::unique_ptr<ServerContextConfig> ServerContextConfigPtr;
+
 } // namespace Ssl
 } // namespace Envoy

include/envoy/upstream/upstream.h

@@ -310,6 +310,7 @@ class PrioritySet {
   COUNTER  (upstream_cx_http1_total)                                                               \
   COUNTER  (upstream_cx_http2_total)                                                               \
   COUNTER  (upstream_cx_connect_fail)                                                              \
+  COUNTER  (upstream_cx_connect_fail_by_sds)                                                       \
   COUNTER  (upstream_cx_connect_timeout)                                                           \
   COUNTER  (upstream_cx_idle_timeout)                                                              \
   COUNTER  (upstream_cx_connect_attempts_exceeded)                                                 \

source/common/common/logger.h

@@ -44,6 +44,7 @@ namespace Logger {
   FUNCTION(router)               \
   FUNCTION(runtime)              \
   FUNCTION(stats)                \
+  FUNCTION(secret)               \
   FUNCTION(testing)              \
   FUNCTION(thrift)               \
   FUNCTION(tracing)              \

source/common/config/BUILD

@@ -242,6 +242,7 @@ envoy_cc_library(
     hdrs = ["protobuf_link_hacks.h"],
     deps = [
         "@envoy_api//envoy/service/discovery/v2:ads_cc",
+        "@envoy_api//envoy/service/discovery/v2:sds_cc",
         "@envoy_api//envoy/service/ratelimit/v2:rls_cc",
     ],
 )

source/common/config/protobuf_link_hacks.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "envoy/service/discovery/v2/ads.pb.h"
+#include "envoy/service/discovery/v2/sds.pb.h"
 #include "envoy/service/ratelimit/v2/rls.pb.h"
 
 namespace Envoy {
@@ -9,4 +10,5 @@ namespace Envoy {
 // This file should be included ONLY if this hack is required.
 const envoy::service::discovery::v2::AdsDummy _ads_dummy;
 const envoy::service::ratelimit::v2::RateLimitRequest _rls_dummy;
+const envoy::service::discovery::v2::SdsDummy _sds_dummy;
 } // namespace Envoy

source/common/config/resources.h

@@ -15,6 +15,7 @@ class TypeUrlValues {
   const std::string Listener{"type.googleapis.com/envoy.api.v2.Listener"};
   const std::string Cluster{"type.googleapis.com/envoy.api.v2.Cluster"};
   const std::string ClusterLoadAssignment{"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"};
+  const std::string Secret{"type.googleapis.com/envoy.api.v2.auth.Secret"};
   const std::string RouteConfiguration{"type.googleapis.com/envoy.api.v2.RouteConfiguration"};
 };
 

source/common/grpc/BUILD

@@ -109,6 +109,7 @@ envoy_cc_library(
     deps = [
         "//include/envoy/grpc:google_grpc_creds_interface",
         "//include/envoy/registry",
+        "//source/common/common:utility_lib",
         "//source/common/config:datasource_lib",
     ],
 )