From 6a3f13bf1ce2b8c54e0838469c12d2519e7c4d25 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 11 Jun 2019 15:31:18 -0400 Subject: [PATCH 01/53] add scope key-builder impl and test Signed-off-by: Xin Zhuang --- api/envoy/api/v2/discovery.proto | 62 +++-- .../v2/http_connection_manager.proto | 13 + bazel/BUILD | 14 + bazel/external/quiche.BUILD | 43 +++ .../configuration/http_conn_man/headers.rst | 7 +- docs/root/intro/version_history.rst | 2 + source/common/grpc/BUILD | 22 +- source/common/grpc/common.cc | 78 ------ source/common/grpc/common.h | 53 +--- source/common/grpc/context_impl.cc | 89 +++++++ source/common/grpc/context_impl.h | 70 +++++ source/common/http/conn_manager_config.h | 2 +- source/common/http/conn_manager_impl.cc | 20 +- source/common/http/conn_manager_impl.h | 10 +- source/common/http/conn_manager_utility.cc | 22 +- source/common/http/http2/codec_impl.cc | 1 + source/common/router/rds_impl.cc | 8 +- source/common/router/scoped_config_impl.cc | 56 ++++ source/common/router/scoped_config_impl.h | 147 ++++++++++ .../filters/http/grpc_http1_bridge/BUILD | 1 + .../grpc_http1_bridge/http1_bridge_filter.cc | 3 +- .../grpc_http1_bridge/http1_bridge_filter.h | 2 +- source/extensions/filters/http/grpc_web/BUILD | 3 +- .../filters/http/grpc_web/grpc_web_filter.cc | 4 +- .../filters/http/grpc_web/grpc_web_filter.h | 2 +- .../network/http_connection_manager/config.cc | 3 + source/extensions/quic_listeners/quiche/BUILD | 27 +- .../quic_listeners/quiche/envoy_quic_alarm.cc | 35 +++ .../quic_listeners/quiche/envoy_quic_alarm.h | 39 +++ .../quiche/envoy_quic_alarm_factory.cc | 22 ++ .../quiche/envoy_quic_alarm_factory.h | 39 +++ .../quic_listeners/quiche/platform/BUILD | 7 - source/extensions/tracers/lightstep/BUILD | 2 + .../tracers/lightstep/lightstep_tracer_impl.h | 3 +- source/server/BUILD | 2 +- source/server/config_validation/server.h | 2 +- source/server/http/admin.cc | 24 +- source/server/http/admin.h | 10 +- source/server/server.h | 4 +- test/common/grpc/BUILD | 14 +- test/common/grpc/common_test.cc | 83 +----- test/common/grpc/context_impl_test.cc | 80 ++++++ test/common/http/conn_manager_impl_test.cc | 48 ++++ test/common/http/conn_manager_utility_test.cc | 24 +- test/common/router/BUILD | 22 +- ...minimized-route_fuzz_test-5077190058704896 | 1 + test/common/router/route_fuzz_test.cc | 19 +- test/common/router/scoped_config_impl_test.cc | 252 ++++++++++++++++++ test/config_test/config_test.cc | 24 ++ .../http1_bridge_filter_test.cc | 6 +- .../http/grpc_web/grpc_web_filter_test.cc | 2 +- test/extensions/quic_listeners/quiche/BUILD | 12 +- .../quiche/envoy_quic_alarm_test.cc | 194 ++++++++++++++ .../lightstep/lightstep_tracer_impl_test.cc | 2 +- test/integration/http2_integration_test.cc | 17 ++ test/integration/http_integration.h | 1 + test/integration/integration_admin_test.cc | 2 + test/mocks/server/BUILD | 1 + test/mocks/server/mocks.h | 5 +- test/server/http/admin_test.cc | 58 +++- test/test_common/simulated_time_system.cc | 2 +- .../test_common/simulated_time_system_test.cc | 6 + 62 files changed, 1499 insertions(+), 329 deletions(-) mode change 100644 => 100755 bazel/BUILD create mode 100644 source/common/grpc/context_impl.cc create mode 100644 source/common/grpc/context_impl.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_alarm.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h create mode 100644 test/common/grpc/context_impl_test.cc create mode 100644 test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5077190058704896 create mode 100644 test/common/router/scoped_config_impl_test.cc create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index fe11fc8fd69c..fb95252a9aec 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -108,7 +108,7 @@ message DiscoveryResponse { // With Delta xDS, the DeltaDiscoveryResponses do not need to include a full // snapshot of the tracked resources. Instead, DeltaDiscoveryResponses are a // diff to the state of a xDS client. -// In Delta XDS there are per resource versions, which allow tracking state at +// In Delta XDS there are per-resource versions, which allow tracking state at // the resource granularity. // An xDS Delta session is always in the context of a gRPC bidirectional // stream. This allows the xDS server to keep track of the state of xDS clients @@ -119,22 +119,27 @@ message DiscoveryResponse { // Optionally, a response message level system_version_info is present for // debugging purposes only. // -// DeltaDiscoveryRequest can be sent in 3 situations: -// 1. Initial message in a xDS bidirectional gRPC stream. -// 2. As a ACK or NACK response to a previous DeltaDiscoveryResponse. -// In this case the response_nonce is set to the nonce value in the Response. -// ACK or NACK is determined by the absence or presence of error_detail. -// 3. Spontaneous DeltaDiscoveryRequest from the client. -// This can be done to dynamically add or remove elements from the tracked -// resource_names set. In this case response_nonce must be omitted. +// DeltaDiscoveryRequest plays two independent roles. Any DeltaDiscoveryRequest +// can be either or both of: [1] informing the server of what resources the +// client has gained/lost interest in (using resource_names_subscribe and +// resource_names_unsubscribe), or [2] (N)ACKing an earlier resource update from +// the server (using response_nonce, with presence of error_detail making it a NACK). +// Additionally, the first message (for a given type_url) of a reconnected gRPC stream +// has a third role: informing the server of the resources (and their versions) +// that the client already possesses, using the initial_resource_versions field. +// +// As with state-of-the-world, when multiple resource types are multiplexed (ADS), +// all requests/acknowledgments/updates are logically walled off by type_url: +// a Cluster ACK exists in a completely separate world from a prior Route NACK. +// In particular, initial_resource_versions being sent at the "start" of every +// gRPC stream actually entails a message for each type_url, each with its own +// initial_resource_versions. message DeltaDiscoveryRequest { // The node making the request. core.Node node = 1; // Type of the resource that is being requested, e.g. - // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". This is implicit - // in requests made via singleton xDS APIs such as CDS, LDS, etc. but is - // required for ADS. + // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". string type_url = 2; // DeltaDiscoveryRequests allow the client to add or remove individual @@ -142,32 +147,35 @@ message DeltaDiscoveryRequest { // All resource names in the resource_names_subscribe list are added to the // set of tracked resources and all resource names in the resource_names_unsubscribe // list are removed from the set of tracked resources. - // Unlike in state-of-the-world xDS, an empty resource_names_subscribe or + // + // *Unlike* state-of-the-world xDS, an empty resource_names_subscribe or // resource_names_unsubscribe list simply means that no resources are to be // added or removed to the resource list. - // The xDS server must send updates for all tracked resources but can also - // send updates for resources the client has not subscribed to. This behavior - // is similar to state-of-the-world xDS. - // These two fields can be set for all types of DeltaDiscoveryRequests - // (initial, ACK/NACK or spontaneous). + // *Like* state-of-the-world xDS, the server must send updates for all tracked + // resources, but can also send updates for resources the client has not subscribed to. // // NOTE: the server must respond with all resources listed in resource_names_subscribe, // even if it believes the client has the most recent version of them. The reason: // the client may have dropped them, but then regained interest before it had a chance // to send the unsubscribe message. See DeltaSubscriptionStateTest.RemoveThenAdd. // + // These two fields can be set in any DeltaDiscoveryRequest, including ACKs + // and initial_resource_versions. + // // A list of Resource names to add to the list of tracked resources. repeated string resource_names_subscribe = 3; // A list of Resource names to remove from the list of tracked resources. repeated string resource_names_unsubscribe = 4; - // This map must be populated when the DeltaDiscoveryRequest is the - // first in a stream (assuming there are any resources - this field's purpose is to enable - // a session to continue in a reconnected gRPC stream, and so will not be used in the very - // first stream of a session). The keys are the resources names of the xDS resources - // known to the xDS client. The values in the map are the associated resource - // level version info. + // Informs the server of the versions of the resources the xDS client knows of, to enable the + // client to continue the same logical xDS session even in the face of gRPC stream reconnection. + // It will not be populated: [1] in the very first stream of a session, since the client will + // not yet have any resources, [2] in any message after the first in a stream (for a given + // type_url), since the server will already be correctly tracking the client's state. + // (In ADS, the first message *of each type_url* of a reconnected stream populates this map.) + // The map's keys are names of xDS resources known to the xDS client. + // The map's values are opaque resource versions. map initial_resource_versions = 5; // When the DeltaDiscoveryRequest is a ACK or NACK message in response @@ -186,8 +194,8 @@ message DeltaDiscoveryResponse { // The version of the response data (used for debugging). string system_version_info = 1; - // The response resources. These are typed resources that match the type url - // in the DeltaDiscoveryRequest. + // The response resources. These are typed resources, whose types must match + // the type_url field. repeated Resource resources = 2 [(gogoproto.nullable) = false]; // field id 3 IS available! @@ -201,7 +209,7 @@ message DeltaDiscoveryResponse { repeated string removed_resources = 6; // The nonce provides a way for DeltaDiscoveryRequests to uniquely - // reference a DeltaDiscoveryResponse. The nonce is required. + // reference a DeltaDiscoveryResponse when (N)ACKing. The nonce is required. string nonce = 5; } diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index be0b3926ff79..78f5aa4c0b1e 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -317,6 +317,7 @@ message HttpConnectionManager { ForwardClientCertDetails forward_client_cert_details = 16 [(validate.rules).enum.defined_only = true]; + // [#comment:next free field: 7] message SetCurrentClientCertDetails { // Whether to forward the subject of the client cert. Defaults to false. google.protobuf.BoolValue subject = 1; @@ -328,6 +329,12 @@ message HttpConnectionManager { // Defaults to false. bool cert = 3; + // Whether to forward the entire client cert chain (including the leaf cert) in URL encoded PEM + // format. This will appear in the XFCC header comma separated from other values with the value + // Chain="PEM". + // Defaults to false. + bool chain = 6; + // Whether to forward the DNS type Subject Alternative Names of the client cert. // Defaults to false. bool dns = 4; @@ -477,6 +484,10 @@ message ScopedRoutes { // Specifies a header field's key value pair to match on. message KvElement { // The separator between key and value (e.g., '=' separates 'k=v;...'). + // If an element is an empty string, the element is ignored. + // If an element contains no separator, the whole element is parsed as key and the + // fragment value is an empty string. + // If there are multiple values for a matched key, the first value is returned. string separator = 1 [(validate.rules).string.min_bytes = 1]; // The key to match on. @@ -485,6 +496,8 @@ message ScopedRoutes { oneof extract_type { // Specifies the zero based index of the element to extract. + // Note Envoy concatenates multiple values of the same header key into a comma separated + // string, the splitting always happens after the concatenation. uint32 index = 3; // Specifies the key value pair to extract the value from. diff --git a/bazel/BUILD b/bazel/BUILD old mode 100644 new mode 100755 index 08c18f4ddd17..c3ac13989d2a --- a/bazel/BUILD +++ b/bazel/BUILD @@ -273,3 +273,17 @@ alias( }, ), ) + +alias( + name = "x86", + actual = select( + { + ":darwin_x86_64": ":darwin_x86_64", + ":ios_x86_64": "ios_x86_64", + "linux_x86_64": "linux_x86_64", + "windows_x86_64": "windows_x86_64", + # If we're not on an x86 platform return a value that will never match in the select() statement calling this since it would have already been matched above. + "//conditions:default": ":darwin_x86_64", + }, + ), +) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 898ea9918c52..c094ad2e5125 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -362,6 +362,37 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_core_arena_scoped_ptr_lib", + hdrs = ["quiche/quic/core/quic_arena_scoped_ptr.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_alarm_lib", + srcs = ["quiche/quic/core/quic_alarm.cc"], + hdrs = ["quiche/quic/core/quic_alarm.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_arena_scoped_ptr_lib", + ":quic_core_time_lib", + ], +) + +envoy_cc_library( + name = "quic_core_alarm_factory_lib", + hdrs = ["quiche/quic/core/quic_alarm_factory.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_alarm_lib", + ":quic_core_one_block_arena_lib", + ], +) + envoy_cc_library( name = "quic_core_buffer_allocator_lib", srcs = [ @@ -387,6 +418,18 @@ envoy_cc_library( deps = [":quic_platform_export"], ) +envoy_cc_library( + name = "quic_core_one_block_arena_lib", + srcs = ["quiche/quic/core/quic_one_block_arena.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_arena_scoped_ptr_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + envoy_cc_library( name = "quic_core_time_lib", srcs = ["quiche/quic/core/quic_time.cc"], diff --git a/docs/root/configuration/http_conn_man/headers.rst b/docs/root/configuration/http_conn_man/headers.rst index f6380190a6b9..492dc850d0ac 100644 --- a/docs/root/configuration/http_conn_man/headers.rst +++ b/docs/root/configuration/http_conn_man/headers.rst @@ -133,9 +133,10 @@ The following keys are supported: 1. ``By`` The Subject Alternative Name (URI type) of the current proxy's certificate. 2. ``Hash`` The SHA 256 digest of the current client certificate. 3. ``Cert`` The entire client certificate in URL encoded PEM format. -4. ``Subject`` The Subject field of the current client certificate. The value is always double-quoted. -5. ``URI`` The URI type Subject Alternative Name field of the current client certificate. -6. ``DNS`` The DNS type Subject Alternative Name field of the current client certificate. A client certificate may contain multiple DNS type Subject Alternative Names, each will be a separate key-value pair. +4. ``Chain`` The entire client certificate chain (including the leaf certificate) in URL encoded PEM format. +5. ``Subject`` The Subject field of the current client certificate. The value is always double-quoted. +6. ``URI`` The URI type Subject Alternative Name field of the current client certificate. +7. ``DNS`` The DNS type Subject Alternative Name field of the current client certificate. A client certificate may contain multiple DNS type Subject Alternative Names, each will be a separate key-value pair. A client certificate may contain multiple Subject Alternative Name types. For details on different Subject Alternative Name types, please refer `RFC 2459`_. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index e6c8d9348237..3b3c45e73538 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -21,6 +21,8 @@ Version history `. * health check: added :ref:`initial jitter ` to add jitter to the first health check in order to prevent thundering herd on Envoy startup. * hot restart: stats are no longer shared between hot restart parent/child via shared memory, but rather by RPC. Hot restart version incremented to 11. +* http: added the ability to pass a URL encoded PEM encoded peer certificate chain in the + :ref:`config_http_conn_man_headers_x-forwarded-client-cert` header. * http: fixed a bug where large unbufferable responses were not tracked in stats and logs correctly. * http: fixed a crashing bug where gRPC local replies would cause segfaults when upstream access logging was on. * http: mitigated a race condition with the :ref:`delayed_close_timeout` where it could trigger while actively flushing a pending write buffer for a downstream connection. diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 15de0f1ecb01..603474289767 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -15,7 +15,7 @@ envoy_cc_library( hdrs = ["typed_async_client.h"], deps = [ ":codec_lib", - ":common_lib", + ":context_lib", "//include/envoy/grpc:async_client_interface", "//source/common/buffer:zero_copy_input_stream_lib", "//source/common/http:async_client_lib", @@ -28,7 +28,7 @@ envoy_cc_library( hdrs = ["async_client_impl.h"], deps = [ ":codec_lib", - ":common_lib", + ":context_lib", ":typed_async_client_lib", "//include/envoy/grpc:async_client_interface", "//source/common/buffer:zero_copy_input_stream_lib", @@ -42,7 +42,7 @@ envoy_cc_library( hdrs = ["async_client_manager_impl.h"], deps = [ ":async_client_lib", - ":common_lib", + ":context_lib", "//include/envoy/grpc:async_client_manager_interface", "//include/envoy/singleton:manager_interface", "//include/envoy/thread_local:thread_local_interface", @@ -78,7 +78,6 @@ envoy_cc_library( hdrs = ["common.h"], external_deps = ["abseil_optional"], deps = [ - "//include/envoy/grpc:context_interface", "//include/envoy/http:header_map_interface", "//include/envoy/http:message_interface", "//include/envoy/stats:stats_interface", @@ -97,6 +96,19 @@ envoy_cc_library( "//source/common/http:message_lib", "//source/common/http:utility_lib", "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "context_lib", + srcs = ["context_impl.cc"], + hdrs = ["context_impl.h"], + external_deps = ["abseil_optional"], + deps = [ + "//include/envoy/grpc:context_interface", + "//include/envoy/http:header_map_interface", + "//include/envoy/stats:stats_interface", + "//source/common/common:hash_lib", "//source/common/stats:symbol_table_lib", ], ) @@ -129,7 +141,7 @@ envoy_cc_library( "grpc", ], deps = [ - ":common_lib", + ":context_lib", ":google_grpc_creds_lib", ":google_grpc_utils_lib", ":typed_async_client_lib", diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 4b28e8e39d1e..43ff985a661f 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -26,29 +26,6 @@ namespace Envoy { namespace Grpc { -Common::Common(Stats::SymbolTable& symbol_table) - : symbol_table_(symbol_table), stat_name_pool_(symbol_table), - grpc_(stat_name_pool_.add("grpc")), grpc_web_(stat_name_pool_.add("grpc-web")), - success_(stat_name_pool_.add("success")), failure_(stat_name_pool_.add("failure")), - total_(stat_name_pool_.add("total")), zero_(stat_name_pool_.add("0")) {} - -// Makes a stat name from a string, if we don't already have one for it. -// This always takes a lock on mutex_, and if we haven't seen the name -// before, it also takes a lock on the symbol table. -// -// TODO(jmarantz): See https://github.com/envoyproxy/envoy/pull/7008 for -// a lock-free approach to creating dynamic stat-names based on requests. -Stats::StatName Common::makeDynamicStatName(absl::string_view name) { - Thread::LockGuard lock(mutex_); - auto iter = stat_name_map_.find(name); - if (iter != stat_name_map_.end()) { - return iter->second; - } - const Stats::StatName stat_name = stat_name_pool_.add(name); - stat_name_map_[std::string(name)] = stat_name; - return stat_name; -} - bool Common::hasGrpcContentType(const Http::HeaderMap& headers) { const Http::HeaderEntry* content_type = headers.ContentType(); // Content type is gRPC if it is exactly "application/grpc" or starts with @@ -72,45 +49,6 @@ bool Common::isGrpcResponseHeader(const Http::HeaderMap& headers, bool end_strea return hasGrpcContentType(headers); } -void Common::chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, - const RequestNames& request_names, const Http::HeaderEntry* grpc_status) { - if (!grpc_status) { - return; - } - - absl::string_view status_str = grpc_status->value().getStringView(); - const bool success = (status_str == "0"); - - // TODO(jmarantz): Perhaps the universe of likely grpc status codes is - // sufficiently bounded that we should precompute status StatNames for popular - // ones beyond "0". - const Stats::StatName status_stat_name = success ? zero_ : makeDynamicStatName(status_str); - const Stats::SymbolTable::StoragePtr stat_name_storage = - symbol_table_.join({protocolStatName(protocol), request_names.service_, request_names.method_, - status_stat_name}); - - cluster.statsScope().counterFromStatName(Stats::StatName(stat_name_storage.get())).inc(); - chargeStat(cluster, protocol, request_names, success); -} - -void Common::chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, - const RequestNames& request_names, bool success) { - const Stats::SymbolTable::StoragePtr prefix_storage = symbol_table_.join( - {protocolStatName(protocol), request_names.service_, request_names.method_}); - const Stats::StatName prefix(prefix_storage.get()); - const Stats::SymbolTable::StoragePtr status = - symbol_table_.join({prefix, successStatName(success)}); - const Stats::SymbolTable::StoragePtr total = symbol_table_.join({prefix, total_}); - - cluster.statsScope().counterFromStatName(Stats::StatName(status.get())).inc(); - cluster.statsScope().counterFromStatName(Stats::StatName(total.get())).inc(); -} - -void Common::chargeStat(const Upstream::ClusterInfo& cluster, const RequestNames& request_names, - bool success) { - chargeStat(cluster, Protocol::Grpc, request_names, success); -} - absl::optional Common::getGrpcStatus(const Http::HeaderMap& trailers) { const Http::HeaderEntry* grpc_status_header = trailers.GrpcStatus(); @@ -130,22 +68,6 @@ std::string Common::getGrpcMessage(const Http::HeaderMap& trailers) { return entry ? std::string(entry->value().getStringView()) : EMPTY_STRING; } -absl::optional -Common::resolveServiceAndMethod(const Http::HeaderEntry* path) { - absl::optional request_names; - if (path == nullptr) { - return request_names; - } - const auto parts = StringUtil::splitToken(path->value().getStringView(), "/"); - if (parts.size() != 2) { - return request_names; - } - const Stats::StatName service = makeDynamicStatName(parts[0]); - const Stats::StatName method = makeDynamicStatName(parts[1]); - request_names = RequestNames{service, method}; - return request_names; -} - Buffer::InstancePtr Common::serializeToGrpcFrame(const Protobuf::Message& message) { // http://www.grpc.io/docs/guides/wire.html // Reserve enough space for the entire message and the 5 byte header. diff --git a/source/common/grpc/common.h b/source/common/grpc/common.h index 82778b5f7d29..fa5fe72f7bc7 100644 --- a/source/common/grpc/common.h +++ b/source/common/grpc/common.h @@ -4,7 +4,6 @@ #include #include "envoy/common/exception.h" -#include "envoy/grpc/context.h" #include "envoy/grpc/status.h" #include "envoy/http/filter.h" #include "envoy/http/header_map.h" @@ -13,7 +12,6 @@ #include "common/common/hash.h" #include "common/grpc/status.h" #include "common/protobuf/protobuf.h" -#include "common/stats/symbol_table_impl.h" #include "absl/types/optional.h" @@ -28,16 +26,8 @@ class Exception : public EnvoyException { const absl::optional grpc_status_; }; -struct Context::RequestNames { - Stats::StatName service_; // supplies the service name. - Stats::StatName method_; // supplies the method name. -}; - -// TODO(jmarantz): Rename 'Common' to 'ContextImpl'. -class Common : public Context { +class Common { public: - explicit Common(Stats::SymbolTable& symbol_table); - /** * @param headers the headers to parse. * @return bool indicating whether content-type is gRPC. @@ -85,23 +75,6 @@ class Common : public Context { */ static void toGrpcTimeout(const std::chrono::milliseconds& timeout, Http::HeaderString& value); - // Context - void chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, - const RequestNames& request_names, const Http::HeaderEntry* grpc_status) override; - void chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, - const RequestNames& request_names, bool success) override; - void chargeStat(const Upstream::ClusterInfo& cluster, const RequestNames& request_names, - bool success) override; - - /** - * Resolve the gRPC service and method from the HTTP2 :path header. - * @param path supplies the :path header. - * @param service supplies the output pointer of the gRPC service. - * @param method supplies the output pointer of the gRPC method. - * @return bool true if both gRPC serve and method have been resolved successfully. - */ - absl::optional resolveServiceAndMethod(const Http::HeaderEntry* path) override; - /** * Serialize protobuf message with gRPC frame header. */ @@ -143,11 +116,6 @@ class Common : public Context { */ static void prependGrpcFrameHeader(Buffer::Instance& buffer); - Stats::StatName successStatName(bool success) const { return success ? success_ : failure_; } - Stats::StatName protocolStatName(Protocol protocol) const { - return protocol == Context::Protocol::Grpc ? grpc_ : grpc_web_; - } - /** * Parse a Buffer::Instance into a Protobuf::Message. * @param buffer containing the data to be parsed. @@ -158,25 +126,6 @@ class Common : public Context { private: static void checkForHeaderOnlyError(Http::Message& http_response); - - // Makes a stat name from a string, if we don't already have one for it. - // This always takes a lock on mutex_, and if we haven't seen the name - // before, it also takes a lock on the symbol table. - // - // TODO(jmarantz): See https://github.com/envoyproxy/envoy/pull/7008 for - // a lock-free approach to creating dynamic stat-names based on requests. - Stats::StatName makeDynamicStatName(absl::string_view name); - - Stats::SymbolTable& symbol_table_; - mutable Thread::MutexBasicLockable mutex_; - Stats::StatNamePool stat_name_pool_ GUARDED_BY(mutex_); - StringMap stat_name_map_ GUARDED_BY(mutex_); - const Stats::StatName grpc_; - const Stats::StatName grpc_web_; - const Stats::StatName success_; - const Stats::StatName failure_; - const Stats::StatName total_; - const Stats::StatName zero_; }; } // namespace Grpc diff --git a/source/common/grpc/context_impl.cc b/source/common/grpc/context_impl.cc new file mode 100644 index 000000000000..94465cd3e10f --- /dev/null +++ b/source/common/grpc/context_impl.cc @@ -0,0 +1,89 @@ +#include "common/grpc/context_impl.h" + +#include +#include + +namespace Envoy { +namespace Grpc { + +ContextImpl::ContextImpl(Stats::SymbolTable& symbol_table) + : symbol_table_(symbol_table), stat_name_pool_(symbol_table), + grpc_(stat_name_pool_.add("grpc")), grpc_web_(stat_name_pool_.add("grpc-web")), + success_(stat_name_pool_.add("success")), failure_(stat_name_pool_.add("failure")), + total_(stat_name_pool_.add("total")), zero_(stat_name_pool_.add("0")) {} + +// Makes a stat name from a string, if we don't already have one for it. +// This always takes a lock on mutex_, and if we haven't seen the name +// before, it also takes a lock on the symbol table. +// +// TODO(jmarantz): See https://github.com/envoyproxy/envoy/pull/7008 for +// a lock-free approach to creating dynamic stat-names based on requests. +Stats::StatName ContextImpl::makeDynamicStatName(absl::string_view name) { + Thread::LockGuard lock(mutex_); + auto iter = stat_name_map_.find(name); + if (iter != stat_name_map_.end()) { + return iter->second; + } + const Stats::StatName stat_name = stat_name_pool_.add(name); + stat_name_map_[std::string(name)] = stat_name; + return stat_name; +} + +void ContextImpl::chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, + const RequestNames& request_names, + const Http::HeaderEntry* grpc_status) { + if (!grpc_status) { + return; + } + + absl::string_view status_str = grpc_status->value().getStringView(); + const bool success = (status_str == "0"); + + // TODO(jmarantz): Perhaps the universe of likely grpc status codes is + // sufficiently bounded that we should precompute status StatNames for popular + // ones beyond "0". + const Stats::StatName status_stat_name = success ? zero_ : makeDynamicStatName(status_str); + const Stats::SymbolTable::StoragePtr stat_name_storage = + symbol_table_.join({protocolStatName(protocol), request_names.service_, request_names.method_, + status_stat_name}); + + cluster.statsScope().counterFromStatName(Stats::StatName(stat_name_storage.get())).inc(); + chargeStat(cluster, protocol, request_names, success); +} + +void ContextImpl::chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, + const RequestNames& request_names, bool success) { + const Stats::SymbolTable::StoragePtr prefix_storage = symbol_table_.join( + {protocolStatName(protocol), request_names.service_, request_names.method_}); + const Stats::StatName prefix(prefix_storage.get()); + const Stats::SymbolTable::StoragePtr status = + symbol_table_.join({prefix, successStatName(success)}); + const Stats::SymbolTable::StoragePtr total = symbol_table_.join({prefix, total_}); + + cluster.statsScope().counterFromStatName(Stats::StatName(status.get())).inc(); + cluster.statsScope().counterFromStatName(Stats::StatName(total.get())).inc(); +} + +void ContextImpl::chargeStat(const Upstream::ClusterInfo& cluster, + const RequestNames& request_names, bool success) { + chargeStat(cluster, Protocol::Grpc, request_names, success); +} + +absl::optional +ContextImpl::resolveServiceAndMethod(const Http::HeaderEntry* path) { + absl::optional request_names; + if (path == nullptr) { + return request_names; + } + const auto parts = StringUtil::splitToken(path->value().getStringView(), "/"); + if (parts.size() != 2) { + return request_names; + } + const Stats::StatName service = makeDynamicStatName(parts[0]); + const Stats::StatName method = makeDynamicStatName(parts[1]); + request_names = RequestNames{service, method}; + return request_names; +} + +} // namespace Grpc +} // namespace Envoy diff --git a/source/common/grpc/context_impl.h b/source/common/grpc/context_impl.h new file mode 100644 index 000000000000..accf3d9c89fe --- /dev/null +++ b/source/common/grpc/context_impl.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +#include "envoy/grpc/context.h" +#include "envoy/http/header_map.h" + +#include "common/common/hash.h" +#include "common/stats/symbol_table_impl.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Grpc { + +struct Context::RequestNames { + Stats::StatName service_; // supplies the service name. + Stats::StatName method_; // supplies the method name. +}; + +class ContextImpl : public Context { +public: + explicit ContextImpl(Stats::SymbolTable& symbol_table); + + // Context + void chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, + const RequestNames& request_names, const Http::HeaderEntry* grpc_status) override; + void chargeStat(const Upstream::ClusterInfo& cluster, Protocol protocol, + const RequestNames& request_names, bool success) override; + void chargeStat(const Upstream::ClusterInfo& cluster, const RequestNames& request_names, + bool success) override; + + /** + * Resolve the gRPC service and method from the HTTP2 :path header. + * @param path supplies the :path header. + * @param service supplies the output pointer of the gRPC service. + * @param method supplies the output pointer of the gRPC method. + * @return bool true if both gRPC serve and method have been resolved successfully. + */ + absl::optional resolveServiceAndMethod(const Http::HeaderEntry* path) override; + + Stats::StatName successStatName(bool success) const { return success ? success_ : failure_; } + Stats::StatName protocolStatName(Protocol protocol) const { + return protocol == Context::Protocol::Grpc ? grpc_ : grpc_web_; + } + +private: + // Makes a stat name from a string, if we don't already have one for it. + // This always takes a lock on mutex_, and if we haven't seen the name + // before, it also takes a lock on the symbol table. + // + // TODO(jmarantz): See https://github.com/envoyproxy/envoy/pull/7008 for + // a lock-free approach to creating dynamic stat-names based on requests. + Stats::StatName makeDynamicStatName(absl::string_view name); + + Stats::SymbolTable& symbol_table_; + mutable Thread::MutexBasicLockable mutex_; + Stats::StatNamePool stat_name_pool_ GUARDED_BY(mutex_); + StringMap stat_name_map_ GUARDED_BY(mutex_); + const Stats::StatName grpc_; + const Stats::StatName grpc_web_; + const Stats::StatName success_; + const Stats::StatName failure_; + const Stats::StatName total_; + const Stats::StatName zero_; +}; + +} // namespace Grpc +} // namespace Envoy diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 66530c2754f3..80f89cb17f55 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -144,7 +144,7 @@ enum class ForwardClientCertType { * Configuration for the fields of the client cert, used for populating the current client cert * information to the next hop. */ -enum class ClientCertDetailsType { Cert, Subject, URI, DNS }; +enum class ClientCertDetailsType { Cert, Chain, Subject, URI, DNS }; /** * Configuration for what addresses should be considered internal beyond the defaults. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 6c7d5459c23a..0c5a221b7bf9 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -180,10 +180,12 @@ void ConnectionManagerImpl::doEndStream(ActiveStream& stream) { // explicitly nulls out response_encoder to avoid the downstream being notified of the // Envoy-internal stream instance being ended. if (stream.response_encoder_ != nullptr && - (!stream.state_.remote_complete_ || !stream.state_.local_complete_)) { + (!stream.state_.remote_complete_ || !stream.state_.codec_saw_local_complete_)) { // Indicate local is complete at this point so that if we reset during a continuation, we don't // raise further data or trailers. + ENVOY_STREAM_LOG(debug, "doEndStream() resetting stream", stream); stream.state_.local_complete_ = true; + stream.state_.codec_saw_local_complete_ = true; stream.response_encoder_->getStream().resetStream(StreamResetReason::LocalReset); reset_stream = true; } @@ -311,8 +313,17 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool void ConnectionManagerImpl::resetAllStreams() { while (!streams_.empty()) { - // Mimic a downstream reset in this case. - streams_.front()->onResetStream(StreamResetReason::ConnectionTermination, absl::string_view()); + // Mimic a downstream reset in this case. We must also remove callbacks here. Though we are + // about to close the connection and will disable further reads, it is possible that flushing + // data out can cause stream callbacks to fire (e.g., low watermark callbacks). + // + // TODO(mattklein123): I tried to actually reset through the codec here, but ran into issues + // with nghttp2 state and being unhappy about sending reset frames after the connection had + // been terminated via GOAWAY. It might be possible to do something better here inside the h2 + // codec but there are no easy answers and this seems simpler. + auto& stream = *streams_.front(); + stream.response_encoder_->getStream().removeCallbacks(stream); + stream.onResetStream(StreamResetReason::ConnectionTermination, absl::string_view()); } } @@ -1524,6 +1535,8 @@ void ConnectionManagerImpl::ActiveStream::encodeTrailers(ActiveStreamEncoderFilt void ConnectionManagerImpl::ActiveStream::maybeEndEncode(bool end_stream) { if (end_stream) { + ASSERT(!state_.codec_saw_local_complete_); + state_.codec_saw_local_complete_ = true; stream_info_.onLastDownstreamTxByteSent(); request_response_timespan_->complete(); connection_manager_.doEndStream(*this); @@ -1549,6 +1562,7 @@ void ConnectionManagerImpl::ActiveStream::onResetStream(StreamResetReason, absl: // 2) The codec TX a codec level reset // 3) The codec RX a reset // If we need to differentiate we need to do it inside the codec. Can start with this. + ENVOY_STREAM_LOG(debug, "stream reset", *this); connection_manager_.stats_.named_.downstream_rq_rx_reset_.inc(); connection_manager_.doDeferredStreamDestroy(*this); } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 02a718e81d8c..dc23f0810b8a 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -451,8 +451,8 @@ class ConnectionManagerImpl : Logger::Loggable, // All state for the stream. Put here for readability. struct State { State() - : remote_complete_(false), local_complete_(false), saw_connection_close_(false), - successful_upgrade_(false), created_filter_chain_(false), + : remote_complete_(false), local_complete_(false), codec_saw_local_complete_(false), + saw_connection_close_(false), successful_upgrade_(false), created_filter_chain_(false), is_internally_created_(false) {} uint32_t filter_call_state_{0}; @@ -463,7 +463,11 @@ class ConnectionManagerImpl : Logger::Loggable, bool decoder_filters_streaming_{true}; bool destroyed_{false}; bool remote_complete_ : 1; - bool local_complete_ : 1; + bool local_complete_ : 1; // This indicates that local is complete prior to filter processing. + // A filter can still stop the stream from being complete as seen + // by the codec. + bool codec_saw_local_complete_ : 1; // This indicates that local is complete as written all + // the way through to the codec. bool saw_connection_close_ : 1; bool successful_upgrade_ : 1; bool created_filter_chain_ : 1; diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 5c8d31fa21be..e81fe4c74a6b 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -16,6 +16,7 @@ #include "common/runtime/uuid_util.h" #include "common/tracing/http_tracer_impl.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" namespace Envoy { @@ -294,38 +295,45 @@ void ConnectionManagerUtility::mutateXfccRequestHeader(HeaderMap& request_header config.forwardClientCert() == ForwardClientCertType::SanitizeSet) { const auto uri_sans_local_cert = connection.ssl()->uriSanLocalCertificate(); if (!uri_sans_local_cert.empty()) { - client_cert_details.push_back("By=" + uri_sans_local_cert[0]); + client_cert_details.push_back(absl::StrCat("By=", uri_sans_local_cert[0])); } const std::string cert_digest = connection.ssl()->sha256PeerCertificateDigest(); if (!cert_digest.empty()) { - client_cert_details.push_back("Hash=" + cert_digest); + client_cert_details.push_back(absl::StrCat("Hash=", cert_digest)); } for (const auto& detail : config.setCurrentClientCertDetails()) { switch (detail) { case ClientCertDetailsType::Cert: { const std::string peer_cert = connection.ssl()->urlEncodedPemEncodedPeerCertificate(); if (!peer_cert.empty()) { - client_cert_details.push_back("Cert=\"" + peer_cert + "\""); + client_cert_details.push_back(absl::StrCat("Cert=\"", peer_cert, "\"")); + } + break; + } + case ClientCertDetailsType::Chain: { + const std::string peer_chain = connection.ssl()->urlEncodedPemEncodedPeerCertificateChain(); + if (!peer_chain.empty()) { + client_cert_details.push_back(absl::StrCat("Chain=\"", peer_chain, "\"")); } break; } case ClientCertDetailsType::Subject: // The "Subject" key still exists even if the subject is empty. - client_cert_details.push_back("Subject=\"" + connection.ssl()->subjectPeerCertificate() + - "\""); + client_cert_details.push_back( + absl::StrCat("Subject=\"", connection.ssl()->subjectPeerCertificate(), "\"")); break; case ClientCertDetailsType::URI: { // The "URI" key still exists even if the URI is empty. const auto sans = connection.ssl()->uriSanPeerCertificate(); const auto& uri_san = sans.empty() ? "" : sans[0]; - client_cert_details.push_back("URI=" + uri_san); + client_cert_details.push_back(absl::StrCat("URI=", uri_san)); break; } case ClientCertDetailsType::DNS: { const std::vector dns_sans = connection.ssl()->dnsSansPeerCertificate(); if (!dns_sans.empty()) { for (const std::string& dns : dns_sans) { - client_cert_details.push_back("DNS=" + dns); + client_cert_details.push_back(absl::StrCat("DNS=", dns)); } } break; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 5fe699fc1a98..5f288926387e 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -511,6 +511,7 @@ int ConnectionImpl::onFrameSend(const nghttp2_frame* frame) { ENVOY_CONN_LOG(trace, "sent frame type={}", connection_, static_cast(frame->hd.type)); switch (frame->hd.type) { case NGHTTP2_GOAWAY: { + ENVOY_CONN_LOG(debug, "sent goaway code={}", connection_, frame->goaway.error_code); if (frame->goaway.error_code != NGHTTP2_NO_ERROR) { return NGHTTP2_ERR_CALLBACK_FAILURE; } diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 5ea463ec37ce..0a291ae4727b 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -129,8 +129,7 @@ void RdsRouteConfigSubscription::onConfigUpdate( void RdsRouteConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) { + const Protobuf::RepeatedPtrField& removed_resources, const std::string&) { if (!removed_resources.empty()) { // TODO(#2500) when on-demand resource loading is supported, an RDS removal may make sense (see // discussion in #6879), and so we should do something other than ignoring here. @@ -139,13 +138,10 @@ void RdsRouteConfigSubscription::onConfigUpdate( "Server sent a delta RDS update attempting to remove a resource (name: {}). Ignoring.", removed_resources[0]); } - Protobuf::RepeatedPtrField unwrapped_resource; if (!added_resources.empty()) { + Protobuf::RepeatedPtrField unwrapped_resource; *unwrapped_resource.Add() = added_resources[0].resource(); onConfigUpdate(unwrapped_resource, added_resources[0].version()); - } else { - onConfigUpdate({}, system_version_info); - return; } } diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 0274381ae850..e32887b7612f 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -3,6 +3,62 @@ namespace Envoy { namespace Router { +std::unique_ptr +HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const { + const Envoy::Http::HeaderEntry* header_entry = + headers.get(Envoy::Http::LowerCaseString(header_value_extractor_config_.name())); + if (header_entry == nullptr) { + return nullptr; + } + + std::vector elements{header_entry->value().getStringView()}; + if (header_value_extractor_config_.element_separator().length() > 0) { + elements = absl::StrSplit(header_entry->value().getStringView(), + header_value_extractor_config_.element_separator()); + } + switch (header_value_extractor_config_.extract_type_case()) { + case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kElement: + for (const auto& element : elements) { + std::pair key_value = absl::StrSplit( + element, absl::MaxSplits(header_value_extractor_config_.element().separator(), 1)); + if (key_value.first == header_value_extractor_config_.element().key()) { + return std::make_unique(key_value.second); + } + } + break; + case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex: + if (header_value_extractor_config_.index() < elements.size()) { + return std::make_unique(elements[header_value_extractor_config_.index()]); + } + break; + default: // EXTRACT_TYPE_NOT_SET + NOT_REACHED_GCOVR_EXCL_LINE; // Caught in constructor already. + } + + return nullptr; +} + +ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder config) + : ScopeKeyBuilderBase(config) { + for (const auto& fragment_builder : config_.fragments()) { + switch (fragment_builder.type_case()) { + case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor: + fragment_builders_.emplace_back(std::make_unique(fragment_builder)); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + } +} + +const ScopeKey ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { + ScopeKey key; + for (const auto& builder : fragment_builders_) { + key.addFragment(builder->computeFragment(headers)); + } + return key; +} + void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 94d376b2a354..3153da37baff 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -1,17 +1,164 @@ #pragma once +#include + #include "envoy/api/v2/srds.pb.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" #include "envoy/thread_local/thread_local.h" +#include "common/protobuf/utility.h" #include "common/router/config_impl.h" #include "common/router/scoped_config_manager.h" namespace Envoy { namespace Router { +using envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes; + +/** + * Scope key fragment base class. + */ +class ScopeKeyFragmentBase { +public: + bool operator==(const ScopeKeyFragmentBase& other) const { + if (typeid(*this) == typeid(other)) { + return equals(other); + } + return false; + } + virtual ~ScopeKeyFragmentBase() = default; + +private: + // Returns true if the two fragments equal else false. + virtual bool equals(const ScopeKeyFragmentBase&) const PURE; +}; + +class ScopeKey { +public: + ScopeKey() = default; + ScopeKey(ScopeKey&& other) = default; + void addFragment(std::unique_ptr&& part) { + contains_null_fragment_ = contains_null_fragment_ || (part == nullptr); + fragments_.emplace_back(std::move(part)); + } + + bool operator!=(const ScopeKey& other) const { return !(*this == other); } + + bool operator==(const ScopeKey& other) const { + if (this->fragments_.size() != other.fragments_.size()) { + return false; + } + if (this->fragments_.size() == 0 || other.fragments_.size() == 0) { + // An empty key equals to nothing, "NULL" != "NULL". + return false; + } + // A "NULL" fragment value equals to nothing. + if (this->contains_null_fragment_ || other.contains_null_fragment_) { + return false; + } + return std::equal(this->fragments_.begin(), this->fragments_.end(), other.fragments_.begin(), + [](const std::unique_ptr& left, + const std::unique_ptr& right) -> bool { + // Both should be non-NULL now. + return *left == *right; + }); + } + +private: + bool contains_null_fragment_{false}; + std::vector> fragments_; +}; + +// String fragment. +class StringKeyFragment : public ScopeKeyFragmentBase { +public: + explicit StringKeyFragment(absl::string_view value) : value_(value) {} + +private: + bool equals(const ScopeKeyFragmentBase& other) const override { + return value_ == static_cast(other).value_; + } + + std::string value_; +}; + +/** + * Base class for fragment builders. + */ +class FragmentBuilderBase { +public: + explicit FragmentBuilderBase(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) + : config_(std::move(config)) {} + virtual ~FragmentBuilderBase() = default; + + // Returns a fragment if the fragment rule applies, a nullptr indicates no fragment could be + // generated from the headers. + virtual std::unique_ptr + computeFragment(const Http::HeaderMap& headers) const PURE; + +protected: + const ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config_; +}; + +class HeaderValueExtractorImpl : public FragmentBuilderBase { +public: + explicit HeaderValueExtractorImpl(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) + : FragmentBuilderBase(config), + header_value_extractor_config_(config_.header_value_extractor()) { + ASSERT(config_.type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor, + "header_value_extractor is not set."); + if (header_value_extractor_config_.extract_type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex) { + if (header_value_extractor_config_.index() != 0 && + header_value_extractor_config_.element_separator().length() == 0) { + throw ProtoValidationException("when element separator is set to an empty string, index " + "should be set to 0 in HeaderValueExtractor.", + header_value_extractor_config_); + } + } + if (header_value_extractor_config_.extract_type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor:: + EXTRACT_TYPE_NOT_SET) { + throw ProtoValidationException("HeaderValueExtractor extract_type not set.", + header_value_extractor_config_); + } + } + + std::unique_ptr + computeFragment(const Http::HeaderMap& headers) const override; + +private: + const ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor& + header_value_extractor_config_; +}; + +/** + * Base class for ScopeKeyBuilder implementations. + */ +class ScopeKeyBuilderBase { +public: + explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder config) : config_(std::move(config)) {} + virtual ~ScopeKeyBuilderBase() = default; + + virtual const ScopeKey computeScopeKey(const Http::HeaderMap& headers) const PURE; + +protected: + ScopedRoutes::ScopeKeyBuilder config_; +}; + +class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { +public: + explicit ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder config); + + const ScopeKey computeScopeKey(const Http::HeaderMap& headers) const override; + +private: + std::vector> fragment_builders_; +}; + /** * TODO(AndresGuedez): implement scoped routing logic. * diff --git a/source/extensions/filters/http/grpc_http1_bridge/BUILD b/source/extensions/filters/http/grpc_http1_bridge/BUILD index c7ceb6cfd4b1..8f664f320c57 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/BUILD +++ b/source/extensions/filters/http/grpc_http1_bridge/BUILD @@ -23,6 +23,7 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:utility_lib", "//source/common/grpc:common_lib", + "//source/common/grpc:context_lib", "//source/common/http:headers_lib", "//source/common/http/http1:codec_lib", ], diff --git a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc index c2929ff46a1a..5a9ffc9869d0 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc +++ b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.cc @@ -9,6 +9,7 @@ #include "common/common/enum_to_int.h" #include "common/common/utility.h" #include "common/grpc/common.h" +#include "common/grpc/context_impl.h" #include "common/http/headers.h" #include "common/http/http1/codec_impl.h" @@ -18,7 +19,7 @@ namespace HttpFilters { namespace GrpcHttp1Bridge { void Http1BridgeFilter::chargeStat(const Http::HeaderMap& headers) { - context_.chargeStat(*cluster_, Grpc::Common::Protocol::Grpc, *request_names_, + context_.chargeStat(*cluster_, Grpc::Context::Protocol::Grpc, *request_names_, headers.GrpcStatus()); } diff --git a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h index 514e3a31d41a..2e697da9ddc0 100644 --- a/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h +++ b/source/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter.h @@ -3,7 +3,7 @@ #include "envoy/http/filter.h" #include "envoy/upstream/cluster_manager.h" -#include "common/grpc/common.h" +#include "common/grpc/context_impl.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/filters/http/grpc_web/BUILD b/source/extensions/filters/http/grpc_web/BUILD index 28d7317948e0..2304f5dc211f 100644 --- a/source/extensions/filters/http/grpc_web/BUILD +++ b/source/extensions/filters/http/grpc_web/BUILD @@ -22,8 +22,9 @@ envoy_cc_library( "//source/common/common:base64_lib", "//source/common/common:utility_lib", "//source/common/grpc:codec_lib", - "//source/common/grpc:common_lib", + "//source/common/grpc:context_lib", "//source/common/http:headers_lib", + "//source/common/http:utility_lib", ], ) diff --git a/source/extensions/filters/http/grpc_web/grpc_web_filter.cc b/source/extensions/filters/http/grpc_web/grpc_web_filter.cc index c54954e3e509..423baa890bca 100644 --- a/source/extensions/filters/http/grpc_web/grpc_web_filter.cc +++ b/source/extensions/filters/http/grpc_web/grpc_web_filter.cc @@ -6,7 +6,7 @@ #include "common/common/base64.h" #include "common/common/empty_string.h" #include "common/common/utility.h" -#include "common/grpc/common.h" +#include "common/grpc/context_impl.h" #include "common/http/headers.h" #include "common/http/utility.h" @@ -234,7 +234,7 @@ void GrpcWebFilter::setupStatTracking(const Http::HeaderMap& headers) { } void GrpcWebFilter::chargeStat(const Http::HeaderMap& headers) { - context_.chargeStat(*cluster_, Grpc::Common::Protocol::GrpcWeb, *request_names_, + context_.chargeStat(*cluster_, Grpc::Context::Protocol::GrpcWeb, *request_names_, headers.GrpcStatus()); } diff --git a/source/extensions/filters/http/grpc_web/grpc_web_filter.h b/source/extensions/filters/http/grpc_web/grpc_web_filter.h index f6a1e47ab574..79c1c562ae7e 100644 --- a/source/extensions/filters/http/grpc_web/grpc_web_filter.h +++ b/source/extensions/filters/http/grpc_web/grpc_web_filter.h @@ -8,7 +8,7 @@ #include "common/buffer/buffer_impl.h" #include "common/common/non_copyable.h" #include "common/grpc/codec.h" -#include "common/grpc/common.h" +#include "common/grpc/context_impl.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 7802508b3b44..002db490e4f1 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -220,6 +220,9 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( if (set_current_client_cert_details.cert()) { set_current_client_cert_details_.push_back(Http::ClientCertDetailsType::Cert); } + if (set_current_client_cert_details.chain()) { + set_current_client_cert_details_.push_back(Http::ClientCertDetailsType::Chain); + } if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(set_current_client_cert_details, subject, false)) { set_current_client_cert_details_.push_back(Http::ClientCertDetailsType::Subject); } diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 17444da358f8..09c338fec0a5 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -8,11 +8,26 @@ load( envoy_package() -# Placeholder library to verify/illustrate depending on a QUICHE build target. -# TODO(mpwarres): remove once real build rules added here. envoy_cc_library( - name = "dummy_lib", - srcs = ["dummy.cc"], - hdrs = ["dummy.h"], - external_deps = ["quiche_http2_platform"], + name = "envoy_quic_alarm_lib", + srcs = ["envoy_quic_alarm.cc"], + hdrs = ["envoy_quic_alarm.h"], + external_deps = ["quiche_quic_platform"], + deps = [ + "//include/envoy/event:timer_interface", + "@com_googlesource_quiche//:quic_core_alarm_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_alarm_factory_lib", + srcs = ["envoy_quic_alarm_factory.cc"], + hdrs = ["envoy_quic_alarm_factory.h"], + external_deps = ["quiche_quic_platform"], + deps = [ + ":envoy_quic_alarm_lib", + "@com_googlesource_quiche//:quic_core_alarm_factory_lib", + "@com_googlesource_quiche//:quic_core_arena_scoped_ptr_lib", + "@com_googlesource_quiche//:quic_core_one_block_arena_lib", + ], ) diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc new file mode 100644 index 000000000000..695a525777ce --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc @@ -0,0 +1,35 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_alarm.h" + +namespace Envoy { +namespace Quic { + +EnvoyQuicAlarm::EnvoyQuicAlarm(Event::Scheduler& scheduler, quic::QuicClock& clock, + quic::QuicArenaScopedPtr delegate) + : QuicAlarm(std::move(delegate)), scheduler_(scheduler), + timer_(scheduler_.createTimer([this]() { Fire(); })), clock_(clock) {} + +void EnvoyQuicAlarm::CancelImpl() { timer_->disableTimer(); } + +void EnvoyQuicAlarm::SetImpl() { + // TODO(#7170) switch to use microseconds if it is supported. + timer_->enableTimer(std::chrono::milliseconds(getDurationBeforeDeadline().ToMilliseconds())); +} + +void EnvoyQuicAlarm::UpdateImpl() { + // Since Timer::enableTimer() overrides its deadline from previous calls, + // there is no need to disable the timer before enabling it again. + SetImpl(); +} + +quic::QuicTime::Delta EnvoyQuicAlarm::getDurationBeforeDeadline() { + quic::QuicTime::Delta duration(quic::QuicTime::Delta::Zero()); + quic::QuicTime now = clock_.ApproximateNow(); + quic::QuicTime tmp = deadline(); + if (tmp > now) { + duration = tmp - now; + } + return duration; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h new file mode 100644 index 000000000000..e50b4afc05ce --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h @@ -0,0 +1,39 @@ +#pragma once + +#include "envoy/event/timer.h" + +#include "common/common/assert.h" + +#include "quiche/quic/core/quic_alarm.h" +#include "quiche/quic/core/quic_time.h" +#include "quiche/quic/platform/api/quic_clock.h" + +namespace Envoy { +namespace Quic { + +// Implements QUIC interface +// https://quiche.googlesource.com/quiche/+/refs/heads/master/quic/core/quic_alarm.h This class +// wraps an Event::Timer object and provide interface for QUIC to interact with the timer. +class EnvoyQuicAlarm : public quic::QuicAlarm { +public: + EnvoyQuicAlarm(Event::Scheduler& scheduler, quic::QuicClock& clock, + quic::QuicArenaScopedPtr delegate); + + ~EnvoyQuicAlarm() override { ASSERT(!IsSet()); }; + + // quic::QuicAlarm + void CancelImpl() override; + void SetImpl() override; + // Overridden to avoid cancel before set. + void UpdateImpl() override; + +private: + quic::QuicTime::Delta getDurationBeforeDeadline(); + + Event::Scheduler& scheduler_; + Event::TimerPtr timer_; + quic::QuicClock& clock_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc new file mode 100644 index 000000000000..5595c9c308e9 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc @@ -0,0 +1,22 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" + +namespace Envoy { +namespace Quic { + +quic::QuicAlarm* EnvoyQuicAlarmFactory::CreateAlarm(quic::QuicAlarm::Delegate* delegate) { + return new EnvoyQuicAlarm(scheduler_, clock_, + quic::QuicArenaScopedPtr(delegate)); +} + +quic::QuicArenaScopedPtr +EnvoyQuicAlarmFactory::CreateAlarm(quic::QuicArenaScopedPtr delegate, + quic::QuicConnectionArena* arena) { + if (arena != nullptr) { + return arena->New(scheduler_, clock_, std::move(delegate)); + } + return quic::QuicArenaScopedPtr( + new EnvoyQuicAlarm(scheduler_, clock_, std::move(delegate))); +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h new file mode 100644 index 000000000000..24f91efc4bc9 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h @@ -0,0 +1,39 @@ +#pragma once + +#include "common/common/non_copyable.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_alarm.h" + +#pragma GCC diagnostic push + +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include "quiche/quic/core/quic_alarm_factory.h" +#include "quiche/quic/core/quic_arena_scoped_ptr.h" +#include "quiche/quic/core/quic_one_block_arena.h" + +#pragma GCC diagnostic pop + +namespace Envoy { +namespace Quic { + +class EnvoyQuicAlarmFactory : public quic::QuicAlarmFactory, NonCopyable { +public: + EnvoyQuicAlarmFactory(Event::Scheduler& scheduler, quic::QuicClock& clock) + : scheduler_(scheduler), clock_(clock) {} + + ~EnvoyQuicAlarmFactory() override {} + + // QuicAlarmFactory + quic::QuicAlarm* CreateAlarm(quic::QuicAlarm::Delegate* delegate) override; + quic::QuicArenaScopedPtr + CreateAlarm(quic::QuicArenaScopedPtr delegate, + quic::QuicConnectionArena* arena) override; + +private: + Event::Scheduler& scheduler_; + quic::QuicClock& clock_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index 158575b9f5ec..49108d806959 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -252,13 +252,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "quic_platform_sleep_impl_lib", - hdrs = ["quic_sleep_impl.h"], - visibility = ["//visibility:public"], - deps = ["@com_googlesource_quiche//:quic_core_time_lib"], -) - envoy_cc_library( name = "spdy_platform_impl_lib", hdrs = [ diff --git a/source/extensions/tracers/lightstep/BUILD b/source/extensions/tracers/lightstep/BUILD index 7b46ebd6e4c4..0f2805e8cd74 100644 --- a/source/extensions/tracers/lightstep/BUILD +++ b/source/extensions/tracers/lightstep/BUILD @@ -21,6 +21,8 @@ envoy_cc_library( external_deps = ["lightstep"], deps = [ "//source/common/config:utility_lib", + "//source/common/grpc:context_lib", + "//source/common/stats:symbol_table_lib", "//source/common/tracing:http_tracer_lib", "//source/extensions/tracers:well_known_names", "//source/extensions/tracers/common/ot:opentracing_driver_lib", diff --git a/source/extensions/tracers/lightstep/lightstep_tracer_impl.h b/source/extensions/tracers/lightstep/lightstep_tracer_impl.h index 224475f9de20..d6046ccd4e40 100644 --- a/source/extensions/tracers/lightstep/lightstep_tracer_impl.h +++ b/source/extensions/tracers/lightstep/lightstep_tracer_impl.h @@ -9,11 +9,12 @@ #include "envoy/tracing/http_tracer.h" #include "envoy/upstream/cluster_manager.h" -#include "common/grpc/common.h" +#include "common/grpc/context_impl.h" #include "common/http/header_map_impl.h" #include "common/http/message_impl.h" #include "common/json/json_loader.h" #include "common/protobuf/protobuf.h" +#include "common/stats/symbol_table_impl.h" #include "extensions/tracers/common/ot/opentracing_driver_impl.h" diff --git a/source/server/BUILD b/source/server/BUILD index f3676164ac8d..3a6770091f36 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -361,7 +361,7 @@ envoy_cc_library( "//source/common/common:version_lib", "//source/common/config:utility_lib", "//source/common/grpc:async_client_manager_lib", - "//source/common/grpc:common_lib", + "//source/common/grpc:context_lib", "//source/common/http:codes_lib", "//source/common/http:context_lib", "//source/common/init:manager_lib", diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index 65a0be91d74b..aab91280ac2c 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -192,7 +192,7 @@ class ValidationInstance : Logger::Loggable, std::unique_ptr listener_manager_; std::unique_ptr overload_manager_; MutexTracer* mutex_tracer_; - Grpc::Common grpc_context_; + Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; Event::TimeSystem& time_system_; }; diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index cb3a22238d01..9ccba827322a 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -155,6 +155,13 @@ void populateFallbackResponseHeaders(Http::Code code, Http::HeaderMap& header_ma header_map.addReference(headers.XContentTypeOptions, headers.XContentTypeOptionValues.Nosniff); } +// Helper method to get filter parameter +absl::optional filterParam(Http::Utility::QueryParams params) { + return (params.find("filter") != params.end()) + ? absl::optional{std::regex(params.at("filter"))} + : absl::nullopt; +} + // Helper method that ensures that we've setting flags based on all the health flag values on the // host. void setHealthFlag(Upstream::Host::HealthFlag flag, const Upstream::Host& host, @@ -696,10 +703,7 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo const bool used_only = params.find("usedonly") != params.end(); const bool has_format = !(params.find("format") == params.end()); - const absl::optional regex = - (params.find("filter") != params.end()) - ? absl::optional{std::regex(params.at("filter"))} - : absl::nullopt; + const absl::optional regex = filterParam(params); std::map all_stats; for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { @@ -753,8 +757,10 @@ Http::Code AdminImpl::handlerPrometheusStats(absl::string_view path_and_query, H Buffer::Instance& response, AdminStream&) { const Http::Utility::QueryParams params = Http::Utility::parseQueryString(path_and_query); const bool used_only = params.find("usedonly") != params.end(); + const absl::optional regex = filterParam(params); PrometheusStatsFormatter::statsAsPrometheus(server_.stats().counters(), server_.stats().gauges(), - server_.stats().histograms(), response, used_only); + server_.stats().histograms(), response, used_only, + regex); return Http::Code::OK; } @@ -788,10 +794,10 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( const std::vector& counters, const std::vector& gauges, const std::vector& histograms, Buffer::Instance& response, - const bool used_only) { + const bool used_only, const absl::optional& regex) { std::unordered_set metric_type_tracker; for (const auto& counter : counters) { - if (!shouldShowMetric(counter, used_only)) { + if (!shouldShowMetric(counter, used_only, regex)) { continue; } @@ -805,7 +811,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( } for (const auto& gauge : gauges) { - if (!shouldShowMetric(gauge, used_only)) { + if (!shouldShowMetric(gauge, used_only, regex)) { continue; } @@ -819,7 +825,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( } for (const auto& histogram : histograms) { - if (!shouldShowMetric(histogram, used_only)) { + if (!shouldShowMetric(histogram, used_only, regex)) { continue; } diff --git a/source/server/http/admin.h b/source/server/http/admin.h index a0c2c8fccb2d..3bd4754a2aec 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -220,6 +220,7 @@ class AdminImpl : public Admin, bool used_only, const absl::optional regex = absl::nullopt, bool pretty_print = false); + static std::string runtimeAsJson(const std::vector>& entries); std::vector sortedHandlers() const; @@ -413,7 +414,8 @@ class PrometheusStatsFormatter { static uint64_t statsAsPrometheus(const std::vector& counters, const std::vector& gauges, const std::vector& histograms, - Buffer::Instance& response, const bool used_only); + Buffer::Instance& response, const bool used_only, + const absl::optional& regex); /** * Format the given tags, returning a string as a comma-separated list * of ="" pairs. @@ -434,8 +436,10 @@ class PrometheusStatsFormatter { * Determine whether a metric has never been emitted and choose to * not show it if we only wanted used metrics. */ - static bool shouldShowMetric(const std::shared_ptr& metric, const bool used_only) { - return !used_only || metric->used(); + static bool shouldShowMetric(const std::shared_ptr& metric, const bool used_only, + const absl::optional& regex) { + return ((!used_only || metric->used()) && + (!regex.has_value() || std::regex_search(metric->name(), regex.value()))); } }; diff --git a/source/server/server.h b/source/server/server.h index e9f7283311de..75eff17064c5 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -22,7 +22,7 @@ #include "common/common/cleanup.h" #include "common/common/logger_delegates.h" #include "common/grpc/async_client_manager_impl.h" -#include "common/grpc/common.h" +#include "common/grpc/context_impl.h" #include "common/http/context_impl.h" #include "common/init/manager_impl.h" #include "common/memory/heap_shrinker.h" @@ -270,7 +270,7 @@ class InstanceImpl : Logger::Loggable, Upstream::HdsDelegatePtr hds_delegate_; std::unique_ptr overload_manager_; Envoy::MutexTracer* mutex_tracer_; - Grpc::Common grpc_context_; + Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; std::unique_ptr process_context_; std::unique_ptr heap_shrinker_; diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index 0a1ca7e3bc8b..ed8df64ca6ca 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -52,14 +52,24 @@ envoy_cc_test( deps = [ "//source/common/grpc:common_lib", "//source/common/http:headers_lib", - "//source/common/stats:fake_symbol_table_lib", "//test/mocks/upstream:upstream_mocks", "//test/proto:helloworld_proto_cc", - "//test/test_common:global_lib", "//test/test_common:utility_lib", ], ) +envoy_cc_test( + name = "context_impl_test", + srcs = ["context_impl_test.cc"], + deps = [ + "//source/common/grpc:common_lib", + "//source/common/http:headers_lib", + "//source/common/stats:fake_symbol_table_lib", + "//test/mocks/upstream:upstream_mocks", + "//test/test_common:global_lib", + ], +) + envoy_cc_test( name = "google_grpc_utils_test", srcs = envoy_select_google_grpc(["google_grpc_utils_test.cc"]), diff --git a/test/common/grpc/common_test.cc b/test/common/grpc/common_test.cc index 98b65423a99e..b2ab23f919ad 100644 --- a/test/common/grpc/common_test.cc +++ b/test/common/grpc/common_test.cc @@ -4,7 +4,6 @@ #include "common/http/headers.h" #include "common/http/message_impl.h" #include "common/http/utility.h" -#include "common/stats/fake_symbol_table_impl.h" #include "test/mocks/upstream/mocks.h" #include "test/proto/helloworld.pb.h" @@ -16,7 +15,7 @@ namespace Envoy { namespace Grpc { -TEST(GrpcCommonTest, GetGrpcStatus) { +TEST(GrpcContextTest, GetGrpcStatus) { Http::TestHeaderMapImpl ok_trailers{{"grpc-status", "0"}}; EXPECT_EQ(Status::Ok, Common::getGrpcStatus(ok_trailers).value()); @@ -33,7 +32,7 @@ TEST(GrpcCommonTest, GetGrpcStatus) { EXPECT_EQ(Status::InvalidCode, Common::getGrpcStatus(invalid_trailers).value()); } -TEST(GrpcCommonTest, GetGrpcMessage) { +TEST(GrpcContextTest, GetGrpcMessage) { Http::TestHeaderMapImpl empty_trailers; EXPECT_EQ("", Common::getGrpcMessage(empty_trailers)); @@ -44,7 +43,7 @@ TEST(GrpcCommonTest, GetGrpcMessage) { EXPECT_EQ("", Common::getGrpcMessage(empty_error_trailers)); } -TEST(GrpcCommonTest, GetGrpcTimeout) { +TEST(GrpcContextTest, GetGrpcTimeout) { Http::TestHeaderMapImpl empty_headers; EXPECT_EQ(std::chrono::milliseconds(0), Common::getGrpcTimeout(empty_headers)); @@ -79,7 +78,7 @@ TEST(GrpcCommonTest, GetGrpcTimeout) { // so we don't test for them. } -TEST(GrpcCommonTest, ToGrpcTimeout) { +TEST(GrpcContextTest, ToGrpcTimeout) { Http::HeaderString value; Common::toGrpcTimeout(std::chrono::milliseconds(0UL), value); @@ -104,43 +103,7 @@ TEST(GrpcCommonTest, ToGrpcTimeout) { EXPECT_EQ("99999999H", value.getStringView()); } -TEST(GrpcCommonTest, ChargeStats) { - NiceMock cluster; - Envoy::Test::Global symbol_table_; - Stats::StatNamePool pool(*symbol_table_); - const Stats::StatName service = pool.add("service"); - const Stats::StatName method = pool.add("method"); - Common::RequestNames request_names{service, method}; - Common common(*symbol_table_); - common.chargeStat(cluster, request_names, true); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.success").value()); - EXPECT_EQ(0U, cluster.stats_store_.counter("grpc.service.method.failure").value()); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.total").value()); - - common.chargeStat(cluster, request_names, false); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.success").value()); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.failure").value()); - EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.total").value()); - - Http::TestHeaderMapImpl trailers; - Http::HeaderEntry& status = trailers.insertGrpcStatus(); - status.value("0", 1); - common.chargeStat(cluster, Context::Protocol::Grpc, request_names, &status); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.0").value()); - EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.success").value()); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.failure").value()); - EXPECT_EQ(3U, cluster.stats_store_.counter("grpc.service.method.total").value()); - - status.value("1", 1); - common.chargeStat(cluster, Context::Protocol::Grpc, request_names, &status); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.0").value()); - EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.1").value()); - EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.success").value()); - EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.failure").value()); - EXPECT_EQ(4U, cluster.stats_store_.counter("grpc.service.method.total").value()); -} - -TEST(GrpcCommonTest, PrepareHeaders) { +TEST(GrpcContextTest, PrepareHeaders) { { Http::MessagePtr message = Common::prepareHeaders("cluster", "service_name", "method_name", absl::nullopt); @@ -213,31 +176,7 @@ TEST(GrpcCommonTest, PrepareHeaders) { } } -TEST(GrpcCommonTest, ResolveServiceAndMethod) { - std::string service; - std::string method; - Http::HeaderMapImpl headers; - Http::HeaderEntry& path = headers.insertPath(); - path.value(std::string("/service_name/method_name")); - Envoy::Test::Global symbol_table; - Common common(*symbol_table); - absl::optional request_names = common.resolveServiceAndMethod(&path); - EXPECT_TRUE(request_names); - EXPECT_EQ("service_name", symbol_table->toString(request_names->service_)); - EXPECT_EQ("method_name", symbol_table->toString(request_names->method_)); - path.value(std::string("")); - EXPECT_FALSE(common.resolveServiceAndMethod(&path)); - path.value(std::string("/")); - EXPECT_FALSE(common.resolveServiceAndMethod(&path)); - path.value(std::string("//")); - EXPECT_FALSE(common.resolveServiceAndMethod(&path)); - path.value(std::string("/service_name")); - EXPECT_FALSE(common.resolveServiceAndMethod(&path)); - path.value(std::string("/service_name/")); - EXPECT_FALSE(common.resolveServiceAndMethod(&path)); -} - -TEST(GrpcCommonTest, GrpcToHttpStatus) { +TEST(GrpcContextTest, GrpcToHttpStatus) { const std::vector> test_set = { {Status::GrpcStatus::Ok, 200}, {Status::GrpcStatus::Canceled, 499}, @@ -263,7 +202,7 @@ TEST(GrpcCommonTest, GrpcToHttpStatus) { } } -TEST(GrpcCommonTest, HttpToGrpcStatus) { +TEST(GrpcContextTest, HttpToGrpcStatus) { const std::vector> test_set = { {400, Status::GrpcStatus::Internal}, {401, Status::GrpcStatus::Unauthenticated}, {403, Status::GrpcStatus::PermissionDenied}, {404, Status::GrpcStatus::Unimplemented}, @@ -276,7 +215,7 @@ TEST(GrpcCommonTest, HttpToGrpcStatus) { } } -TEST(GrpcCommonTest, HasGrpcContentType) { +TEST(GrpcContextTest, HasGrpcContentType) { { Http::TestHeaderMapImpl headers{}; EXPECT_FALSE(Common::hasGrpcContentType(headers)); @@ -295,7 +234,7 @@ TEST(GrpcCommonTest, HasGrpcContentType) { EXPECT_FALSE(isGrpcContentType("application/grpc-web+foo")); } -TEST(GrpcCommonTest, IsGrpcResponseHeader) { +TEST(GrpcContextTest, IsGrpcResponseHeader) { Http::TestHeaderMapImpl grpc_status_only{{":status", "500"}, {"grpc-status", "14"}}; EXPECT_TRUE(Common::isGrpcResponseHeader(grpc_status_only, true)); EXPECT_FALSE(Common::isGrpcResponseHeader(grpc_status_only, false)); @@ -311,7 +250,7 @@ TEST(GrpcCommonTest, IsGrpcResponseHeader) { EXPECT_FALSE(Common::isGrpcResponseHeader(json_response_header, false)); } -TEST(GrpcCommonTest, ValidateResponse) { +TEST(GrpcContextTest, ValidateResponse) { { Http::ResponseMessageImpl response( Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}}); @@ -363,7 +302,7 @@ TEST(GrpcCommonTest, ValidateResponse) { } // Ensure that the correct gPRC header is constructed for a Buffer::Instance. -TEST(GrpcCommonTest, PrependGrpcFrameHeader) { +TEST(GrpcContextTest, PrependGrpcFrameHeader) { auto buffer = std::make_unique(); buffer->add("test", 4); std::array expected_header; diff --git a/test/common/grpc/context_impl_test.cc b/test/common/grpc/context_impl_test.cc new file mode 100644 index 000000000000..ace45801ca11 --- /dev/null +++ b/test/common/grpc/context_impl_test.cc @@ -0,0 +1,80 @@ +#include + +#include "common/grpc/common.h" +#include "common/grpc/context_impl.h" +#include "common/http/headers.h" +#include "common/http/message_impl.h" +#include "common/http/utility.h" +#include "common/stats/fake_symbol_table_impl.h" + +#include "test/mocks/upstream/mocks.h" +#include "test/test_common/global.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Grpc { + +TEST(GrpcContextTest, ChargeStats) { + NiceMock cluster; + Envoy::Test::Global symbol_table_; + Stats::StatNamePool pool(*symbol_table_); + const Stats::StatName service = pool.add("service"); + const Stats::StatName method = pool.add("method"); + Context::RequestNames request_names{service, method}; + ContextImpl context(*symbol_table_); + context.chargeStat(cluster, request_names, true); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.success").value()); + EXPECT_EQ(0U, cluster.stats_store_.counter("grpc.service.method.failure").value()); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.total").value()); + + context.chargeStat(cluster, request_names, false); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.success").value()); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.failure").value()); + EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.total").value()); + + Http::TestHeaderMapImpl trailers; + Http::HeaderEntry& status = trailers.insertGrpcStatus(); + status.value("0", 1); + context.chargeStat(cluster, Context::Protocol::Grpc, request_names, &status); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.0").value()); + EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.success").value()); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.failure").value()); + EXPECT_EQ(3U, cluster.stats_store_.counter("grpc.service.method.total").value()); + + status.value("1", 1); + context.chargeStat(cluster, Context::Protocol::Grpc, request_names, &status); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.0").value()); + EXPECT_EQ(1U, cluster.stats_store_.counter("grpc.service.method.1").value()); + EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.success").value()); + EXPECT_EQ(2U, cluster.stats_store_.counter("grpc.service.method.failure").value()); + EXPECT_EQ(4U, cluster.stats_store_.counter("grpc.service.method.total").value()); +} + +TEST(GrpcContextTest, ResolveServiceAndMethod) { + std::string service; + std::string method; + Http::HeaderMapImpl headers; + Http::HeaderEntry& path = headers.insertPath(); + path.value(std::string("/service_name/method_name")); + Envoy::Test::Global symbol_table; + ContextImpl context(*symbol_table); + absl::optional request_names = context.resolveServiceAndMethod(&path); + EXPECT_TRUE(request_names); + EXPECT_EQ("service_name", symbol_table->toString(request_names->service_)); + EXPECT_EQ("method_name", symbol_table->toString(request_names->method_)); + path.value(std::string("")); + EXPECT_FALSE(context.resolveServiceAndMethod(&path)); + path.value(std::string("/")); + EXPECT_FALSE(context.resolveServiceAndMethod(&path)); + path.value(std::string("//")); + EXPECT_FALSE(context.resolveServiceAndMethod(&path)); + path.value(std::string("/service_name")); + EXPECT_FALSE(context.resolveServiceAndMethod(&path)); + path.value(std::string("/service_name/")); + EXPECT_FALSE(context.resolveServiceAndMethod(&path)); +} + +} // namespace Grpc +} // namespace Envoy diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 81a555eb2c90..9324add9ca30 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2297,6 +2297,7 @@ TEST_F(HttpConnectionManagerImplTest, DownstreamProtocolError) { throw CodecProtocolException("protocol error"); })); + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); // A protocol exception should result in reset of the streams followed by a remote or local close @@ -3352,6 +3353,53 @@ TEST_F(HttpConnectionManagerImplTest, FilterHeadReply) { conn_manager_->onData(fake_input, false); } +// Verify that if an encoded stream has been ended, but gets stopped by a filter chain, we end +// up resetting the stream in the doEndStream() path (e.g., via filter reset due to timeout, etc.), +// we emit a reset to the codec. +TEST_F(HttpConnectionManagerImplTest, ResetWithStoppedFilter) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + decoder->decodeHeaders(std::move(headers), true); + data.drain(4); + })); + + setupFilterChain(1, 1); + + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, true)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus { + decoder_filters_[0]->callbacks_->sendLocalReply(Code::BadRequest, "Bad request", nullptr, + absl::nullopt, ""); + return FilterHeadersStatus::Continue; + })); + + EXPECT_CALL(*encoder_filters_[0], encodeHeaders(_, false)) + .WillOnce(Invoke([&](HeaderMap& headers, bool) -> FilterHeadersStatus { + EXPECT_EQ("11", headers.ContentLength()->value().getStringView()); + return FilterHeadersStatus::Continue; + })); + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)); + EXPECT_CALL(*encoder_filters_[0], encodeData(_, true)) + .WillOnce(Invoke([&](Buffer::Instance&, bool) -> FilterDataStatus { + return FilterDataStatus::StopIterationAndBuffer; + })); + + EXPECT_CALL(*encoder_filters_[0], encodeComplete()); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + EXPECT_CALL(response_encoder_.stream_, resetStream(_)); + expectOnDestroy(); + encoder_filters_[0]->callbacks_->resetStream(); +} + TEST_F(HttpConnectionManagerImplTest, FilterContinueAndEndStreamHeaders) { InSequence s; setup(false, ""); diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index f78bce8b01ee..fad349c3d9de 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -769,6 +769,9 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSetForwardClientCert) { EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("%3D%3Dabc%0Ade%3D"); EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); + std::string expected_chain_pem(expected_pem + "%3D%3Dlmn%0Aop%3D"); + EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificateChain()) + .WillOnce(ReturnRef(expected_chain_pem)); std::vector expected_dns = {"www.example.com"}; EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillOnce(Return(expected_dns)); ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); @@ -777,6 +780,7 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSetForwardClientCert) { std::vector details = std::vector(); details.push_back(Http::ClientCertDetailsType::URI); details.push_back(Http::ClientCertDetailsType::Cert); + details.push_back(Http::ClientCertDetailsType::Chain); details.push_back(Http::ClientCertDetailsType::DNS); ON_CALL(config_, setCurrentClientCertDetails()).WillByDefault(ReturnRef(details)); TestHeaderMapImpl headers; @@ -788,6 +792,7 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSetForwardClientCert) { "Hash=abcdefg;" "URI=test://foo.com/fe;" "Cert=\"%3D%3Dabc%0Ade%3D\";" + "Chain=\"%3D%3Dabc%0Ade%3D%3D%3Dlmn%0Aop%3D\";" "DNS=www.example.com", headers.get_("x-forwarded-client-cert")); } @@ -807,6 +812,9 @@ TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCert) { EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("%3D%3Dabc%0Ade%3D"); EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); + std::string expected_chain_pem(expected_pem + "%3D%3Dlmn%0Aop%3D"); + EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificateChain()) + .WillOnce(ReturnRef(expected_chain_pem)); std::vector expected_dns = {"www.example.com"}; EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillOnce(Return(expected_dns)); ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); @@ -815,6 +823,7 @@ TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCert) { std::vector details = std::vector(); details.push_back(Http::ClientCertDetailsType::URI); details.push_back(Http::ClientCertDetailsType::Cert); + details.push_back(Http::ClientCertDetailsType::Chain); details.push_back(Http::ClientCertDetailsType::DNS); ON_CALL(config_, setCurrentClientCertDetails()).WillByDefault(ReturnRef(details)); TestHeaderMapImpl headers{{"x-forwarded-client-cert", "By=test://foo.com/fe;" @@ -824,10 +833,11 @@ TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCert) { EXPECT_EQ((MutateRequestRet{"10.0.0.3:50000", false}), callMutateRequestHeaders(headers, Protocol::Http2)); EXPECT_TRUE(headers.has("x-forwarded-client-cert")); - EXPECT_EQ("By=test://foo.com/fe;URI=test://bar.com/be;DNS=test.com;DNS=test.com," - "By=test://foo.com/be;Hash=abcdefg;URI=test://foo.com/fe;" - "Cert=\"%3D%3Dabc%0Ade%3D\";DNS=www.example.com", - headers.get_("x-forwarded-client-cert")); + EXPECT_EQ( + "By=test://foo.com/fe;URI=test://bar.com/be;DNS=test.com;DNS=test.com," + "By=test://foo.com/be;Hash=abcdefg;URI=test://foo.com/fe;" + "Cert=\"%3D%3Dabc%0Ade%3D\";Chain=\"%3D%3Dabc%0Ade%3D%3D%3Dlmn%0Aop%3D\";DNS=www.example.com", + headers.get_("x-forwarded-client-cert")); } // This test assumes the following scenario: @@ -876,6 +886,9 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCert) { EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("abcde="); EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); + std::string expected_chain_pem(expected_pem + "lmnop="); + EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificateChain()) + .WillOnce(ReturnRef(expected_chain_pem)); ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::SanitizeSet)); @@ -883,6 +896,7 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCert) { details.push_back(Http::ClientCertDetailsType::Subject); details.push_back(Http::ClientCertDetailsType::URI); details.push_back(Http::ClientCertDetailsType::Cert); + details.push_back(Http::ClientCertDetailsType::Chain); ON_CALL(config_, setCurrentClientCertDetails()).WillByDefault(ReturnRef(details)); TestHeaderMapImpl headers{ {"x-forwarded-client-cert", "By=test://foo.com/fe;URI=test://bar.com/be"}}; @@ -892,7 +906,7 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCert) { EXPECT_TRUE(headers.has("x-forwarded-client-cert")); EXPECT_EQ("By=test://foo.com/be;Hash=abcdefg;Subject=\"/C=US/ST=CA/L=San " "Francisco/OU=Lyft/CN=test.lyft.com\";URI=test://foo.com/" - "fe;Cert=\"abcde=\"", + "fe;Cert=\"abcde=\";Chain=\"abcde=lmnop=\"", headers.get_("x-forwarded-client-cert")); } diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 9e5a061f7370..c6dd8f9ae6d5 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -78,6 +78,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "scoped_config_impl_test", + srcs = ["scoped_config_impl_test.cc"], + external_deps = [ + "abseil_strings", + ], + deps = [ + "//source/common/router:scoped_config_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "scoped_rds_test", srcs = ["scoped_rds_test.cc"], @@ -173,7 +185,15 @@ envoy_directory_genrule( filegroup( name = "route_corpus", testonly = 1, - srcs = [":corpus_from_config_impl"] + glob(["route_corpus/**"]), + srcs = select({ + # TODO(asraa): Clean this up for cross-compilation. Right now we assume + # the host and target are the same on x86 builds, so we only execute the + # corpus generation binary on x86 platforms. + "//bazel:x86": [":corpus_from_config_impl"], + "//conditions:default": [], + }) + glob([ + "route_corpus/**", + ]), ) envoy_cc_fuzz_test( diff --git a/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5077190058704896 b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5077190058704896 new file mode 100644 index 000000000000..06cb05362396 --- /dev/null +++ b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5077190058704896 @@ -0,0 +1 @@ +config { internal_only_headers: "\0 " } \ No newline at end of file diff --git a/test/common/router/route_fuzz_test.cc b/test/common/router/route_fuzz_test.cc index f12de7a7837e..206bcd876575 100644 --- a/test/common/router/route_fuzz_test.cc +++ b/test/common/router/route_fuzz_test.cc @@ -11,13 +11,30 @@ namespace Envoy { namespace Router { namespace { +// Return a new RouteConfiguration with invalid characters replaces in all header fields. +envoy::api::v2::RouteConfiguration +replaceInvalidHeaders(envoy::api::v2::RouteConfiguration route_config) { + envoy::api::v2::RouteConfiguration clean_config = route_config; + clean_config.mutable_request_headers_to_add()->CopyFrom( + Fuzz::replaceInvalidHeaders(route_config.request_headers_to_add())); + clean_config.mutable_response_headers_to_add()->CopyFrom( + Fuzz::replaceInvalidHeaders(route_config.response_headers_to_add())); + auto internal_only_headers = clean_config.mutable_internal_only_headers(); + std::for_each(internal_only_headers->begin(), internal_only_headers->end(), + [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); + auto request_headers_to_remove = clean_config.mutable_request_headers_to_remove(); + std::for_each(request_headers_to_remove->begin(), request_headers_to_remove->end(), + [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); + return clean_config; +} + // TODO(htuch): figure out how to generate via a genrule from config_impl_test the full corpus. DEFINE_PROTO_FUZZER(const test::common::router::RouteTestCase& input) { try { NiceMock stream_info; NiceMock factory_context; MessageUtil::validate(input.config()); - ConfigImpl config(input.config(), factory_context, true); + ConfigImpl config(replaceInvalidHeaders(input.config()), factory_context, true); Http::TestHeaderMapImpl headers = Fuzz::fromHeaders(input.headers()); // It's a precondition of routing that {:authority, :path, x-forwarded-proto} headers exists, // HCM enforces this. diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc new file mode 100644 index 000000000000..98d32562e2e1 --- /dev/null +++ b/test/common/router/scoped_config_impl_test.cc @@ -0,0 +1,252 @@ +#include + +#include "common/router/scoped_config_impl.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Router { +namespace { + +using ::Envoy::Http::TestHeaderMapImpl; + +class FooFragment : public ScopeKeyFragmentBase { +private: + virtual bool equals(const ScopeKeyFragmentBase&) const { return true; }; +}; + +TEST(ScopeKeyFragmentBaseTest, EqualSign) { + FooFragment foo; + StringKeyFragment bar("a random string"); + + EXPECT_FALSE(foo == bar); +} + +TEST(StringKeyFragmentTest, Empty) { + StringKeyFragment a(""); + StringKeyFragment b(""); + EXPECT_EQ(a, b); + + StringKeyFragment non_empty("ABC"); + + EXPECT_FALSE(a == non_empty); +} + +TEST(StringKeyFragmentTest, Normal) { + StringKeyFragment str("Abc"); + + StringKeyFragment same_str("Abc"); + EXPECT_EQ(str, same_str); + + StringKeyFragment upper_cased_str("ABC"); + EXPECT_FALSE(str == upper_cased_str); + + StringKeyFragment another_str("DEF"); + EXPECT_FALSE(str == another_str); +} + +TEST(HeaderValueExtractorImplTest, InvalidConfig) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + // Type not set. + EXPECT_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); + + // Index non-zero when element separator is an empty string. + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: '' + index: 1 +)EOF"; + TestUtility::loadFromYaml(yaml_plain, config); + + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, + "when element separator is set to an empty string, index should be set " + "to 0 in HeaderValueExtractor.+"); + // extract_type not set. + yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: '' +)EOF"; + TestUtility::loadFromYaml(yaml_plain, config); + + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, + "HeaderValueExtractor extract_type not set.+"); +} + +TEST(HeaderValueExtractorImplTest, HeaderExtractionByIndex) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: ',' + index: 1 +)EOF"; + + TestUtility::loadFromYaml(yaml_plain, config); + HeaderValueExtractorImpl extractor(config); + std::unique_ptr fragment = + extractor.computeFragment(TestHeaderMapImpl{{"foo_header", "part-0,part-1:value_bluh"}}); + + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"part-1:value_bluh"}); + + // No such header. + fragment = extractor.computeFragment(TestHeaderMapImpl{{"bar_header", "part-0"}}); + EXPECT_EQ(fragment, nullptr); + + // Empty header value. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", ""}, + }); + EXPECT_EQ(fragment, nullptr); + + // Index out of bound. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0"}, + }); + EXPECT_EQ(fragment, nullptr); + + // Element is empty. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0,,,bluh"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment("")); +} + +TEST(HeaderValueExtractorImplTest, HeaderExtractionByKey) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: ';' + element: + key: 'bar' + separator: '=>' +)EOF"; + + TestUtility::loadFromYaml(yaml_plain, config); + HeaderValueExtractorImpl extractor(config); + std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0;bar=>bluh;foo=>foo_value"}, + }); + + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"bluh"}); + + // No such header. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"bluh", "part-0;"}, + }); + EXPECT_EQ(fragment, nullptr); + + // Empty header value. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", ""}, + }); + EXPECT_EQ(fragment, nullptr); + + // No such key. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0"}, + }); + EXPECT_EQ(fragment, nullptr); + + // Empty value. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bluh;;bar=>;foo=>last_value"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{""}); + + // Duplicate values, the first value returned. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bluh;;bar=>value1;bar=>value2;bluh;;bar=>last_value"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"value1"}); + + // No separator in the element, value is set to empty string. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bluh;;bar;bar=>value2;bluh;;bar=>last_value"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{""}); +} + +TEST(HeaderValueExtractorImplTest, ElementSeparatorEmpty) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: '' + element: + key: 'bar' + separator: '=' +)EOF"; + + TestUtility::loadFromYaml(yaml_plain, config); + HeaderValueExtractorImpl extractor(config); + std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bar=b;c=d;e=f"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"b;c=d;e=f"}); + + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "a=b;bar=d;e=f"}, + }); + EXPECT_EQ(fragment, nullptr); +} + +// Helper function which makes a ScopeKey from a list of strings. +ScopeKey makeKey(const std::vector& parts) { + ScopeKey key; + for (const auto& part : parts) { + if (part != nullptr) { + key.addFragment(std::make_unique(part)); + } else { + key.addFragment(nullptr); + } + } + return key; +} + +TEST(ScopeKeyTest, Unmatches) { + ScopeKey key1; + ScopeKey key2; + // Empty key != empty key. + EXPECT_NE(key1, key2); + + // Empty key != non-empty key. + EXPECT_NE(key1, makeKey({""})); + + EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + + // A null part matches nothing. + EXPECT_NE(makeKey({"a", "b", "c"}), makeKey({"a", "b", nullptr})); + + // A null part doesn't match "". + EXPECT_NE(makeKey({"a", "b", ""}), makeKey({"a", "b", nullptr})); + + // Two keys of different length won't match. + EXPECT_NE(makeKey({"a", "b"}), makeKey({"a", "b", "c"})); + // Case sensitive. + EXPECT_NE(makeKey({"a", "b"}), makeKey({"A", "b"})); +} + +TEST(ScopeKeyTest, Matches) { + // An empty string fragment equals another. + EXPECT_EQ(makeKey({"", ""}), makeKey({"", ""})); + EXPECT_EQ(makeKey({"a", "", ""}), makeKey({"a", "", ""})); + + // Non empty fragments comparision. + EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); +} + +} // namespace +} // namespace Router +} // namespace Envoy diff --git a/test/config_test/config_test.cc b/test/config_test/config_test.cc index 515fd8bec51c..b552cb9ec31b 100644 --- a/test/config_test/config_test.cc +++ b/test/config_test/config_test.cc @@ -6,6 +6,7 @@ #include "common/common/fmt.h" #include "common/protobuf/utility.h" +#include "common/runtime/runtime_features.h" #include "server/config_validation/server.h" #include "server/configuration_impl.h" @@ -39,6 +40,15 @@ OptionsImpl asConfigYaml(const OptionsImpl& src, Api::Api& api) { src.localAddressIpVersion()); } +class ScopedRuntimeInjector { +public: + ScopedRuntimeInjector(Runtime::Loader& runtime) { + Runtime::LoaderSingleton::initialize(&runtime); + } + + ~ScopedRuntimeInjector() { Runtime::LoaderSingleton::clear(); } +}; + } // namespace class ConfigTest { @@ -56,6 +66,20 @@ class ConfigTest { return api_->fileSystem().fileReadToEnd(file); })); + // Here we setup runtime to mimic the actual deprecated feature list used in the + // production code. Note that this test is actually more strict than production because + // in production runtime is not setup until after the bootstrap config is loaded. This seems + // better for configuration tests. + ScopedRuntimeInjector scoped_runtime(server_.runtime()); + ON_CALL(server_.runtime_loader_.snapshot_, deprecatedFeatureEnabled(_)) + .WillByDefault(Invoke([](const std::string& key) { + if (Runtime::RuntimeFeaturesDefaults::get().disallowedByDefault(key)) { + return false; + } else { + return true; + } + })); + envoy::config::bootstrap::v2::Bootstrap bootstrap; Server::InstanceUtil::loadBootstrapConfig(bootstrap, options_, server_.messageValidationVisitor(), *api_); diff --git a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc index ee9c679cecca..d7c40ab86991 100644 --- a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc +++ b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc @@ -28,16 +28,16 @@ namespace { class GrpcHttp1BridgeFilterTest : public testing::Test { public: - GrpcHttp1BridgeFilterTest() : common_(*symbol_table_), filter_(common_) { + GrpcHttp1BridgeFilterTest() : context_(*symbol_table_), filter_(context_) { filter_.setDecoderFilterCallbacks(decoder_callbacks_); filter_.setEncoderFilterCallbacks(encoder_callbacks_); ON_CALL(decoder_callbacks_.stream_info_, protocol()).WillByDefault(ReturnPointee(&protocol_)); } - ~GrpcHttp1BridgeFilterTest() { filter_.onDestroy(); } + ~GrpcHttp1BridgeFilterTest() override { filter_.onDestroy(); } Envoy::Test::Global symbol_table_; - Grpc::Common common_; + Grpc::ContextImpl context_; Http1BridgeFilter filter_; NiceMock decoder_callbacks_; NiceMock encoder_callbacks_; diff --git a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc index aa9915db1595..1500708fe3c6 100644 --- a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc +++ b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc @@ -107,7 +107,7 @@ class GrpcWebFilterTest : public testing::TestWithParam symbol_table_; - Grpc::Common grpc_context_; + Grpc::ContextImpl grpc_context_; GrpcWebFilter filter_; NiceMock decoder_callbacks_; NiceMock encoder_callbacks_; diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index 3195a9bff53a..ce334636c88e 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -13,11 +13,13 @@ load( envoy_package() envoy_cc_test( - name = "dummy_test", - srcs = ["dummy_test.cc"], - external_deps = ["quiche_http2_platform"], + name = "envoy_quic_alarm_test", + srcs = ["envoy_quic_alarm_test.cc"], + external_deps = ["quiche_quic_platform"], deps = [ - "//source/extensions/quic_listeners/quiche:dummy_lib", - "//test/test_common:utility_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_lib", + "//source/extensions/quic_listeners/quiche/platform:envoy_quic_clock_lib", + "//test/test_common:simulated_time_system_lib", ], ) diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc new file mode 100644 index 000000000000..6dc18d378150 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc @@ -0,0 +1,194 @@ +#include "common/event/libevent_scheduler.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_alarm.h" +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" +#include "extensions/quic_listeners/quiche/platform/envoy_quic_clock.h" + +#include "test/test_common/simulated_time_system.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using Envoy::Event::Dispatcher; +using quic::QuicTime; + +namespace Envoy { +namespace Quic { + +class TestDelegate : public quic::QuicAlarm::Delegate { +public: + TestDelegate() : fired_(false) {} + + // quic::QuicAlarm::Delegate + void OnAlarm() override { fired_ = true; } + + bool fired() const { return fired_; } + void set_fired(bool fired) { fired_ = fired; } + +private: + bool fired_; +}; + +class EnvoyQuicAlarmTest : public ::testing::Test { +public: + EnvoyQuicAlarmTest() + : clock_(time_system_), scheduler_(time_system_.createScheduler(base_scheduler_)), + alarm_factory_(*scheduler_, clock_) {} + + void advanceMsAndLoop(int64_t delay_ms) { + time_system_.sleep(std::chrono::milliseconds(delay_ms)); + base_scheduler_.run(Dispatcher::RunType::NonBlock); + } + +protected: + Event::SimulatedTimeSystemHelper time_system_; + EnvoyQuicClock clock_; + Event::LibeventScheduler base_scheduler_; + Event::SchedulerPtr scheduler_; + EnvoyQuicAlarmFactory alarm_factory_; + quic::QuicConnectionArena arena_; +}; + +TEST_F(EnvoyQuicAlarmTest, CreateAlarmByFactory) { + auto unowned_delegate = new TestDelegate(); + quic::QuicAlarm* alarm = alarm_factory_.CreateAlarm(unowned_delegate); + alarm->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + // Advance 9us, alarm shouldn't fire. + advanceMsAndLoop(9); + EXPECT_FALSE(unowned_delegate->fired()); + // Advance 1us, alarm should have fired. + advanceMsAndLoop(1); + EXPECT_TRUE(unowned_delegate->fired()); + delete alarm; + + unowned_delegate = new TestDelegate(); + quic::QuicArenaScopedPtr alarm_ptr = alarm_factory_.CreateAlarm( + quic::QuicArenaScopedPtr(unowned_delegate), &arena_); + EXPECT_FALSE(alarm_ptr->IsSet()); + unowned_delegate = new TestDelegate(); + alarm_ptr = alarm_factory_.CreateAlarm( + quic::QuicArenaScopedPtr(unowned_delegate), nullptr); + EXPECT_FALSE(alarm_ptr->IsSet()); +} + +TEST_F(EnvoyQuicAlarmTest, CreateAlarmAndCancel) { + auto unowned_delegate1 = new TestDelegate(); + quic::QuicArenaScopedPtr alarm1(alarm_factory_.CreateAlarm(unowned_delegate1)); + alarm1->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + EXPECT_TRUE(alarm1->IsSet()); + auto unowned_delegate2 = new TestDelegate(); + quic::QuicArenaScopedPtr alarm2(alarm_factory_.CreateAlarm(unowned_delegate2)); + alarm2->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + EXPECT_TRUE(alarm2->IsSet()); + + alarm1->Cancel(); + EXPECT_FALSE(alarm1->IsSet()); + // Advance 10us, alarm1 shouldn't fire, but alarm2 should. + advanceMsAndLoop(10); + EXPECT_TRUE(unowned_delegate2->fired()); + EXPECT_FALSE(unowned_delegate1->fired()); +} + +TEST_F(EnvoyQuicAlarmTest, CreateAlarmAndReset) { + auto unowned_delegate1 = new TestDelegate(); + quic::QuicArenaScopedPtr alarm1(alarm_factory_.CreateAlarm(unowned_delegate1)); + alarm1->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + auto unowned_delegate2 = new TestDelegate(); + quic::QuicArenaScopedPtr alarm2(alarm_factory_.CreateAlarm(unowned_delegate2)); + alarm2->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + EXPECT_TRUE(alarm2->IsSet()); + + // Reset alarm1 to a different deadline. + alarm1->Cancel(); + alarm1->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(5)); + // Advance 9us, alarm1 should have fired but alarm2 shouldn't. + advanceMsAndLoop(9); + EXPECT_TRUE(unowned_delegate1->fired()); + EXPECT_FALSE(unowned_delegate2->fired()); + + advanceMsAndLoop(1); + EXPECT_TRUE(unowned_delegate2->fired()); +} + +TEST_F(EnvoyQuicAlarmTest, CreateAlarmAndUpdate) { + auto unowned_delegate1 = new TestDelegate(); + quic::QuicArenaScopedPtr alarm1(alarm_factory_.CreateAlarm(unowned_delegate1)); + alarm1->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + auto unowned_delegate2 = new TestDelegate(); + quic::QuicArenaScopedPtr alarm2(alarm_factory_.CreateAlarm(unowned_delegate2)); + alarm2->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + EXPECT_TRUE(alarm2->IsSet()); + + // Update alarm1 to an earlier deadline. + alarm1->Update(clock_.Now() + QuicTime::Delta::FromMilliseconds(5), + quic::QuicTime::Delta::Zero()); + // Advance 9us, alarm1 should have fired but alarm2 shouldn't. + advanceMsAndLoop(9); + EXPECT_TRUE(unowned_delegate1->fired()); + EXPECT_FALSE(unowned_delegate2->fired()); + + advanceMsAndLoop(1); + EXPECT_TRUE(unowned_delegate2->fired()); +} + +TEST_F(EnvoyQuicAlarmTest, PostponeDeadline) { + auto unowned_delegate = new TestDelegate(); + quic::QuicArenaScopedPtr alarm(alarm_factory_.CreateAlarm(unowned_delegate)); + alarm->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + advanceMsAndLoop(9); + EXPECT_FALSE(unowned_delegate->fired()); + // Postpone deadline to a later time. + alarm->Update(clock_.Now() + QuicTime::Delta::FromMilliseconds(5), quic::QuicTime::Delta::Zero()); + advanceMsAndLoop(1); + EXPECT_EQ(10, (clock_.Now() - quic::QuicTime::Zero()).ToMilliseconds()); + // alarm shouldn't fire at old deadline. + EXPECT_FALSE(unowned_delegate->fired()); + + advanceMsAndLoop(4); + // alarm should fire at new deadline. + EXPECT_TRUE(unowned_delegate->fired()); +} + +TEST_F(EnvoyQuicAlarmTest, SetAlarmToPastTime) { + advanceMsAndLoop(100); + EXPECT_EQ(100, (clock_.Now() - quic::QuicTime::Zero()).ToMilliseconds()); + auto unowned_delegate = new TestDelegate(); + quic::QuicArenaScopedPtr alarm(alarm_factory_.CreateAlarm(unowned_delegate)); + // alarm becomes active upon Set(). + alarm->Set(clock_.Now() - QuicTime::Delta::FromMilliseconds(10)); + EXPECT_FALSE(unowned_delegate->fired()); + base_scheduler_.run(Dispatcher::RunType::NonBlock); + EXPECT_TRUE(unowned_delegate->fired()); +} + +TEST_F(EnvoyQuicAlarmTest, UpdateAlarmWithPastDeadline) { + auto unowned_delegate = new TestDelegate(); + quic::QuicArenaScopedPtr alarm(alarm_factory_.CreateAlarm(unowned_delegate)); + alarm->Set(clock_.Now() + QuicTime::Delta::FromMilliseconds(10)); + advanceMsAndLoop(9); + EXPECT_EQ(9, (clock_.Now() - quic::QuicTime::Zero()).ToMilliseconds()); + EXPECT_FALSE(unowned_delegate->fired()); + // alarm becomes active upon Update(). + alarm->Update(clock_.Now() - QuicTime::Delta::FromMilliseconds(1), quic::QuicTime::Delta::Zero()); + base_scheduler_.run(Dispatcher::RunType::NonBlock); + EXPECT_TRUE(unowned_delegate->fired()); + unowned_delegate->set_fired(false); + advanceMsAndLoop(1); + // alarm shouldn't fire at the original deadline. + EXPECT_FALSE(unowned_delegate->fired()); +} + +TEST_F(EnvoyQuicAlarmTest, CancelActiveAlarm) { + advanceMsAndLoop(100); + EXPECT_EQ(100, (clock_.Now() - quic::QuicTime::Zero()).ToMilliseconds()); + auto unowned_delegate = new TestDelegate(); + quic::QuicArenaScopedPtr alarm(alarm_factory_.CreateAlarm(unowned_delegate)); + // alarm becomes active upon Set(). + alarm->Set(clock_.Now() - QuicTime::Delta::FromMilliseconds(10)); + alarm->Cancel(); + base_scheduler_.run(Dispatcher::RunType::NonBlock); + EXPECT_FALSE(unowned_delegate->fired()); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc index 09920ae029da..b253dd30899e 100644 --- a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc +++ b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc @@ -89,7 +89,7 @@ class LightStepDriverTest : public testing::Test { StreamInfo::MockStreamInfo stream_info_; Envoy::Test::Global symbol_table_; - Grpc::Common grpc_context_; + Grpc::ContextImpl grpc_context_; NiceMock tls_; Stats::IsolatedStoreImpl stats_; std::unique_ptr driver_; diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 4df3a31141cb..9100cc57e194 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -430,6 +430,23 @@ TEST_P(Http2IntegrationTest, GrpcRouterNotFound) { TEST_P(Http2IntegrationTest, GrpcRetry) { testGrpcRetry(); } +// Verify the case where there is an HTTP/2 codec/protocol error with an active stream. +TEST_P(Http2IntegrationTest, CodecErrorAfterStreamStart) { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Sends a request. + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 10); + waitForNextUpstreamRequest(); + + // Send bogus raw data on the connection. + Buffer::OwnedImpl bogus_data("some really bogus data"); + codec_client_->rawConnection().write(bogus_data, false); + + // Verifies reset is received. + response->waitForReset(); +} + TEST_P(Http2IntegrationTest, BadMagic) { initialize(); Buffer::OwnedImpl buffer("hello"); diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index c914831cb6b6..00beabdd0114 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -39,6 +39,7 @@ class IntegrationCodecClient : public Http::CodecClientProd { bool waitForDisconnect(std::chrono::milliseconds time_to_wait = std::chrono::milliseconds(0)); Network::ClientConnection* connection() const { return connection_.get(); } Network::ConnectionEvent last_connection_event() const { return last_connection_event_; } + Network::Connection& rawConnection() { return *connection_; } private: struct ConnectionCallbacks : public Network::ConnectionCallbacks { diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 09adf7ca6382..aa0a0fdb3f56 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -534,6 +534,8 @@ TEST_P(StatsMatcherIntegrationTest, ExcludeMultipleExact) { // blocks on its creation (see waitForCounterGe and the suite of waitFor* functions). // If this invariant is changed, this test must be rewritten. TEST_P(StatsMatcherIntegrationTest, IncludeExact) { + // Stats matching does not play well with LDS, at least in test. See #7215. + use_lds_ = false; stats_matcher_.mutable_inclusion_list()->add_patterns()->set_exact( "listener_manager.listener_create_success"); initialize(); diff --git a/test/mocks/server/BUILD b/test/mocks/server/BUILD index fd64aafbf173..f6384d37070d 100644 --- a/test/mocks/server/BUILD +++ b/test/mocks/server/BUILD @@ -26,6 +26,7 @@ envoy_cc_mock( "//include/envoy/server:worker_interface", "//include/envoy/ssl:context_manager_interface", "//include/envoy/upstream:health_checker_interface", + "//source/common/grpc:context_lib", "//source/common/http:context_lib", "//source/common/secret:secret_manager_impl_lib", "//source/common/singleton:manager_impl_lib", diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 57fa1b858650..08c08f7bfccf 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -20,6 +20,7 @@ #include "envoy/stats/scope.h" #include "envoy/thread/thread.h" +#include "common/grpc/context_impl.h" #include "common/http/context_impl.h" #include "common/secret/secret_manager_impl.h" #include "common/stats/fake_symbol_table_impl.h" @@ -403,7 +404,7 @@ class MockInstance : public Instance { testing::NiceMock listener_manager_; testing::NiceMock overload_manager_; Singleton::ManagerPtr singleton_manager_; - Grpc::Common grpc_context_; + Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; }; @@ -478,7 +479,7 @@ class MockFactoryContext : public virtual FactoryContext { Stats::IsolatedStoreImpl listener_scope_; Event::GlobalTimeSystem time_system_; testing::NiceMock overload_manager_; - Grpc::Common grpc_context_; + Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; testing::NiceMock api_; }; diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 0d9fcfd63d7d..d8ac9debd063 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1391,8 +1391,8 @@ TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { {{"another_tag_name_4", "another_tag_4-value"}}); Buffer::OwnedImpl response; - auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); EXPECT_EQ(2UL, size); } @@ -1411,8 +1411,8 @@ TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { {{"another_tag_name_4", "another_tag_4-value"}}); Buffer::OwnedImpl response; - auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); EXPECT_EQ(4UL, size); } @@ -1430,8 +1430,8 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { addHistogram(histogram); Buffer::OwnedImpl response; - auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -1483,8 +1483,8 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { addHistogram(histogram); Buffer::OwnedImpl response; - auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -1535,8 +1535,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); Buffer::OwnedImpl response; - auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); EXPECT_EQ(5UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter @@ -1595,8 +1595,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); Buffer::OwnedImpl response; - auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, true); + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + true, absl::nullopt); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram @@ -1645,7 +1645,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { Buffer::OwnedImpl response; auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, - response, used_only); + response, used_only, absl::nullopt); EXPECT_EQ(0UL, size); } @@ -1656,10 +1656,40 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { Buffer::OwnedImpl response; auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, - response, used_only); + response, used_only, absl::nullopt); EXPECT_EQ(1UL, size); } } +TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { + addCounter("cluster.test_1.upstream_cx_total", {{"a.tag-name", "a.tag-value"}}); + addCounter("cluster.test_2.upstream_cx_total", {{"another_tag_name", "another_tag-value"}}); + addGauge("cluster.test_3.upstream_cx_total", {{"another_tag_name_3", "another_tag_3-value"}}); + addGauge("cluster.test_4.upstream_cx_total", {{"another_tag_name_4", "another_tag_4-value"}}); + + const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = std::make_shared>(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); + addHistogram(histogram1); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, response, false, + absl::optional{std::regex("cluster.test_1.upstream_cx_total")}); + EXPECT_EQ(1UL, size); + + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter +envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + } // namespace Server } // namespace Envoy diff --git a/test/test_common/simulated_time_system.cc b/test/test_common/simulated_time_system.cc index 5cf614ba6c1f..1e700f650832 100644 --- a/test/test_common/simulated_time_system.cc +++ b/test/test_common/simulated_time_system.cc @@ -62,7 +62,7 @@ class SimulatedTimeSystemHelper::Alarm : public Timer { void enableTimer(const std::chrono::milliseconds& duration) override; bool enabled() override { Thread::LockGuard lock(time_system_.mutex_); - return armed_; + return armed_ || base_timer_->enabled(); } void disableTimerLockHeld() EXCLUSIVE_LOCKS_REQUIRED(time_system_.mutex_); diff --git a/test/test_common/simulated_time_system_test.cc b/test/test_common/simulated_time_system_test.cc index 941aae844202..399ee31a87b1 100644 --- a/test/test_common/simulated_time_system_test.cc +++ b/test/test_common/simulated_time_system_test.cc @@ -233,6 +233,12 @@ TEST_F(SimulatedTimeSystemTest, DuplicateTimer) { thread->join(); } +TEST_F(SimulatedTimeSystemTest, Enabled) { + TimerPtr timer = scheduler_->createTimer({}); + timer->enableTimer(std::chrono::milliseconds(0)); + EXPECT_TRUE(timer->enabled()); +} + } // namespace } // namespace Test } // namespace Event From bb55a4498d3792c6d704cd5b964dad4e61e35bf3 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 11 Jun 2019 17:14:26 -0400 Subject: [PATCH 02/53] fix typo Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 98d32562e2e1..50c61eee3d5e 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -243,7 +243,7 @@ TEST(ScopeKeyTest, Matches) { EXPECT_EQ(makeKey({"", ""}), makeKey({"", ""})); EXPECT_EQ(makeKey({"a", "", ""}), makeKey({"a", "", ""})); - // Non empty fragments comparision. + // Non empty fragments comparison. EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); } From 8d7b4da5ab00393ba963b8311d696dff617a3b56 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 11 Jun 2019 22:14:11 -0400 Subject: [PATCH 03/53] move impl code from header to cc file. Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 45 +++++++++++++++++++++ source/common/router/scoped_config_impl.h | 46 ++-------------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index e32887b7612f..4b5c995c52f8 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -3,6 +3,51 @@ namespace Envoy { namespace Router { +bool ScopeKey::operator!=(const ScopeKey& other) const { return !(*this == other); } + +bool ScopeKey::operator==(const ScopeKey& other) const { + if (this->fragments_.size() != other.fragments_.size()) { + return false; + } + if (this->fragments_.size() == 0 || other.fragments_.size() == 0) { + // An empty key equals to nothing, "NULL" != "NULL". + return false; + } + // A "NULL" fragment value equals to nothing. + if (this->contains_null_fragment_ || other.contains_null_fragment_) { + return false; + } + return std::equal(this->fragments_.begin(), this->fragments_.end(), other.fragments_.begin(), + [](const std::unique_ptr& left, + const std::unique_ptr& right) -> bool { + // Both should be non-NULL now. + return *left == *right; + }); +} + +HeaderValueExtractorImpl::HeaderValueExtractorImpl( + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) + : FragmentBuilderBase(config), + header_value_extractor_config_(config_.header_value_extractor()) { + ASSERT(config_.type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor, + "header_value_extractor is not set."); + if (header_value_extractor_config_.extract_type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex) { + if (header_value_extractor_config_.index() != 0 && + header_value_extractor_config_.element_separator().length() == 0) { + throw ProtoValidationException("when element separator is set to an empty string, index " + "should be set to 0 in HeaderValueExtractor.", + header_value_extractor_config_); + } + } + if (header_value_extractor_config_.extract_type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::EXTRACT_TYPE_NOT_SET) { + throw ProtoValidationException("HeaderValueExtractor extract_type not set.", + header_value_extractor_config_); + } +} + std::unique_ptr HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const { const Envoy::Http::HeaderEntry* header_entry = diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 3153da37baff..ac9a8fcaee34 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -44,29 +44,12 @@ class ScopeKey { fragments_.emplace_back(std::move(part)); } - bool operator!=(const ScopeKey& other) const { return !(*this == other); } + bool operator!=(const ScopeKey& other) const; - bool operator==(const ScopeKey& other) const { - if (this->fragments_.size() != other.fragments_.size()) { - return false; - } - if (this->fragments_.size() == 0 || other.fragments_.size() == 0) { - // An empty key equals to nothing, "NULL" != "NULL". - return false; - } - // A "NULL" fragment value equals to nothing. - if (this->contains_null_fragment_ || other.contains_null_fragment_) { - return false; - } - return std::equal(this->fragments_.begin(), this->fragments_.end(), other.fragments_.begin(), - [](const std::unique_ptr& left, - const std::unique_ptr& right) -> bool { - // Both should be non-NULL now. - return *left == *right; - }); - } + bool operator==(const ScopeKey& other) const; private: + // If there is a NULL fragment in the key. bool contains_null_fragment_{false}; std::vector> fragments_; }; @@ -104,28 +87,7 @@ class FragmentBuilderBase { class HeaderValueExtractorImpl : public FragmentBuilderBase { public: - explicit HeaderValueExtractorImpl(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) - : FragmentBuilderBase(config), - header_value_extractor_config_(config_.header_value_extractor()) { - ASSERT(config_.type_case() == - ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor, - "header_value_extractor is not set."); - if (header_value_extractor_config_.extract_type_case() == - ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex) { - if (header_value_extractor_config_.index() != 0 && - header_value_extractor_config_.element_separator().length() == 0) { - throw ProtoValidationException("when element separator is set to an empty string, index " - "should be set to 0 in HeaderValueExtractor.", - header_value_extractor_config_); - } - } - if (header_value_extractor_config_.extract_type_case() == - ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor:: - EXTRACT_TYPE_NOT_SET) { - throw ProtoValidationException("HeaderValueExtractor extract_type not set.", - header_value_extractor_config_); - } - } + explicit HeaderValueExtractorImpl(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config); std::unique_ptr computeFragment(const Http::HeaderMap& headers) const override; From 8bcddcec0d22b1c0f06d1e2d2271f5e10604ec59 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 12 Jun 2019 14:11:57 -0400 Subject: [PATCH 04/53] add review fixes and unit test case for keybuidler Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 16 ++-- source/common/router/scoped_config_impl.h | 20 ++-- test/common/router/scoped_config_impl_test.cc | 93 +++++++++++++++---- 3 files changed, 99 insertions(+), 30 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 4b5c995c52f8..4aeff6665760 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -13,10 +13,6 @@ bool ScopeKey::operator==(const ScopeKey& other) const { // An empty key equals to nothing, "NULL" != "NULL". return false; } - // A "NULL" fragment value equals to nothing. - if (this->contains_null_fragment_ || other.contains_null_fragment_) { - return false; - } return std::equal(this->fragments_.begin(), this->fragments_.end(), other.fragments_.begin(), [](const std::unique_ptr& left, const std::unique_ptr& right) -> bool { @@ -96,12 +92,18 @@ ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder config) } } -const ScopeKey ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { +std::unique_ptr +ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { ScopeKey key; for (const auto& builder : fragment_builders_) { - key.addFragment(builder->computeFragment(headers)); + // returns nullopt if a null fragment is found. + std::unique_ptr fragment = builder->computeFragment(headers); + if (fragment == nullptr) { + return nullptr; + } + key.addFragment(std::move(fragment)); } - return key; + return std::make_unique(std::move(key)); } void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index ac9a8fcaee34..b1fd0ee8652b 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -22,6 +22,8 @@ using envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes; */ class ScopeKeyFragmentBase { public: + bool operator!=(const ScopeKeyFragmentBase& other) const { return !(*this == other); } + bool operator==(const ScopeKeyFragmentBase& other) const { if (typeid(*this) == typeid(other)) { return equals(other); @@ -35,13 +37,18 @@ class ScopeKeyFragmentBase { virtual bool equals(const ScopeKeyFragmentBase&) const PURE; }; +/** + * Scope Key is composed of non-null fragments. + * + **/ class ScopeKey { public: ScopeKey() = default; ScopeKey(ScopeKey&& other) = default; - void addFragment(std::unique_ptr&& part) { - contains_null_fragment_ = contains_null_fragment_ || (part == nullptr); - fragments_.emplace_back(std::move(part)); + + // Caller should guarantee the fragment is not nullptr. + void addFragment(std::unique_ptr&& fragment) { + fragments_.emplace_back(std::move(fragment)); } bool operator!=(const ScopeKey& other) const; @@ -49,8 +56,6 @@ class ScopeKey { bool operator==(const ScopeKey& other) const; private: - // If there is a NULL fragment in the key. - bool contains_null_fragment_{false}; std::vector> fragments_; }; @@ -105,7 +110,8 @@ class ScopeKeyBuilderBase { explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder config) : config_(std::move(config)) {} virtual ~ScopeKeyBuilderBase() = default; - virtual const ScopeKey computeScopeKey(const Http::HeaderMap& headers) const PURE; + // Computes scope key for given headers, returns nullptr if a key can't be computed. + virtual std::unique_ptr computeScopeKey(const Http::HeaderMap& headers) const PURE; protected: ScopedRoutes::ScopeKeyBuilder config_; @@ -115,7 +121,7 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { public: explicit ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder config); - const ScopeKey computeScopeKey(const Http::HeaderMap& headers) const override; + std::unique_ptr computeScopeKey(const Http::HeaderMap& headers) const override; private: std::vector> fragment_builders_; diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 50c61eee3d5e..5cf21de16af0 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -21,7 +21,7 @@ TEST(ScopeKeyFragmentBaseTest, EqualSign) { FooFragment foo; StringKeyFragment bar("a random string"); - EXPECT_FALSE(foo == bar); + EXPECT_NE(foo, bar); } TEST(StringKeyFragmentTest, Empty) { @@ -31,7 +31,7 @@ TEST(StringKeyFragmentTest, Empty) { StringKeyFragment non_empty("ABC"); - EXPECT_FALSE(a == non_empty); + EXPECT_NE(a, non_empty); } TEST(StringKeyFragmentTest, Normal) { @@ -41,13 +41,13 @@ TEST(StringKeyFragmentTest, Normal) { EXPECT_EQ(str, same_str); StringKeyFragment upper_cased_str("ABC"); - EXPECT_FALSE(str == upper_cased_str); + EXPECT_NE(str, upper_cased_str); StringKeyFragment another_str("DEF"); - EXPECT_FALSE(str == another_str); + EXPECT_NE(str, another_str); } -TEST(HeaderValueExtractorImplTest, InvalidConfig) { +TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; // Type not set. EXPECT_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); @@ -206,11 +206,7 @@ TEST(HeaderValueExtractorImplTest, ElementSeparatorEmpty) { ScopeKey makeKey(const std::vector& parts) { ScopeKey key; for (const auto& part : parts) { - if (part != nullptr) { - key.addFragment(std::make_unique(part)); - } else { - key.addFragment(nullptr); - } + key.addFragment(std::make_unique(part)); } return key; } @@ -226,14 +222,9 @@ TEST(ScopeKeyTest, Unmatches) { EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); - // A null part matches nothing. - EXPECT_NE(makeKey({"a", "b", "c"}), makeKey({"a", "b", nullptr})); - - // A null part doesn't match "". - EXPECT_NE(makeKey({"a", "b", ""}), makeKey({"a", "b", nullptr})); - // Two keys of different length won't match. EXPECT_NE(makeKey({"a", "b"}), makeKey({"a", "b", "c"})); + // Case sensitive. EXPECT_NE(makeKey({"a", "b"}), makeKey({"A", "b"})); } @@ -247,6 +238,76 @@ TEST(ScopeKeyTest, Matches) { EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); } +TEST(ScopeKeyBuilderImplTest, Parse) { + std::string yaml_plain = R"EOF( + fragments: + - header_value_extractor: + name: 'foo_header' + element_separator: ',' + element: + key: 'bar' + separator: '=' + - header_value_extractor: + name: 'bar_header' + element_separator: ';' + index: 2 +)EOF"; + + ScopedRoutes::ScopeKeyBuilder config; + TestUtility::loadFromYaml(yaml_plain, config); + ScopeKeyBuilderImpl key_builder(config); + + std::unique_ptr key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar=bar_value,e=f"}, + {"bar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_NE(key, nullptr); + EXPECT_EQ(*key, makeKey({"bar_value", "index2"})); + + // Empty string fragment is fine. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar,e=f"}, + {"bar_header", "a=b;bar=bar_value;"}, + }); + EXPECT_NE(key, nullptr); + EXPECT_EQ(*key, makeKey({"", ""})); + + // Key not found. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,meh,e=f"}, + {"bar_header", "a=b;bar=bar_value;"}, + }); + EXPECT_EQ(key, nullptr); + + // Index out of bound. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar=bar_value,e=f"}, + {"bar_header", "a=b;bar=bar_value"}, + }); + EXPECT_EQ(key, nullptr); + + // Header missing. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar=bar_value,e=f"}, + {"foobar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_EQ(key, nullptr); + + // Header value empty. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", ""}, + {"bar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_EQ(key, nullptr); + + // Case sensitive. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,Bar=bar_value,e=f"}, + {"bar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_EQ(key, nullptr); +} + } // namespace } // namespace Router } // namespace Envoy From b1d0d5c08f9a4a9768824f171fa9be9994022e1d Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 12 Jun 2019 15:51:18 -0400 Subject: [PATCH 05/53] clang-tidy fix Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 4 ++-- test/common/router/scoped_config_impl_test.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 4aeff6665760..07f86a6a1243 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -9,7 +9,7 @@ bool ScopeKey::operator==(const ScopeKey& other) const { if (this->fragments_.size() != other.fragments_.size()) { return false; } - if (this->fragments_.size() == 0 || other.fragments_.size() == 0) { + if (this->fragments_.empty() || other.fragments_.empty()) { // An empty key equals to nothing, "NULL" != "NULL". return false; } @@ -23,7 +23,7 @@ bool ScopeKey::operator==(const ScopeKey& other) const { HeaderValueExtractorImpl::HeaderValueExtractorImpl( ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) - : FragmentBuilderBase(config), + : FragmentBuilderBase(std::move(config)), header_value_extractor_config_(config_.header_value_extractor()) { ASSERT(config_.type_case() == ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor, diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 5cf21de16af0..e8d8917e1a51 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -14,7 +14,7 @@ using ::Envoy::Http::TestHeaderMapImpl; class FooFragment : public ScopeKeyFragmentBase { private: - virtual bool equals(const ScopeKeyFragmentBase&) const { return true; }; + bool equals(const ScopeKeyFragmentBase&) const override { return true; }; }; TEST(ScopeKeyFragmentBaseTest, EqualSign) { From 1007ee08b86bef33f84fa9c08e3594ee5ab31449 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 12 Jun 2019 16:08:26 -0400 Subject: [PATCH 06/53] use expect_debug_death for assertion test Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index e8d8917e1a51..1060f6948e44 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -50,7 +50,7 @@ TEST(StringKeyFragmentTest, Normal) { TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; // Type not set. - EXPECT_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); + EXPECT_DEBUG_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); // Index non-zero when element separator is an empty string. std::string yaml_plain = R"EOF( From 184ff2f501e11995cdb515f24c454849b3d08aa1 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 12 Jun 2019 17:41:39 -0400 Subject: [PATCH 07/53] fix failed test in opt mode Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 1060f6948e44..39524b60a2f0 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -49,8 +49,10 @@ TEST(StringKeyFragmentTest, Normal) { TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; - // Type not set. + // Type not set, ASSERT only fails in debug mode. +#if !defined(NDEBUG) EXPECT_DEBUG_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); +#endif // !defined(NDEBUG) // Index non-zero when element separator is an empty string. std::string yaml_plain = R"EOF( From fe6b643e8f51f6b8216ad55cbd0f851adb88d58e Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 13 Jun 2019 17:36:25 -0400 Subject: [PATCH 08/53] add assert around addFragment and test, and a few review fix Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 4 +--- source/common/router/scoped_config_impl.h | 6 +++++- test/common/router/scoped_config_impl_test.cc | 10 ++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 07f86a6a1243..49e5a44d4496 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -6,14 +6,12 @@ namespace Router { bool ScopeKey::operator!=(const ScopeKey& other) const { return !(*this == other); } bool ScopeKey::operator==(const ScopeKey& other) const { - if (this->fragments_.size() != other.fragments_.size()) { - return false; - } if (this->fragments_.empty() || other.fragments_.empty()) { // An empty key equals to nothing, "NULL" != "NULL". return false; } return std::equal(this->fragments_.begin(), this->fragments_.end(), other.fragments_.begin(), + other.fragments_.end(), [](const std::unique_ptr& left, const std::unique_ptr& right) -> bool { // Both should be non-NULL now. diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index b1fd0ee8652b..3485895c07f6 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -39,15 +39,19 @@ class ScopeKeyFragmentBase { /** * Scope Key is composed of non-null fragments. - * **/ class ScopeKey { public: ScopeKey() = default; ScopeKey(ScopeKey&& other) = default; + // Scopekey is not copy-assignable and copy-constructible as it contains unique_ptr inside itself. + ScopeKey(const ScopeKey&) = delete; + ScopeKey operator=(const ScopeKey&) = delete; + // Caller should guarantee the fragment is not nullptr. void addFragment(std::unique_ptr&& fragment) { + ASSERT(fragment != nullptr, "null fragment not allowed in ScopeKey."); fragments_.emplace_back(std::move(fragment)); } diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 39524b60a2f0..c086ed03a5a5 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -52,6 +52,9 @@ TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { // Type not set, ASSERT only fails in debug mode. #if !defined(NDEBUG) EXPECT_DEBUG_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); +#else + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, + "HeaderValueExtractor extract_type not set.+"); #endif // !defined(NDEBUG) // Index non-zero when element separator is an empty string. @@ -213,6 +216,13 @@ ScopeKey makeKey(const std::vector& parts) { return key; } +TEST(ScopeKeyDeathTest, AddNullFragment) { + ScopeKey key; +#if !defined(NDEBUG) + EXPECT_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); +#endif // !defined(NDEBUG) +} + TEST(ScopeKeyTest, Unmatches) { ScopeKey key1; ScopeKey key2; From 75718d96a9ad9bb07c122e9d9b525d93a0bccf30 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 21 Jun 2019 16:45:17 -0400 Subject: [PATCH 09/53] fix based on feedbacks Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index c086ed03a5a5..5ba2da84fd18 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -51,7 +51,7 @@ TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; // Type not set, ASSERT only fails in debug mode. #if !defined(NDEBUG) - EXPECT_DEBUG_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); + EXPECT_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); #else EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, "HeaderValueExtractor extract_type not set.+"); @@ -218,9 +218,7 @@ ScopeKey makeKey(const std::vector& parts) { TEST(ScopeKeyDeathTest, AddNullFragment) { ScopeKey key; -#if !defined(NDEBUG) - EXPECT_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); -#endif // !defined(NDEBUG) + EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); } TEST(ScopeKeyTest, Unmatches) { From d84839dcc9d8664a2bd634ab5a1576b62876b7ed Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 25 Jun 2019 10:20:50 -0400 Subject: [PATCH 10/53] fix nits Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 18 +++++++++--------- source/common/router/scoped_config_impl.h | 16 ++++++++-------- source/common/router/scoped_rds.cc | 2 +- test/common/router/scoped_config_impl_test.cc | 19 +++++++++---------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 49e5a44d4496..ee3ce1318568 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -6,11 +6,11 @@ namespace Router { bool ScopeKey::operator!=(const ScopeKey& other) const { return !(*this == other); } bool ScopeKey::operator==(const ScopeKey& other) const { - if (this->fragments_.empty() || other.fragments_.empty()) { + if (fragments_.empty() || other.fragments_.empty()) { // An empty key equals to nothing, "NULL" != "NULL". return false; } - return std::equal(this->fragments_.begin(), this->fragments_.end(), other.fragments_.begin(), + return std::equal(fragments_.begin(), fragments_.end(), other.fragments_.begin(), other.fragments_.end(), [](const std::unique_ptr& left, const std::unique_ptr& right) -> bool { @@ -20,7 +20,7 @@ bool ScopeKey::operator==(const ScopeKey& other) const { } HeaderValueExtractorImpl::HeaderValueExtractorImpl( - ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder&& config) : FragmentBuilderBase(std::move(config)), header_value_extractor_config_(config_.header_value_extractor()) { ASSERT(config_.type_case() == @@ -29,9 +29,8 @@ HeaderValueExtractorImpl::HeaderValueExtractorImpl( if (header_value_extractor_config_.extract_type_case() == ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex) { if (header_value_extractor_config_.index() != 0 && - header_value_extractor_config_.element_separator().length() == 0) { - throw ProtoValidationException("when element separator is set to an empty string, index " - "should be set to 0 in HeaderValueExtractor.", + header_value_extractor_config_.element_separator().empty()) { + throw ProtoValidationException("Index > 0 for empty string element separator.", header_value_extractor_config_); } } @@ -77,12 +76,13 @@ HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const return nullptr; } -ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder config) - : ScopeKeyBuilderBase(config) { +ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config) + : ScopeKeyBuilderBase(std::move(config)) { for (const auto& fragment_builder : config_.fragments()) { switch (fragment_builder.type_case()) { case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor: - fragment_builders_.emplace_back(std::make_unique(fragment_builder)); + fragment_builders_.emplace_back(std::make_unique( + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder(fragment_builder))); break; default: NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 3485895c07f6..1fea085986cb 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -73,7 +73,7 @@ class StringKeyFragment : public ScopeKeyFragmentBase { return value_ == static_cast(other).value_; } - std::string value_; + const std::string value_; }; /** @@ -81,7 +81,7 @@ class StringKeyFragment : public ScopeKeyFragmentBase { */ class FragmentBuilderBase { public: - explicit FragmentBuilderBase(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config) + explicit FragmentBuilderBase(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder&& config) : config_(std::move(config)) {} virtual ~FragmentBuilderBase() = default; @@ -96,7 +96,7 @@ class FragmentBuilderBase { class HeaderValueExtractorImpl : public FragmentBuilderBase { public: - explicit HeaderValueExtractorImpl(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config); + explicit HeaderValueExtractorImpl(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder&& config); std::unique_ptr computeFragment(const Http::HeaderMap& headers) const override; @@ -111,19 +111,19 @@ class HeaderValueExtractorImpl : public FragmentBuilderBase { */ class ScopeKeyBuilderBase { public: - explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder config) : config_(std::move(config)) {} + explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder&& config) : config_(config) {} virtual ~ScopeKeyBuilderBase() = default; // Computes scope key for given headers, returns nullptr if a key can't be computed. virtual std::unique_ptr computeScopeKey(const Http::HeaderMap& headers) const PURE; protected: - ScopedRoutes::ScopeKeyBuilder config_; + const ScopedRoutes::ScopeKeyBuilder config_; }; class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { public: - explicit ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder config); + explicit ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config); std::unique_ptr computeScopeKey(const Http::HeaderMap& headers) const override; @@ -143,9 +143,9 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { public: ThreadLocalScopedConfigImpl( - envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) - : scope_key_builder_(std::move(scope_key_builder)) {} + : scope_key_builder_(scope_key_builder) {} void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); void removeRoutingScope(const std::string& scope_name); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 0ea1c4ed5a2d..d8ae81833c4c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -159,7 +159,7 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( MutableConfigProviderCommonBase::subscription_.get())), rds_config_source_(std::move(rds_config_source)) { initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(scope_key_builder); + return std::make_shared(std::move(scope_key_builder)); }); } diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 5ba2da84fd18..9c9a06a25934 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -51,9 +51,9 @@ TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; // Type not set, ASSERT only fails in debug mode. #if !defined(NDEBUG) - EXPECT_DEATH(HeaderValueExtractorImpl{config}, "header_value_extractor is not set."); + EXPECT_DEATH(HeaderValueExtractorImpl(std::move(config)), "header_value_extractor is not set."); #else - EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl(std::move(config)), ProtoValidationException, "HeaderValueExtractor extract_type not set.+"); #endif // !defined(NDEBUG) @@ -66,9 +66,8 @@ TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { )EOF"; TestUtility::loadFromYaml(yaml_plain, config); - EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, - "when element separator is set to an empty string, index should be set " - "to 0 in HeaderValueExtractor.+"); + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl(std::move(config)), ProtoValidationException, + "Index > 0 for empty string element separator."); // extract_type not set. yaml_plain = R"EOF( header_value_extractor: @@ -77,7 +76,7 @@ TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { )EOF"; TestUtility::loadFromYaml(yaml_plain, config); - EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl{config}, ProtoValidationException, + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl(std::move(config)), ProtoValidationException, "HeaderValueExtractor extract_type not set.+"); } @@ -91,7 +90,7 @@ TEST(HeaderValueExtractorImplTest, HeaderExtractionByIndex) { )EOF"; TestUtility::loadFromYaml(yaml_plain, config); - HeaderValueExtractorImpl extractor(config); + HeaderValueExtractorImpl extractor(std::move(config)); std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{{"foo_header", "part-0,part-1:value_bluh"}}); @@ -134,7 +133,7 @@ TEST(HeaderValueExtractorImplTest, HeaderExtractionByKey) { )EOF"; TestUtility::loadFromYaml(yaml_plain, config); - HeaderValueExtractorImpl extractor(config); + HeaderValueExtractorImpl extractor(std::move(config)); std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{ {"foo_header", "part-0;bar=>bluh;foo=>foo_value"}, }); @@ -194,7 +193,7 @@ TEST(HeaderValueExtractorImplTest, ElementSeparatorEmpty) { )EOF"; TestUtility::loadFromYaml(yaml_plain, config); - HeaderValueExtractorImpl extractor(config); + HeaderValueExtractorImpl extractor(std::move(config)); std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{ {"foo_header", "bar=b;c=d;e=f"}, }); @@ -265,7 +264,7 @@ TEST(ScopeKeyBuilderImplTest, Parse) { ScopedRoutes::ScopeKeyBuilder config; TestUtility::loadFromYaml(yaml_plain, config); - ScopeKeyBuilderImpl key_builder(config); + ScopeKeyBuilderImpl key_builder(std::move(config)); std::unique_ptr key = key_builder.computeScopeKey(TestHeaderMapImpl{ {"foo_header", "a=b,bar=bar_value,e=f"}, From 27ff0a3e1699a41c5c6c531b676bb6f603863277 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 25 Jun 2019 10:51:53 -0400 Subject: [PATCH 11/53] switch from constructing proto memeber by value(copy-elision) to more explicit constructing by moving Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.h | 9 ++++----- source/common/router/scoped_rds.cc | 4 +++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 1fea085986cb..808b442838c4 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -111,7 +111,8 @@ class HeaderValueExtractorImpl : public FragmentBuilderBase { */ class ScopeKeyBuilderBase { public: - explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder&& config) : config_(config) {} + explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder&& config) + : config_(std::move(config)) {} virtual ~ScopeKeyBuilderBase() = default; // Computes scope key for given headers, returns nullptr if a key can't be computed. @@ -142,10 +143,8 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { */ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { public: - ThreadLocalScopedConfigImpl( - envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder&& - scope_key_builder) - : scope_key_builder_(scope_key_builder) {} + ThreadLocalScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) + : scope_key_builder_(std::move(scope_key_builder)) {} void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); void removeRoutingScope(const std::string& scope_name); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index d8ae81833c4c..86e1d1dc88b8 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -159,7 +159,9 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( MutableConfigProviderCommonBase::subscription_.get())), rds_config_source_(std::move(rds_config_source)) { initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(std::move(scope_key_builder)); + return std::make_shared( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( + scope_key_builder)); }); } From 36a6e72ced467e3031bdfc16a55ad61e0a9dadbf Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 2 Jul 2019 15:36:21 -0400 Subject: [PATCH 12/53] refactor scoped config impl a little bit, andd SRDS config update impl Signed-off-by: Xin Zhuang --- source/common/router/BUILD | 15 +- source/common/router/scoped_config_impl.cc | 34 ++-- source/common/router/scoped_config_impl.h | 75 ++++++-- source/common/router/scoped_config_manager.cc | 22 --- source/common/router/scoped_config_manager.h | 49 ----- source/common/router/scoped_rds.cc | 106 ++++++---- source/common/router/scoped_rds.h | 72 +++++-- .../network/http_connection_manager/config.cc | 22 ++- test/common/router/BUILD | 2 + test/common/router/scoped_config_impl_test.cc | 182 +++++++++++++++++- test/common/router/scoped_rds_test.cc | 87 ++++++++- test/mocks/router/mocks.cc | 15 +- test/mocks/router/mocks.h | 35 +++- 13 files changed, 533 insertions(+), 183 deletions(-) delete mode 100644 source/common/router/scoped_config_manager.cc delete mode 100644 source/common/router/scoped_config_manager.h diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 445cc2232673..3a499d27fae0 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -146,22 +146,16 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "scoped_config_manager_lib", - srcs = ["scoped_config_manager.cc"], - hdrs = ["scoped_config_manager.h"], - deps = [ - "@envoy_api//envoy/api/v2:srds_cc", - ], -) - envoy_cc_library( name = "scoped_config_lib", srcs = ["scoped_config_impl.cc"], hdrs = ["scoped_config_impl.h"], + external_deps = [ + "abseil_str_format", + ], deps = [ ":config_lib", - ":scoped_config_manager_lib", + "//include/envoy/router:rds_interface", "//include/envoy/router:scopes_interface", "//include/envoy/thread_local:thread_local_interface", "@envoy_api//envoy/api/v2:srds_cc", @@ -176,6 +170,7 @@ envoy_cc_library( deps = [ ":scoped_config_lib", "//include/envoy/config:subscription_interface", + "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index ee3ce1318568..3ff2f754df1b 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -10,13 +10,7 @@ bool ScopeKey::operator==(const ScopeKey& other) const { // An empty key equals to nothing, "NULL" != "NULL". return false; } - return std::equal(fragments_.begin(), fragments_.end(), other.fragments_.begin(), - other.fragments_.end(), - [](const std::unique_ptr& left, - const std::unique_ptr& right) -> bool { - // Both should be non-NULL now. - return *left == *right; - }); + return this->hash() == other.hash(); } HeaderValueExtractorImpl::HeaderValueExtractorImpl( @@ -104,13 +98,31 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { return std::make_unique(std::move(key)); } -void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} +void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope( + const ScopedRouteInfoConstSharedPtr& scoped_route_info) { + scoped_route_info_by_name_.try_emplace(scoped_route_info->scopeName(), scoped_route_info); + scoped_route_info_by_key_.try_emplace(scoped_route_info->scopeKey().hash(), scoped_route_info); +} -void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} +void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { + const auto iter = scoped_route_info_by_name_.find(scope_name); + if (iter != scoped_route_info_by_name_.end()) { + scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); + scoped_route_info_by_name_.erase(iter); + } +} Router::ConfigConstSharedPtr -ThreadLocalScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { - return std::make_shared(); +ThreadLocalScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { + std::unique_ptr scope_key = scope_key_builder_.computeScopeKey(headers); + if (scope_key == nullptr) { + return nullptr; + } + auto iter = scoped_route_info_by_key_.find(scope_key->hash()); + if (iter != scoped_route_info_by_key_.end()) { + return iter->second->routeConfig(); + } + return nullptr; } } // namespace Router diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 808b442838c4..891439c6401a 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -4,13 +4,17 @@ #include "envoy/api/v2/srds.pb.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/router/rds.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" #include "envoy/thread_local/thread_local.h" +#include "common/common/hash.h" #include "common/protobuf/utility.h" #include "common/router/config_impl.h" -#include "common/router/scoped_config_manager.h" + +#include "absl/numeric/int128.h" +#include "absl/strings/str_format.h" namespace Envoy { namespace Router { @@ -26,15 +30,14 @@ class ScopeKeyFragmentBase { bool operator==(const ScopeKeyFragmentBase& other) const { if (typeid(*this) == typeid(other)) { - return equals(other); + return hash() == other.hash(); } return false; } virtual ~ScopeKeyFragmentBase() = default; -private: - // Returns true if the two fragments equal else false. - virtual bool equals(const ScopeKeyFragmentBase&) const PURE; + // Hash of the fragment. + virtual uint64_t hash() const PURE; }; /** @@ -52,14 +55,22 @@ class ScopeKey { // Caller should guarantee the fragment is not nullptr. void addFragment(std::unique_ptr&& fragment) { ASSERT(fragment != nullptr, "null fragment not allowed in ScopeKey."); + updateHash(*fragment); fragments_.emplace_back(std::move(fragment)); } + uint64_t hash() const { return hash_; } bool operator!=(const ScopeKey& other) const; - bool operator==(const ScopeKey& other) const; private: + // Update the key's hash with the new fragment hash. + void updateHash(const ScopeKeyFragmentBase& fragment) { + absl::uint128 buffer = absl::MakeUint128(hash_, fragment.hash()); + hash_ = HashUtil::xxHash64(absl::string_view(reinterpret_cast(&buffer), 16)); + } + + uint64_t hash_{0}; std::vector> fragments_; }; @@ -68,11 +79,9 @@ class StringKeyFragment : public ScopeKeyFragmentBase { public: explicit StringKeyFragment(absl::string_view value) : value_(value) {} -private: - bool equals(const ScopeKeyFragmentBase& other) const override { - return value_ == static_cast(other).value_; - } + uint64_t hash() const override { return HashUtil::xxHash64(value_); } +private: const std::string value_; }; @@ -132,9 +141,44 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { std::vector> fragment_builders_; }; +// ScopedRouteConfiguration and corresponding RouteConfigProvider. +class ScopedRouteInfo { +public: + ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, + std::shared_ptr&& route_provider) + : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { + ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); + ASSERT(route_provider_->config()->name() == config_proto_.route_configuration_name(), + absl::StrFormat( + "RouteConfigProvider's name '%s' doesn't match route_configuration_name '%s'.", + route_provider_->config()->name(), config_proto_.route_configuration_name())); + // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of + // Fragment. + for (const auto& fragment : config_proto_.key().fragments()) { + switch (fragment.type_case()) { + case envoy::api::v2::ScopedRouteConfiguration::Key::Fragment::kStringKey: + scope_key_.addFragment(std::make_unique(fragment.string_key())); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + } + } + Router::ConfigConstSharedPtr routeConfig() const { return route_provider_->config(); } + const ScopeKey& scopeKey() const { return scope_key_; } + const envoy::api::v2::ScopedRouteConfiguration& configProto() const { return config_proto_; } + const std::string& scopeName() const { return config_proto_.name(); } + +private: + const envoy::api::v2::ScopedRouteConfiguration config_proto_; + ScopeKey scope_key_; + std::shared_ptr route_provider_; +}; +using ScopedRouteInfoConstSharedPtr = std::shared_ptr; +// Ordered map for consistent config dumping. +using ScopedRouteMap = std::map; + /** - * TODO(AndresGuedez): implement scoped routing logic. - * * Each Envoy worker is assigned an instance of this type. When config updates are received, * addOrUpdateRoutingScope() and removeRoutingScope() are called to update the set of scoped routes. * @@ -153,8 +197,11 @@ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::Thr Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const override; private: - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder - scope_key_builder_; + ScopeKeyBuilderImpl scope_key_builder_; + // From scope name to cached ScopedRouteInfo. + absl::flat_hash_map scoped_route_info_by_name_; + // Hash by ScopeKey hash to lookup in constant time. + absl::flat_hash_map scoped_route_info_by_key_; }; /** diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc deleted file mode 100644 index 2a5b75f3b29c..000000000000 --- a/source/common/router/scoped_config_manager.cc +++ /dev/null @@ -1,22 +0,0 @@ -#include "common/router/scoped_config_manager.h" - -#include "envoy/common/exception.h" - -#include "common/common/fmt.h" - -namespace Envoy { -namespace Router { - -ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( - const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { - auto scoped_route_info = std::make_shared(config_proto); - scoped_route_map_[config_proto.name()] = scoped_route_info; - return scoped_route_info; -} - -bool ScopedConfigManager::removeRoutingScope(const std::string& name) { - return scoped_route_map_.erase(name) == 0; -} - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h deleted file mode 100644 index 5f8dd6fda878..000000000000 --- a/source/common/router/scoped_config_manager.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/api/v2/srds.pb.h" - -namespace Envoy { -namespace Router { - -// The internal representation of the configuration distributed via the ScopedRouteConfiguration -// proto. -class ScopedRouteInfo { -public: - ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) - : config_proto_(config_proto) {} - - // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. - - const envoy::api::v2::ScopedRouteConfiguration config_proto_; -}; -using ScopedRouteInfoConstSharedPtr = std::shared_ptr; - -// A manager for routing configuration scopes. -// An instance of the manager is owned by each ScopedRdsConfigSubscription. When config updates are -// received (on the main thread), the manager is called to track changes to the set of scoped route -// configurations and build s as needed. -class ScopedConfigManager { -public: - // Ordered map for consistent config dumping. - using ScopedRouteMap = std::map; - - // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the - // set of scopes matched against the scope keys built for each HTTP request. - ScopedRouteInfoConstSharedPtr - addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, - const std::string& version_info); - - // Removes a routing scope from the set of scopes matched against each HTTP request. - bool removeRoutingScope(const std::string& scope_name); - - const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } - -private: - ScopedRouteMap scoped_route_map_; -}; - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 86e1d1dc88b8..8ff394fddb49 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -17,9 +17,7 @@ using Envoy::Config::ConfigProviderPtr; namespace Envoy { namespace Router { - namespace ScopedRoutesConfigProviderUtil { - ConfigProviderPtr create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, @@ -77,14 +75,17 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + envoy::api::v2::core::ConfigSource rds_config_source, ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance( "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()), - name_(name), + factory_context_(factory_context), name_(name), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), - validation_visitor_(factory_context.messageValidationVisitor()) { + rds_config_source_(std::move(rds_config_source)), + validation_visitor_(factory_context.messageValidationVisitor()), stat_prefix_(stat_prefix), + srds_config_provider_manager_(config_provider_manager) { subscription_ = factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( scoped_rds.scoped_rds_config_source(), @@ -94,59 +95,88 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( } void ScopedRdsConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) { - std::vector scoped_routes; - for (const auto& resource_any : resources) { - scoped_routes.emplace_back(MessageUtil::anyConvert( - resource_any, validation_visitor_)); - } - std::unordered_set resource_names; - for (const auto& scoped_route : scoped_routes) { - if (!resource_names.insert(scoped_route.name()).second) { + // Dedup. + absl::flat_hash_set unique_resource_names; + for (const auto& resource : added_resources) { + // TODO(stevenzzzzz): we need to discuss about what to do when two ScopedRouteConfiguration + // which has the same ScopeKey. + if (!unique_resource_names.insert(resource.name()).second) { throw EnvoyException( - fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + fmt::format("duplicate scoped route configuration {} found", resource.name())); } } - for (const auto& scoped_route : scoped_routes) { - MessageUtil::validate(scoped_route); + + std::vector scoped_route_configs; + for (const auto& resource : added_resources) { + auto scoped_route_config = MessageUtil::anyConvert( + resource.resource(), validation_visitor_); + MessageUtil::validate(scoped_route_config); + scoped_route_configs.emplace_back(std::move(scoped_route_config)); } - // TODO(AndresGuedez): refactor such that it can be shared with other delta APIs (e.g., CDS). - std::vector exception_msgs; - // We need to keep track of which scoped routes we might need to remove. - ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = - scoped_config_manager_.scopedRouteMap(); - for (auto& scoped_route : scoped_routes) { - const std::string& scoped_route_name = scoped_route.name(); - scoped_routes_to_remove.erase(scoped_route_name); - ScopedRouteInfoConstSharedPtr scoped_route_info = - scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); + envoy::config::filter::network::http_connection_manager::v2::Rds rds; + rds.mutable_config_source()->MergeFrom(rds_config_source_); + for (auto& scoped_route_config : scoped_route_configs) { + ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_config.name()); + rds.set_route_config_name(scoped_route_config.route_configuration_name()); + ScopedRouteInfoConstSharedPtr scoped_route_info = std::make_shared( + std::move(scoped_route_config), + srds_config_provider_manager_.getRouteConfigProvider(factory_context_, rds, stat_prefix_)); + scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; applyDeltaConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { auto* thread_local_scoped_config = const_cast( static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); }); } - - for (const auto& scoped_route : scoped_routes_to_remove) { - const std::string scoped_route_name = scoped_route.first; + for (const auto& scoped_route_name : removed_resources) { ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - scoped_config_manager_.removeRoutingScope(scoped_route_name); + scoped_route_map_.erase(scoped_route_name); applyDeltaConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { auto* thread_local_scoped_config = const_cast( static_cast(config.get())); thread_local_scoped_config->removeRoutingScope(scoped_route_name); }); } - ConfigSubscriptionCommonBase::onConfigUpdate(); setLastConfigInfo(absl::optional({absl::nullopt, version_info})); stats_.config_reload_.inc(); } +void ScopedRdsConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + std::vector scoped_routes; + for (const auto& resource_any : resources) { + auto scoped_route = MessageUtil::anyConvert( + resource_any, validation_visitor_); + MessageUtil::validate(scoped_route); + scoped_routes.emplace_back(std::move(scoped_route)); + } + ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; + Protobuf::RepeatedPtrField to_add_repeated; + Protobuf::RepeatedPtrField to_remove_repeated; + + for (auto& scoped_route : scoped_routes) { + const std::string& scoped_route_name = scoped_route.name(); + scoped_routes_to_remove.erase(scoped_route_name); + auto* to_add = to_add_repeated.Add(); + to_add->set_name(scoped_route.name()); + to_add->set_version(version_info); + to_add->mutable_resource()->PackFrom(scoped_route); + } + + for (const auto& scoped_route : scoped_routes_to_remove) { + *to_remove_repeated.Add() = scoped_route.first; + } + onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); +} + ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription, Server::Configuration::FactoryContext& factory_context, @@ -155,8 +185,6 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopeKeyBuilder& scope_key_builder) : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, ConfigProvider::ApiType::Delta), - subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())), rds_config_source_(std::move(rds_config_source)) { initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared( @@ -165,6 +193,9 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( }); } +Envoy::Config::ConfigSharedPtr ScopedRdsConfigProvider::getConfig() { + return std::dynamic_pointer_cast(tls_->get()); +} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); for (const auto& element : configSubscriptions()) { @@ -177,10 +208,9 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const const ScopedRdsConfigSubscription* typed_subscription = static_cast(subscription.get()); dynamic_config->set_name(typed_subscription->name()); - const ScopedConfigManager::ScopedRouteMap& scoped_route_map = - typed_subscription->scopedRouteMap(); + const ScopedRouteMap& scoped_route_map = typed_subscription->scopedRouteMap(); for (const auto& it : scoped_route_map) { - dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->config_proto_); + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->configProto()); } TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), *dynamic_config->mutable_last_updated()); @@ -222,6 +252,8 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( static_cast(optarg) .scoped_routes_name_, factory_context, stat_prefix, + static_cast(optarg) + .rds_config_source_, static_cast(config_provider_manager)); }); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index f0cc72f71c15..61a0226b8f47 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -4,6 +4,8 @@ #include "envoy/api/v2/srds.pb.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/router/route_config_provider_manager.h" #include "envoy/stats/scope.h" #include "common/config/config_provider_impl.h" @@ -87,15 +89,14 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + envoy::api::v2::core::ConfigSource rds_config_source, ScopedRoutesConfigProviderManager& config_provider_manager); ~ScopedRdsConfigSubscription() override = default; const std::string& name() const { return name_; } - const ScopedConfigManager::ScopedRouteMap& scopedRouteMap() const { - return scoped_config_manager_.scopedRouteMap(); - } + const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: // Envoy::Config::ConfigSubscriptionCommonBase @@ -104,10 +105,9 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override; - void onConfigUpdate(const Protobuf::RepeatedPtrField&, - const Protobuf::RepeatedPtrField&, const std::string&) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info) override; void onConfigUpdateFailed(const EnvoyException*) override { ConfigSubscriptionCommonBase::onConfigUpdateFailed(); } @@ -117,12 +117,17 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio .name(); } + ScopedRouteMap scoped_route_map_; + // For creating RDS subscriptions. + Server::Configuration::FactoryContext& factory_context_; const std::string name_; std::unique_ptr subscription_; Stats::ScopePtr scope_; ScopedRdsStats stats_; - ScopedConfigManager scoped_config_manager_; + const envoy::api::v2::core::ConfigSource rds_config_source_; ProtobufMessage::ValidationVisitor& validation_visitor_; + const std::string stat_prefix_; + ScopedRoutesConfigProviderManager& srds_config_provider_manager_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; @@ -137,19 +142,18 @@ class ScopedRdsConfigProvider : public Envoy::Config::DeltaMutableConfigProvider const envoy::config::filter::network::http_connection_manager::v2:: ScopedRoutes::ScopeKeyBuilder& scope_key_builder); - ScopedRdsConfigSubscription& subscription() { return *subscription_; } + ScopedRdsConfigSubscription& subscription() { + return *static_cast(subscription_.get()); + } // getConfig() is overloaded (const/non-const only). Make all base getConfig()s visible to avoid // compiler warnings. using DeltaMutableConfigProviderBase::getConfig; // Envoy::Config::DeltaMutableConfigProviderBase - Envoy::Config::ConfigSharedPtr getConfig() override { - return std::dynamic_pointer_cast(tls_->get()); - } + Envoy::Config::ConfigSharedPtr getConfig() override; private: - ScopedRdsConfigSubscription* subscription_; const envoy::api::v2::core::ConfigSource rds_config_source_; }; @@ -157,8 +161,10 @@ class ScopedRdsConfigProvider : public Envoy::Config::DeltaMutableConfigProvider // (xds) config providers. class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderManagerImplBase { public: - ScopedRoutesConfigProviderManager(Server::Admin& admin) - : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes") {} + ScopedRoutesConfigProviderManager( + Server::Admin& admin, Router::RouteConfigProviderManager& route_config_provider_manager) + : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes"), + route_config_provider_manager_(route_config_provider_manager) {} ~ScopedRoutesConfigProviderManager() override = default; @@ -183,6 +189,42 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa std::vector>&& config_protos, Server::Configuration::FactoryContext& factory_context, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; + + RouteConfigProviderManager& route_config_provider_manager() { + return route_config_provider_manager_; + } + + std::shared_ptr + getRouteConfigProvider(Server::Configuration::FactoryContext& factory_context, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + std::string stat_name) { + // envoy::config::filter::network::http_connection_manager::v2::Rds rds; + // rds.set_route_config_name(std::move(route_config_name)); + // rds.mutable_config_source()->MergeFrom(rds_config_source); + // Unique ID for a (rds_config_source, route_config_name). + const uint64_t route_provider_id = MessageUtil::hash(rds); + auto iter = cached_route_providers_.find(route_provider_id); + if (iter != cached_route_providers_.end()) { + return iter->second; + } + std::shared_ptr provider = + route_config_provider_manager_.createRdsRouteConfigProvider(rds, factory_context, + stat_name); + cached_route_providers_.try_emplace(route_provider_id, provider); + return provider; + } + void + deleteRouteConfigSource(envoy::config::filter::network::http_connection_manager::v2::Rds& rds) { + const uint64_t route_provider_id = MessageUtil::hash(rds); + if (!cached_route_providers_.erase(route_provider_id)) { + ENVOY_LOG_MISC(warn, "Try to delete non-existing route config source {}", rds.DebugString()); + } + } + +private: + RouteConfigProviderManager& route_config_provider_manager_; + // From ConfigSource fingerprint to shared RouteConfigProvider. + absl::flat_hash_map> cached_route_providers_; }; // The optional argument passed to the ConfigProviderManager::create*() functions. diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 04edfdba79c4..0068101b9260 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -7,6 +7,7 @@ #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.validate.h" #include "envoy/filesystem/filesystem.h" +#include "envoy/registry/registry.h" #include "envoy/server/admin.h" #include "common/access_log/access_log_impl.h" @@ -89,10 +90,12 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( return std::make_shared(context.admin()); }); - std::shared_ptr scoped_routes_config_provider_manager = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), [&context] { - return std::make_shared(context.admin()); + std::shared_ptr scoped_routes_config_provider_manager = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), + [&context, route_config_provider_manager] { + return std::make_shared( + context.admin(), *route_config_provider_manager); }); std::shared_ptr filter_config(new HttpConnectionManagerConfig( @@ -100,10 +103,10 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( *scoped_routes_config_provider_manager)); // This lambda captures the shared_ptrs created above, thus preserving the - // reference count. Moreover, keep in mind the capture list determines - // destruction order. - return [route_config_provider_manager, scoped_routes_config_provider_manager, filter_config, - &context, date_provider](Network::FilterManager& filter_manager) -> void { + // reference count. + // Keep in mind the lambda capture list **doesn't** determine the destruction order, the + return [scoped_routes_config_provider_manager, route_config_provider_manager, date_provider, + filter_config, &context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl( *filter_config, context.drainDecision(), context.random(), context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), @@ -171,6 +174,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( 0 #endif ))) { + // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will // be managed by the scoped routing logic instead. switch (config.route_specifier_case()) { @@ -182,8 +186,6 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( break; case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kScopedRoutes: - ENVOY_LOG(warn, "Scoped routing has been enabled but it is not yet fully implemented! HTTP " - "request routing DOES NOT work (yet) with this configuration."); scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::create( config, context_, stats_prefix_, scoped_routes_config_provider_manager_); break; diff --git a/test/common/router/BUILD b/test/common/router/BUILD index eb07c6558972..c47295ee6ae4 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -85,6 +85,7 @@ envoy_cc_test( ], deps = [ "//source/common/router:scoped_config_lib", + "//test/mocks/router:router_mocks", "//test/test_common:utility_lib", ], ) @@ -102,6 +103,7 @@ envoy_cc_test( "//source/common/router:scoped_rds_lib", "//source/server/http:admin_lib", "//test/mocks/init:init_mocks", + "//test/mocks/router:router_mocks", "//test/mocks/server:server_mocks", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 9c9a06a25934..ce02e3baacf3 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -2,6 +2,7 @@ #include "common/router/scoped_config_impl.h" +#include "test/mocks/router/mocks.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -11,10 +12,12 @@ namespace Router { namespace { using ::Envoy::Http::TestHeaderMapImpl; +using ::testing::NiceMock; +using ::testing::Return; class FooFragment : public ScopeKeyFragmentBase { -private: - bool equals(const ScopeKeyFragmentBase&) const override { return true; }; +public: + uint64_t hash() const override { return 1; } }; TEST(ScopeKeyFragmentBaseTest, EqualSign) { @@ -24,14 +27,33 @@ TEST(ScopeKeyFragmentBaseTest, EqualSign) { EXPECT_NE(foo, bar); } +TEST(ScopeKeyFragmentBaseTest, HashStable) { + FooFragment foo1; + FooFragment foo2; + + // Two FooFragments equal because their hash equals. + EXPECT_EQ(foo1, foo2); + EXPECT_EQ(foo1.hash(), foo2.hash()); + + // Hash value doesn't change. + StringKeyFragment a("abcdefg"); + auto hash_value = a.hash(); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(hash_value, a.hash()); + EXPECT_EQ(StringKeyFragment("abcdefg").hash(), hash_value); + } +} + TEST(StringKeyFragmentTest, Empty) { StringKeyFragment a(""); StringKeyFragment b(""); EXPECT_EQ(a, b); + EXPECT_EQ(a.hash(), b.hash()); StringKeyFragment non_empty("ABC"); EXPECT_NE(a, non_empty); + EXPECT_NE(a.hash(), non_empty.hash()); } TEST(StringKeyFragmentTest, Normal) { @@ -317,6 +339,162 @@ TEST(ScopeKeyBuilderImplTest, Parse) { EXPECT_EQ(key, nullptr); } +class ScopedRouteInfoTest : public testing::Test { +public: + void SetUp() override { + std::string yaml_plain = R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: foo + - string_key: bar +)EOF"; + TestUtility::loadFromYaml(yaml_plain, scoped_route_config_); + + route_config_ = std::make_shared>(); + route_config_->name_ = "foo_route"; + + route_config_provider_ = std::make_shared(); + EXPECT_CALL(*route_config_provider_, config()).WillRepeatedly(Return(route_config_)); + } + + envoy::api::v2::ScopedRouteConfiguration scoped_route_config_; + std::shared_ptr route_config_; + std::unique_ptr info_; + std::shared_ptr route_config_provider_; +}; + +TEST_F(ScopedRouteInfoTest, Creation) { + envoy::api::v2::ScopedRouteConfiguration config_copy = scoped_route_config_; + info_ = + std::make_unique(std::move(scoped_route_config_), route_config_provider_); + EXPECT_EQ(info_->routeConfig().get(), route_config_.get()); + EXPECT_TRUE(TestUtility::protoEqual(info_->configProto(), config_copy)); + EXPECT_EQ(info_->scopeName(), "foo_scope"); + EXPECT_EQ(info_->scopeKey(), makeKey({"foo", "bar"})); +} + +class ScopedRouteInfoDeathTest : public ScopedRouteInfoTest {}; + +TEST_F(ScopedRouteInfoDeathTest, AssertFailure) { + EXPECT_DEBUG_DEATH(ScopedRouteInfo(std::move(scoped_route_config_), nullptr), + "ScopedRouteInfo expects a valid RouteConfigProvider."); + + route_config_->name_ = "bar_route"; + EXPECT_DEBUG_DEATH( + ScopedRouteInfo(std::move(scoped_route_config_), route_config_provider_), + "RouteConfigProvider's name 'bar_route' doesn't match route_configuration_name 'foo_route'."); +} + +class ThreadLocalScopedConfigImplTest : public testing::Test { +public: + void SetUp() override { + std::string yaml_plain = R"EOF( + fragments: + - header_value_extractor: + name: 'foo_header' + element_separator: ',' + element: + key: 'bar' + separator: '=' + - header_value_extractor: + name: 'bar_header' + element_separator: ';' + index: 2 +)EOF"; + TestUtility::loadFromYaml(yaml_plain, key_builder_config_); + + scope_info_a_ = makeScopedRouteInfo(R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: foo + - string_key: bar +)EOF"); + scope_info_b_ = makeScopedRouteInfo(R"EOF( + name: bar_scope + route_configuration_name: bar_route + key: + fragments: + - string_key: bar + - string_key: baz +)EOF"); + } + std::shared_ptr makeScopedRouteInfo(const std::string& route_config_yaml) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + TestUtility::loadFromYaml(route_config_yaml, scoped_route_config); + + std::shared_ptr route_config = std::make_shared>(); + route_config->name_ = scoped_route_config.route_configuration_name(); + + std::shared_ptr route_config_provider = + std::make_shared(); + EXPECT_CALL(*route_config_provider, config()).WillRepeatedly(Return(route_config)); + return std::make_shared(std::move(scoped_route_config), route_config_provider); + } + + std::shared_ptr scope_info_a_; + std::shared_ptr scope_info_b_; + ScopedRoutes::ScopeKeyBuilder key_builder_config_; + std::unique_ptr scoped_config_impl_; +}; + +TEST_F(ThreadLocalScopedConfigImplTest, PickRoute) { + scoped_config_impl_ = + std::make_unique(std::move(key_builder_config_)); + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); + + // Key (foo, bar) maps to scope_info_a_. + ConfigConstSharedPtr route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + }); + EXPECT_EQ(route_config, scope_info_a_->routeConfig()); + + // Key (bar, baz) maps to scope_info_b_. + route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=bar,"}, + {"bar_header", ";val1;baz;val3"}, + }); + EXPECT_EQ(route_config, scope_info_b_->routeConfig()); + + // No such key (bar, NOT_BAZ). + route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",key=value,bar=bar,"}, + {"bar_header", ";val1;NOT_BAZ;val3"}, + }); + EXPECT_EQ(route_config, nullptr); +} + +TEST_F(ThreadLocalScopedConfigImplTest, Update) { + scoped_config_impl_ = + std::make_unique(std::move(key_builder_config_)); + + TestHeaderMapImpl headers{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + }; + // Empty ScopeConfig. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Add scope_key (bar, baz). + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Add scope_key (foo, bar). + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); + // Found scope_info_a_. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), scope_info_a_->routeConfig()); + + // Remvoe scope "foo_scope". + scoped_config_impl_->removeRoutingScope("foo_scope"); + // scope_info_a_ is gone. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); +} + } // namespace } // namespace Router } // namespace Envoy diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 1476dac9dce0..6280e13ccaf0 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -6,16 +6,22 @@ #include "common/router/scoped_rds.h" +#include "test/mocks/router/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::ByMove; +using testing::DoAll; using testing::InSequence; +using testing::Invoke; using testing::Return; +using testing::ReturnRefOfCopy; namespace Envoy { namespace Router { @@ -45,18 +51,54 @@ class ScopedRoutesTestBase : public testing::Test { protected: ScopedRoutesTestBase() { EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); - config_provider_manager_ = - std::make_unique(factory_context_.admin_); + config_provider_manager_ = std::make_unique( + factory_context_.admin_, route_config_provider_manager_); + EXPECT_CALL(route_config_provider_manager_, createRdsRouteConfigProvider(_, _, _)) + .WillRepeatedly(Invoke( + [this](const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Server::Configuration::FactoryContext&, + const std::string&) -> RouteConfigProviderPtr { + auto iter = cached_route_configs_.find(rds.route_config_name()); + if (iter == cached_route_configs_.end()) { + cached_route_configs_[rds.route_config_name()] = std::make_shared(); + EXPECT_CALL(*cached_route_configs_[rds.route_config_name()], name()) + .WillRepeatedly(ReturnRefOfCopy(rds.route_config_name())); + } + auto provider = std::make_unique>(); + EXPECT_CALL(*provider, config()) + .WillRepeatedly(Invoke([this, rds]() -> ConfigConstSharedPtr { + std::cerr << "DDD returns config() for :" << rds.route_config_name() + << std::endl; + return cached_route_configs_[rds.route_config_name()]; + })); + return provider; + })); } ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } + // The delta style API helper. + Protobuf::RepeatedPtrField + anyToResource(Protobuf::RepeatedPtrField& resources, std::string version) { + Protobuf::RepeatedPtrField added_resources; + for (const auto& resource_any : resources) { + auto config = TestUtility::anyConvert(resource_any); + auto* to_add = added_resources.Add(); + to_add->set_name(config.name()); + to_add->set_version(version); + to_add->mutable_resource()->PackFrom(config); + } + return added_resources; + } + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } NiceMock factory_context_; Upstream::ClusterManager::ClusterInfoMap cluster_map_; Upstream::MockClusterMockPrioritySet cluster_; std::unique_ptr config_provider_manager_; + MockRouteConfigProviderManager route_config_provider_manager_; + absl::flat_hash_map> cached_route_configs_; Event::SimulatedTimeSystem time_system_; envoy::api::v2::core::ConfigSource rds_config_source_; }; @@ -70,7 +112,11 @@ class ScopedRdsTest : public ScopedRoutesTestBase { name: foo_scoped_routes scope_key_builder: fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: X-Google-VIP + element: + key: x-foo-key + separator: ; )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; TestUtility::loadFromYaml(config_yaml, scoped_routes_config); @@ -101,6 +147,9 @@ route_configuration_name: foo_routes parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources, "1"), ProtoValidationException); + EXPECT_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), + ProtoValidationException); + // 'route_configuration_name' validation: value must be > 1 byte. const std::string config_yaml2 = R"EOF( name: foo_scope @@ -112,6 +161,8 @@ name: foo_scope Protobuf::RepeatedPtrField resources2; parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources2, "1"), ProtoValidationException); + EXPECT_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), + ProtoValidationException); // 'key' validation: must define at least 1 fragment. const std::string config_yaml3 = R"EOF( @@ -122,6 +173,8 @@ route_configuration_name: foo_routes Protobuf::RepeatedPtrField resources3; parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources3, "1"), ProtoValidationException); + EXPECT_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), + ProtoValidationException); } // Tests that multiple uniquely named resources are allowed in config updates. @@ -149,6 +202,12 @@ route_configuration_name: foo_routes EXPECT_EQ( 1UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + + // Delta API. + EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources, "2"), {}, "2")); + EXPECT_EQ( + 2UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); } // Tests that only one resource is provided during a config update. @@ -167,6 +226,10 @@ route_configuration_name: foo_routes parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW_WITH_MESSAGE(subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, "duplicate scoped route configuration foo_scope found"); + + EXPECT_THROW_WITH_MESSAGE( + subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), + EnvoyException, "duplicate scoped route configuration foo_scope found"); } // Tests a config update failure. @@ -185,7 +248,8 @@ TEST_F(ScopedRdsTest, ConfigUpdateFailure) { using ScopedRoutesConfigProviderManagerTest = ScopedRoutesTestBase; -// Tests that the /config_dump handler returns the corresponding scoped routing config. +// Tests that the /config_dump handler returns the corresponding scoped routing +// config. TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); @@ -200,7 +264,7 @@ TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { dynamic_scoped_route_configs: )EOF", expected_config_dump); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump)); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); @@ -214,7 +278,9 @@ stat_prefix: foo name: $0 scope_key_builder: fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: X-Google-VIP + index: 0 $1 )EOF"; const std::string inline_scoped_route_configs_yaml = R"EOF( @@ -256,7 +322,7 @@ stat_prefix: foo dynamic_scoped_route_configs: )EOF", expected_config_dump); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump2)); const std::string scoped_rds_config_yaml = R"EOF( scoped_rds: @@ -311,7 +377,7 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump3 = MessageUtil::downcastAndValidate( *message_ptr); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump3)); resources.Clear(); factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, @@ -343,14 +409,15 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump4 = MessageUtil::downcastAndValidate( *message_ptr); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump4.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderManagerTest; // Tests that SRDS only allows creation of delta static config providers. TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { - // Use match all regex due to lack of distinctive matchable output for coverage test. + // Use match all regex due to lack of distinctive matchable output for + // coverage test. EXPECT_DEATH(config_provider_manager_->createStaticConfigProvider( parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index f99496356e95..151f867cf493 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -116,11 +116,22 @@ MockRoute::MockRoute() { } MockRoute::~MockRoute() {} +MockRouteConfigProvider::MockRouteConfigProvider() { + ON_CALL(*this, config()).WillByDefault(Return(route_config_)); +} + MockRouteConfigProviderManager::MockRouteConfigProviderManager() {} MockRouteConfigProviderManager::~MockRouteConfigProviderManager() {} -MockScopedConfig::MockScopedConfig() {} -MockScopedConfig::~MockScopedConfig() {} +MockScopedConfig::MockScopedConfig() { + ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); +} + +MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() + : config_(std::make_shared()) { + ON_CALL(*this, getConfig()).WillByDefault(Return(config_)); + ON_CALL(*this, apiType()).WillByDefault(Return(ApiType::Delta)); +} } // namespace Router } // namespace Envoy diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index f84aacda8ab8..54f9cbe5a89a 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -8,6 +8,8 @@ #include #include +#include "envoy/common/time.h" +#include "envoy/config/config_provider.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/dispatcher.h" #include "envoy/json/json_object.h" @@ -30,6 +32,7 @@ namespace Envoy { namespace Router { +using ::testing::NiceMock; class MockDirectResponseEntry : public DirectResponseEntry { public: @@ -379,6 +382,19 @@ class MockConfig : public Config { std::string name_{"fake_config"}; }; +class MockRouteConfigProvider : public RouteConfigProvider { +public: + MockRouteConfigProvider(); + ~MockRouteConfigProvider() = default; + + MOCK_METHOD0(config, ConfigConstSharedPtr()); + MOCK_CONST_METHOD0(configInfo, absl::optional()); + MOCK_CONST_METHOD0(lastUpdated, SystemTime()); + MOCK_METHOD0(onConfigUpdate, void()); + + std::shared_ptr> route_config_{new NiceMock()}; +}; + class MockRouteConfigProviderManager : public RouteConfigProviderManager { public: MockRouteConfigProviderManager(); @@ -397,9 +413,26 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { class MockScopedConfig : public ScopedConfig { public: MockScopedConfig(); - ~MockScopedConfig(); + ~MockScopedConfig() = default; MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); + + std::shared_ptr route_config_{new NiceMock()}; +}; + +class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { +public: + MockScopedRouteConfigProvider(); + ~MockScopedRouteConfigProvider() = default; + + // Config::ConfigProvider + MOCK_CONST_METHOD0(lastUpdated, SystemTime()); + MOCK_CONST_METHOD0(getConfigProto, Protobuf::Message*()); + MOCK_CONST_METHOD0(getConfigProtos, Envoy::Config::ConfigProvider::ConfigProtoVector()); + MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); + MOCK_CONST_METHOD0(apiType, ApiType()); + + std::shared_ptr config_; }; } // namespace Router From 74038f4614925679510abcc80d6b8ce45324fa93 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 2 Jul 2019 16:43:43 -0400 Subject: [PATCH 13/53] fix typo Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index ce02e3baacf3..771c93aeb19b 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -489,7 +489,7 @@ TEST_F(ThreadLocalScopedConfigImplTest, Update) { // Found scope_info_a_. EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), scope_info_a_->routeConfig()); - // Remvoe scope "foo_scope". + // Remove scope "foo_scope". scoped_config_impl_->removeRoutingScope("foo_scope"); // scope_info_a_ is gone. EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); From ff7ccdb24ab7bd4c0bce2916ce2ab5252ef0bb69 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 8 Jul 2019 09:10:06 -0400 Subject: [PATCH 14/53] fix srds integration test and add some more test on scopekey Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.h | 3 +- source/common/router/scoped_rds.h | 4 -- test/common/router/scoped_config_impl_test.cc | 11 ++++- test/common/router/scoped_rds_test.cc | 2 - .../scoped_rds_integration_test.cc | 41 +++++++++++++++---- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 891439c6401a..242c48123bae 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -148,7 +148,8 @@ class ScopedRouteInfo { std::shared_ptr&& route_provider) : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); - ASSERT(route_provider_->config()->name() == config_proto_.route_configuration_name(), + ASSERT(!route_provider_->configInfo().has_value() || + route_provider_->config()->name() == config_proto_.route_configuration_name(), absl::StrFormat( "RouteConfigProvider's name '%s' doesn't match route_configuration_name '%s'.", route_provider_->config()->name(), config_proto_.route_configuration_name())); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 61a0226b8f47..ac7ba74b3d34 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -198,10 +198,6 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa getRouteConfigProvider(Server::Configuration::FactoryContext& factory_context, envoy::config::filter::network::http_connection_manager::v2::Rds& rds, std::string stat_name) { - // envoy::config::filter::network::http_connection_manager::v2::Rds rds; - // rds.set_route_config_name(std::move(route_config_name)); - // rds.mutable_config_source()->MergeFrom(rds_config_source); - // Unique ID for a (rds_config_source, route_config_name). const uint64_t route_provider_id = MessageUtil::hash(rds); auto iter = cached_route_providers_.find(route_provider_id); if (iter != cached_route_providers_.end()) { diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 771c93aeb19b..b4bfe4f434a3 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -253,6 +253,10 @@ TEST(ScopeKeyTest, Unmatches) { EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + // Order matters. + EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + EXPECT_NE(makeKey({"a", "c", "b"}), makeKey({"a", "b", "c"})); + // Two keys of different length won't match. EXPECT_NE(makeKey({"a", "b"}), makeKey({"a", "b", "c"})); @@ -265,7 +269,7 @@ TEST(ScopeKeyTest, Matches) { EXPECT_EQ(makeKey({"", ""}), makeKey({"", ""})); EXPECT_EQ(makeKey({"a", "", ""}), makeKey({"a", "", ""})); - // Non empty fragments comparison. + // Non empty fragments comparison. EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); } @@ -357,8 +361,11 @@ class ScopedRouteInfoTest : public testing::Test { route_config_provider_ = std::make_shared(); EXPECT_CALL(*route_config_provider_, config()).WillRepeatedly(Return(route_config_)); + EXPECT_CALL(*route_config_provider_, configInfo()) + .WillRepeatedly(Return(RouteConfigProvider::ConfigInfo{route_configuration_, ""})); } + envoy::api::v2::RouteConfiguration route_configuration_; envoy::api::v2::ScopedRouteConfiguration scoped_route_config_; std::shared_ptr route_config_; std::unique_ptr info_; @@ -430,7 +437,7 @@ class ThreadLocalScopedConfigImplTest : public testing::Test { route_config->name_ = scoped_route_config.route_configuration_name(); std::shared_ptr route_config_provider = - std::make_shared(); + std::make_shared>(); EXPECT_CALL(*route_config_provider, config()).WillRepeatedly(Return(route_config)); return std::make_shared(std::move(scoped_route_config), route_config_provider); } diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 6280e13ccaf0..9bd1f185fc42 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -67,8 +67,6 @@ class ScopedRoutesTestBase : public testing::Test { auto provider = std::make_unique>(); EXPECT_CALL(*provider, config()) .WillRepeatedly(Invoke([this, rds]() -> ConfigConstSharedPtr { - std::cerr << "DDD returns config() for :" << rds.route_config_name() - << std::endl; return cached_route_configs_[rds.route_config_name()]; })); return provider; diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 32e916da6d9a..13f63fb1e42a 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -48,7 +48,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, http_connection_manager) { const std::string& scope_key_builder_config_yaml = R"EOF( fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: X-Google-VIP + element: + key: x-foo-key + separator: ; )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder; @@ -124,6 +128,15 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); } + void sendRdsResponse(const std::string& route_config, const std::string& version) { + envoy::api::v2::DiscoveryResponse response; + response.set_version_info(version); + response.set_type_url(Config::TypeUrl::get().RouteConfiguration); + response.add_resources()->PackFrom( + TestUtility::parseYaml(route_config)); + rds_upstream_info_.stream_->sendGrpcMessage(response); + } + void sendScopedRdsResponse(const std::vector& resource_protos, const std::string& version) { ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); @@ -159,19 +172,31 @@ route_configuration_name: foo_route1 )EOF"; const std::string scope_route2 = R"EOF( name: foo_scope2 -route_configuration_name: foo_route2 +route_configuration_name: foo_route1 key: fragments: - string_key: x-foo-key )EOF"; - on_server_init_function_ = [this, &scope_route1, &scope_route2]() { + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} +)EOF"; + + on_server_init_function_ = [&]() { createScopedRdsStream(); sendScopedRdsResponse({scope_route1, scope_route2}, "1"); + createRdsStream(); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_1"), "1"); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_2"), "2"); }; initialize(); - - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", @@ -179,14 +204,14 @@ route_configuration_name: foo_route2 const std::string scope_route3 = R"EOF( name: foo_scope3 -route_configuration_name: foo_route3 +route_configuration_name: foo_route1 key: fragments: - string_key: x-baz-key )EOF"; sendScopedRdsResponse({scope_route3}, "2"); - - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 3); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_3"), "3"); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 6927017134761466251UL); From cf387204d502a379e06bb57c21ec560d94bf7990 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 9 Jul 2019 15:59:47 -0400 Subject: [PATCH 15/53] save review fixes Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 1 + source/common/router/scoped_config_impl.h | 16 +-- source/common/router/scoped_rds.cc | 102 ++++++++++-------- source/common/router/scoped_rds.h | 29 ++--- .../network/http_connection_manager/config.cc | 4 +- test/common/router/scoped_rds_test.cc | 27 +++-- test/mocks/router/mocks.h | 4 +- 7 files changed, 97 insertions(+), 86 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 3ff2f754df1b..d324db781198 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -107,6 +107,7 @@ void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope( void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { const auto iter = scoped_route_info_by_name_.find(scope_name); if (iter != scoped_route_info_by_name_.end()) { + ASSERT(scoped_route_info_by_key_.count(iter->second->scopeKey().hash()) == 1); scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); scoped_route_info_by_name_.erase(iter); } diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 242c48123bae..b569ed3b4f06 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -77,12 +77,14 @@ class ScopeKey { // String fragment. class StringKeyFragment : public ScopeKeyFragmentBase { public: - explicit StringKeyFragment(absl::string_view value) : value_(value) {} + explicit StringKeyFragment(absl::string_view value) + : value_(value), hash_(HashUtil::xxHash64(value_)) {} - uint64_t hash() const override { return HashUtil::xxHash64(value_); } + uint64_t hash() const override { return hash_; } private: const std::string value_; + const uint64_t hash_; }; /** @@ -148,11 +150,11 @@ class ScopedRouteInfo { std::shared_ptr&& route_provider) : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); - ASSERT(!route_provider_->configInfo().has_value() || - route_provider_->config()->name() == config_proto_.route_configuration_name(), - absl::StrFormat( - "RouteConfigProvider's name '%s' doesn't match route_configuration_name '%s'.", - route_provider_->config()->name(), config_proto_.route_configuration_name())); + ASSERT( + !route_provider_->configInfo().has_value() || + route_provider_->config()->name() == config_proto_.route_configuration_name(), + fmt::format("RouteConfigProvider's name '{}' doesn't match route_configuration_name '{}'.", + route_provider_->config()->name(), config_proto_.route_configuration_name())); // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of // Fragment. for (const auto& fragment : config_proto_.key().fragments()) { diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 8ff394fddb49..51762b90effd 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -7,6 +7,7 @@ #include "common/common/assert.h" #include "common/common/logger.h" +#include "common/common/utility.h" // Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all // ConfigProvider related types for consistency. @@ -99,76 +100,89 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) { - // Dedup. + // TODO(stevenzzzzz): we need to discuss about what to do when two ScopedRouteConfiguration + // which has the same ScopeKey. + bool any_applied = false; + std::vector exception_msgs; absl::flat_hash_set unique_resource_names; - for (const auto& resource : added_resources) { - // TODO(stevenzzzzz): we need to discuss about what to do when two ScopedRouteConfiguration - // which has the same ScopeKey. - if (!unique_resource_names.insert(resource.name()).second) { - throw EnvoyException( - fmt::format("duplicate scoped route configuration {} found", resource.name())); - } - } - - std::vector scoped_route_configs; - for (const auto& resource : added_resources) { - auto scoped_route_config = MessageUtil::anyConvert( - resource.resource(), validation_visitor_); - MessageUtil::validate(scoped_route_config); - scoped_route_configs.emplace_back(std::move(scoped_route_config)); - } - envoy::config::filter::network::http_connection_manager::v2::Rds rds; rds.mutable_config_source()->MergeFrom(rds_config_source_); - for (auto& scoped_route_config : scoped_route_configs) { - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_config.name()); - rds.set_route_config_name(scoped_route_config.route_configuration_name()); - ScopedRouteInfoConstSharedPtr scoped_route_info = std::make_shared( - std::move(scoped_route_config), - srds_config_provider_manager_.getRouteConfigProvider(factory_context_, rds, stat_prefix_)); - scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; - applyDeltaConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + for (const auto& resource : added_resources) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + try { + scoped_route_config = MessageUtil::anyConvert( + resource.resource(), validation_visitor_); + MessageUtil::validate(scoped_route_config); + if (!unique_resource_names.insert(scoped_route_config.name()).second) { + throw EnvoyException(fmt::format("duplicate scoped route configuration {} found", + scoped_route_config.name())); + } + const std::string scope_name = scoped_route_config.name(); + rds.set_route_config_name(scoped_route_config.route_configuration_name()); + ScopedRouteInfoConstSharedPtr scoped_route_info = std::make_shared( + std::move(scoped_route_config), srds_config_provider_manager_.createRouteConfigProvider( + factory_context_, rds, stat_prefix_)); + scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; + applyDeltaConfigUpdate( + [scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - }); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + }); + any_applied = true; + ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scope_name); + } catch (const EnvoyException& e) { + exception_msgs.emplace_back(fmt::format("{}", e.what())); + } } - for (const auto& scoped_route_name : removed_resources) { - ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - scoped_route_map_.erase(scoped_route_name); - applyDeltaConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { + for (const auto& scope_name : removed_resources) { + scoped_route_map_.erase(scope_name); + applyDeltaConfigUpdate([scope_name](const ConfigProvider::ConfigConstSharedPtr& config) { auto* thread_local_scoped_config = const_cast( static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scoped_route_name); + thread_local_scoped_config->removeRoutingScope(scope_name); }); + any_applied = true; + ENVOY_LOG(debug, "srds: remove scoped route '{}'", scope_name); } ConfigSubscriptionCommonBase::onConfigUpdate(); - setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + if (any_applied) { + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + } stats_.config_reload_.inc(); + if (!exception_msgs.empty()) { + throw EnvoyException(fmt::format("Error adding/updating scoped route(s): {}", + StringUtil::join(exception_msgs, ", "))); + } } +// TODO(stevenzzzz): consider generalizing this function as it overlaps with +// CdsApiImpl::onConfigUpdate. void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { - std::vector scoped_routes; + absl::flat_hash_map scoped_routes; for (const auto& resource_any : resources) { + // Throws (thus rejects all) on any error. auto scoped_route = MessageUtil::anyConvert( resource_any, validation_visitor_); MessageUtil::validate(scoped_route); - scoped_routes.emplace_back(std::move(scoped_route)); + if (!scoped_routes.try_emplace(scoped_route.name(), std::move(scoped_route)).second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + } } ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; Protobuf::RepeatedPtrField to_add_repeated; Protobuf::RepeatedPtrField to_remove_repeated; - - for (auto& scoped_route : scoped_routes) { - const std::string& scoped_route_name = scoped_route.name(); - scoped_routes_to_remove.erase(scoped_route_name); + for (auto& iter : scoped_routes) { + const std::string& scope_name = iter.first; + scoped_routes_to_remove.erase(scope_name); auto* to_add = to_add_repeated.Add(); - to_add->set_name(scoped_route.name()); + to_add->set_name(scope_name); to_add->set_version(version_info); - to_add->mutable_resource()->PackFrom(scoped_route); + to_add->mutable_resource()->PackFrom(iter.second); } for (const auto& scoped_route : scoped_routes_to_remove) { diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index ac7ba74b3d34..0c912e1ef89f 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -194,33 +194,18 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa return route_config_provider_manager_; } - std::shared_ptr - getRouteConfigProvider(Server::Configuration::FactoryContext& factory_context, - envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - std::string stat_name) { - const uint64_t route_provider_id = MessageUtil::hash(rds); - auto iter = cached_route_providers_.find(route_provider_id); - if (iter != cached_route_providers_.end()) { - return iter->second; - } - std::shared_ptr provider = - route_config_provider_manager_.createRdsRouteConfigProvider(rds, factory_context, - stat_name); - cached_route_providers_.try_emplace(route_provider_id, provider); - return provider; - } - void - deleteRouteConfigSource(envoy::config::filter::network::http_connection_manager::v2::Rds& rds) { - const uint64_t route_provider_id = MessageUtil::hash(rds); - if (!cached_route_providers_.erase(route_provider_id)) { - ENVOY_LOG_MISC(warn, "Try to delete non-existing route config source {}", rds.DebugString()); - } + std::shared_ptr createRouteConfigProvider( + Server::Configuration::FactoryContext& factory_context, + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + const std::string& stat_name) { + return route_config_provider_manager_.createRdsRouteConfigProvider(rds, factory_context, + stat_name); } private: RouteConfigProviderManager& route_config_provider_manager_; // From ConfigSource fingerprint to shared RouteConfigProvider. - absl::flat_hash_map> cached_route_providers_; + absl::flat_hash_map> cached_route_providers_; }; // The optional argument passed to the ConfigProviderManager::create*() functions. diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 0068101b9260..6fc06812e835 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -7,7 +7,6 @@ #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.validate.h" #include "envoy/filesystem/filesystem.h" -#include "envoy/registry/registry.h" #include "envoy/server/admin.h" #include "common/access_log/access_log_impl.h" @@ -104,7 +103,8 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( // This lambda captures the shared_ptrs created above, thus preserving the // reference count. - // Keep in mind the lambda capture list **doesn't** determine the destruction order, the + // Keep in mind the lambda capture list **doesn't** determine the destruction order, but it's fine + // as these captured objects are also global singletons. return [scoped_routes_config_provider_manager, route_config_provider_manager, date_provider, filter_config, &context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl( diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 9bd1f185fc42..a847e441e907 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -111,7 +111,7 @@ name: foo_scoped_routes scope_key_builder: fragments: - header_value_extractor: - name: X-Google-VIP + name: Addr element: key: x-foo-key separator: ; @@ -145,8 +145,10 @@ route_configuration_name: foo_routes parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources, "1"), ProtoValidationException); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), - ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), + EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'route_configuration_name' validation: value must be > 1 byte. const std::string config_yaml2 = R"EOF( @@ -159,8 +161,10 @@ name: foo_scope Protobuf::RepeatedPtrField resources2; parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources2, "1"), ProtoValidationException); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), - ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + subscription_callbacks_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), + EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'key' validation: must define at least 1 fragment. const std::string config_yaml3 = R"EOF( @@ -171,8 +175,11 @@ route_configuration_name: foo_routes Protobuf::RepeatedPtrField resources3; parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources3, "1"), ProtoValidationException); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), - ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + subscription_callbacks_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), + EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed .*value is " + "required.*"); } // Tests that multiple uniquely named resources are allowed in config updates. @@ -227,7 +234,9 @@ route_configuration_name: foo_routes EXPECT_THROW_WITH_MESSAGE( subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), - EnvoyException, "duplicate scoped route configuration foo_scope found"); + EnvoyException, + "Error adding/updating scoped route(s): duplicate scoped route configuration foo_scope " + "found"); } // Tests a config update failure. @@ -277,7 +286,7 @@ stat_prefix: foo scope_key_builder: fragments: - header_value_extractor: - name: X-Google-VIP + name: Addr index: 0 $1 )EOF"; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 54f9cbe5a89a..0c9113c13788 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -392,7 +392,7 @@ class MockRouteConfigProvider : public RouteConfigProvider { MOCK_CONST_METHOD0(lastUpdated, SystemTime()); MOCK_METHOD0(onConfigUpdate, void()); - std::shared_ptr> route_config_{new NiceMock()}; + std::shared_ptr> route_config_{new NiceMock()}; }; class MockRouteConfigProviderManager : public RouteConfigProviderManager { @@ -417,7 +417,7 @@ class MockScopedConfig : public ScopedConfig { MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); - std::shared_ptr route_config_{new NiceMock()}; + std::shared_ptr route_config_{new NiceMock()}; }; class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { From 5cad3eef7693dbad901035e550c89ffba559fe37 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 9 Jul 2019 18:22:57 -0400 Subject: [PATCH 16/53] add conflict detection for scope keys Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 51 ++++++++++++++++++--------- source/common/router/scoped_rds.h | 3 ++ test/common/router/scoped_rds_test.cc | 48 ++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 51762b90effd..a7a299bb1959 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -99,9 +99,6 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) { - - // TODO(stevenzzzzz): we need to discuss about what to do when two ScopedRouteConfiguration - // which has the same ScopeKey. bool any_applied = false; std::vector exception_msgs; absl::flat_hash_set unique_resource_names; @@ -114,14 +111,21 @@ void ScopedRdsConfigSubscription::onConfigUpdate( resource.resource(), validation_visitor_); MessageUtil::validate(scoped_route_config); if (!unique_resource_names.insert(scoped_route_config.name()).second) { - throw EnvoyException(fmt::format("duplicate scoped route configuration {} found", + throw EnvoyException(fmt::format("duplicate scoped route configuration '{}' found", scoped_route_config.name())); } - const std::string scope_name = scoped_route_config.name(); rds.set_route_config_name(scoped_route_config.route_configuration_name()); ScopedRouteInfoConstSharedPtr scoped_route_info = std::make_shared( std::move(scoped_route_config), srds_config_provider_manager_.createRouteConfigProvider( factory_context_, rds, stat_prefix_)); + // Detect if there is key conflict between two scopes. + auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); + if (iter != scope_name_by_hash_.end() && iter->second != scoped_route_info->scopeName()) { + throw EnvoyException( + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + iter->second, scoped_route_info->scopeName())); + } + scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; applyDeltaConfigUpdate( [scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { @@ -131,20 +135,24 @@ void ScopedRdsConfigSubscription::onConfigUpdate( thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); }); any_applied = true; - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scope_name); + ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_info->scopeName()); } catch (const EnvoyException& e) { exception_msgs.emplace_back(fmt::format("{}", e.what())); } } for (const auto& scope_name : removed_resources) { - scoped_route_map_.erase(scope_name); - applyDeltaConfigUpdate([scope_name](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scope_name); - }); - any_applied = true; - ENVOY_LOG(debug, "srds: remove scoped route '{}'", scope_name); + auto iter = scoped_route_map_.find(scope_name); + if (iter != scoped_route_map_.end()) { + scope_name_by_hash_.erase(iter->second->scopeKey().hash()); + scoped_route_map_.erase(iter); + applyDeltaConfigUpdate([scope_name](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scope_name); + }); + any_applied = true; + ENVOY_LOG(debug, "srds: remove scoped route '{}'", scope_name); + } } ConfigSubscriptionCommonBase::onConfigUpdate(); if (any_applied) { @@ -163,14 +171,25 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { absl::flat_hash_map scoped_routes; + absl::flat_hash_map scope_key_by_key_hash; for (const auto& resource_any : resources) { // Throws (thus rejects all) on any error. auto scoped_route = MessageUtil::anyConvert( resource_any, validation_visitor_); MessageUtil::validate(scoped_route); - if (!scoped_routes.try_emplace(scoped_route.name(), std::move(scoped_route)).second) { + const std::string scope_name = scoped_route.name(); + auto scope_config_inserted = scoped_routes.try_emplace(scope_name, std::move(scoped_route)); + if (!scope_config_inserted.second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration '{}' found", scoped_route.name())); + } + const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config = + scope_config_inserted.first->second; + uint64_t key_fingerprint = MessageUtil::hash(scoped_route_config.key()); + if (!scope_key_by_key_hash.try_emplace(key_fingerprint, scope_name).second) { throw EnvoyException( - fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + scope_key_by_key_hash[key_fingerprint], scope_name)); } } ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 0c912e1ef89f..99e981d8ece4 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -117,7 +117,10 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio .name(); } + // ScopedRouteInfo by scope name. ScopedRouteMap scoped_route_map_; + // A map of (hash, scope-name), used to detect the key conflict between scopes. + absl::flat_hash_map scope_name_by_hash_; // For creating RDS subscriptions. Server::Configuration::FactoryContext& factory_context_; const std::string name_; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index a847e441e907..686f601f03a7 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -182,7 +182,7 @@ route_configuration_name: foo_routes "required.*"); } -// Tests that multiple uniquely named resources are allowed in config updates. +// Tests that multiple uniquely named non-conflict resources are allowed in config updates. TEST_F(ScopedRdsTest, MultipleResources) { setup(); @@ -200,7 +200,7 @@ name: foo_scope2 route_configuration_name: foo_routes key: fragments: - - string_key: x-foo-key + - string_key: x-bar-key )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "1")); @@ -215,6 +215,46 @@ route_configuration_name: foo_routes factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); } +// Tests that conflict resources are detected. +TEST_F(ScopedRdsTest, MultipleResourcesWithKeyConflict) { + setup(); + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + EXPECT_THROW_WITH_REGEX( + subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, + ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); + EXPECT_EQ( + // Fully rejected. + 0UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + + // Delta API. + EXPECT_THROW_WITH_REGEX( + subscription_callbacks_->onConfigUpdate(anyToResource(resources, "2"), {}, "2"), + EnvoyException, + ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); + EXPECT_EQ( + // Partially reject. + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); +} + // Tests that only one resource is provided during a config update. TEST_F(ScopedRdsTest, InvalidDuplicateResource) { setup(); @@ -230,12 +270,12 @@ route_configuration_name: foo_routes parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW_WITH_MESSAGE(subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, - "duplicate scoped route configuration foo_scope found"); + "duplicate scoped route configuration 'foo_scope' found"); EXPECT_THROW_WITH_MESSAGE( subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, - "Error adding/updating scoped route(s): duplicate scoped route configuration foo_scope " + "Error adding/updating scoped route(s): duplicate scoped route configuration 'foo_scope' " "found"); } From 50e8fb5e1008c82e2b7f278fb1c57de1dee9fb0c Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 11 Jul 2019 23:33:31 -0400 Subject: [PATCH 17/53] Review feedbacks fix, and some more changes to the config provider framework: * The current framework doesn't have a mechanism to enforce all registered DeltaMutableConfigProviderBase of a DeltaConfigSubscriptionInstance share the same config instance (will addresss it in another PR if desired), fix the bug in Del taConfigSubscriptionInstance::applyDeltaConfigUpdate. * Add a complete_cb forDeltaConfigSubscriptionInstance::applyDeltaConfigUpdate() and DeltaMutableConfigProviderBase::onConfigUpdate(update_fn, complete_cb), as RouteConfigProvider instances has to be deleted in main thread, we need to de struct the ScopedRouteInfo object in the complete_cb. Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.cc | 19 +++++++++++++----- source/common/config/config_provider_impl.h | 17 ++++++++++++---- source/common/router/scoped_config_impl.h | 2 ++ source/common/router/scoped_rds.cc | 20 +++++++++++-------- source/common/router/scoped_rds.h | 2 -- .../config/config_provider_impl_test.cc | 6 +++--- test/common/router/scoped_rds_test.cc | 13 ++++++++++-- .../scoped_rds_integration_test.cc | 4 ++-- 8 files changed, 57 insertions(+), 26 deletions(-) diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index ed209e8427ad..d10a74b96f0b 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -76,7 +76,7 @@ bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Messa } void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( - const std::function& update_fn) { + const std::function& update_fn, Event::PostCb complete_cb) { // The Config implementation is assumed to be shared across the config providers bound to this // subscription, therefore, simply propagating the update to all worker threads for a single bound // provider will be sufficient. @@ -92,10 +92,19 @@ void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( // needed. Such logic could be generalized as part of this framework such that this function owns // the diffing and issues the corresponding call to add/modify/remove a resource according to a // vector of functions passed by the caller. - auto* typed_provider = - static_cast(getAnyBoundMutableConfigProvider()); - ConfigSharedPtr config = typed_provider->getConfig(); - typed_provider->onConfigUpdate([config, update_fn]() { update_fn(config); }); + // For now each config provider has its own copy of config, we need to propagate the update to + // every provider. + for (auto* provider : mutable_config_providers_) { + auto* typed_provider = static_cast(provider); + typed_provider->onConfigUpdate( + [update_fn, typed_provider]() { + // Note: this lambda is run on every worker thread, getting the config from within the + // lambda ensures us getting the per-worker config. + ConfigSharedPtr config = typed_provider->getConfig(); + update_fn(config); + }, + std::move(complete_cb)); + } } ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 2553b3976e59..29c4df358ec2 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -291,8 +291,11 @@ class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { * subscription. * * @param update_fn the callback to run on each worker thread. + * @param complete_cb the callback to run on each worker thread. NOTE it's called each time a + * registered provider's update propagation finishes. */ - void applyDeltaConfigUpdate(const std::function& update_fn); + void applyDeltaConfigUpdate(const std::function& update_fn, + Event::PostCb complete_cb = Event::PostCb()); }; /** @@ -409,10 +412,16 @@ class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { /** * Propagates a delta config update to all workers. - * @param updateCb the callback to run on each worker. + * @param update_cb the callback to run on each worker. + * @param complete_cb the callback to run in main thread after the update propagation is done on + * every worker thread. */ - void onConfigUpdate(Envoy::Event::PostCb update_cb) { - tls_->runOnAllThreads(std::move(update_cb)); + void onConfigUpdate(Envoy::Event::PostCb update_cb, Event::PostCb complete_cb) { + if (complete_cb) { + tls_->runOnAllThreads(std::move(update_cb), std::move(complete_cb)); + } else { + tls_->runOnAllThreads(std::move(update_cb)); + } } protected: diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index b569ed3b4f06..8e5fff1582f8 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -167,6 +167,8 @@ class ScopedRouteInfo { } } } + ~ScopedRouteInfo() = default; + Router::ConfigConstSharedPtr routeConfig() const { return route_provider_->config(); } const ScopeKey& scopeKey() const { return scope_key_; } const envoy::api::v2::ScopedRouteConfiguration& configProto() const { return config_proto_; } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index a7a299bb1959..a53e8591119c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -143,13 +143,17 @@ void ScopedRdsConfigSubscription::onConfigUpdate( for (const auto& scope_name : removed_resources) { auto iter = scoped_route_map_.find(scope_name); if (iter != scoped_route_map_.end()) { + ScopedRouteInfoConstSharedPtr to_be_deleted = iter->second; scope_name_by_hash_.erase(iter->second->scopeKey().hash()); scoped_route_map_.erase(iter); - applyDeltaConfigUpdate([scope_name](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scope_name); - }); + applyDeltaConfigUpdate( + [scope_name](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scope_name); + }, + // We need to delete the associated RouteConfigProvider in main thread. + [to_be_deleted]() { /*to_be_deleted is destructed in main thread.*/ }); any_applied = true; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scope_name); } @@ -171,7 +175,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { absl::flat_hash_map scoped_routes; - absl::flat_hash_map scope_key_by_key_hash; + absl::flat_hash_map scope_name_by_key_hash; for (const auto& resource_any : resources) { // Throws (thus rejects all) on any error. auto scoped_route = MessageUtil::anyConvert( @@ -186,10 +190,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config = scope_config_inserted.first->second; uint64_t key_fingerprint = MessageUtil::hash(scoped_route_config.key()); - if (!scope_key_by_key_hash.try_emplace(key_fingerprint, scope_name).second) { + if (!scope_name_by_key_hash.try_emplace(key_fingerprint, scope_name).second) { throw EnvoyException( fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", - scope_key_by_key_hash[key_fingerprint], scope_name)); + scope_name_by_key_hash[key_fingerprint], scope_name)); } } ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 99e981d8ece4..45948d0ffbed 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -207,8 +207,6 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa private: RouteConfigProviderManager& route_config_provider_manager_; - // From ConfigSource fingerprint to shared RouteConfigProvider. - absl::flat_hash_map> cached_route_providers_; }; // The optional argument passed to the ConfigProviderManager::create*() functions. diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 2b3e320a7a62..8dcf22c39527 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -737,9 +737,9 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { // Issue a second config update to validate that having multiple providers bound to the // subscription causes a single update to the underlying shared config implementation. subscription.onConfigUpdate(untyped_dummy_configs, "2"); - // NOTE: the config implementation is append only and _does not_ track updates/removals to the - // config proto set, so the expectation is to double the size of the set. - EXPECT_EQ(provider1->config()->numProtos(), 4); + // NOTE: the two providers share the same config, each config update propagation would add 2 + // protos to the same config's proto vector, so the expectation is 6 here. + EXPECT_EQ(provider1->config()->numProtos(), 6); EXPECT_EQ(provider1->configProtoInfoVector().value().version_, "2"); } diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 686f601f03a7..f250ce4d06ba 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -92,8 +92,6 @@ class ScopedRoutesTestBase : public testing::Test { Event::SimulatedTimeSystem& timeSystem() { return time_system_; } NiceMock factory_context_; - Upstream::ClusterManager::ClusterInfoMap cluster_map_; - Upstream::MockClusterMockPrioritySet cluster_; std::unique_ptr config_provider_manager_; MockRouteConfigProviderManager route_config_provider_manager_; absl::flat_hash_map> cached_route_configs_; @@ -253,6 +251,17 @@ route_configuration_name: foo_routes // Partially reject. 1UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // foo_scope update is applied. + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .size(), + 1UL); + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .count("foo_scope"), + 1); } // Tests that only one resource is provided during a config update. diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 13f63fb1e42a..2c294366ce3b 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -49,7 +49,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, const std::string& scope_key_builder_config_yaml = R"EOF( fragments: - header_value_extractor: - name: X-Google-VIP + name: Addr element: key: x-foo-key separator: ; @@ -175,7 +175,7 @@ name: foo_scope2 route_configuration_name: foo_route1 key: fragments: - - string_key: x-foo-key + - string_key: x-bar-key )EOF"; const std::string route_config_tmpl = R"EOF( From cb5324f56bff24dc26a058126002236c0285531b Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 12 Jul 2019 13:05:39 -0400 Subject: [PATCH 18/53] make SRDS the SotW onConfigUpdate API a quasi-incremental one per dicussino w/ htuch Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 15 ++++----- test/common/router/scoped_rds_test.cc | 46 ++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index a53e8591119c..43470add6e85 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -169,8 +169,13 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } } -// TODO(stevenzzzz): consider generalizing this function as it overlaps with +// TODO(stevenzzzz): see issue #7508, consider generalizing this function as it overlaps with // CdsApiImpl::onConfigUpdate. +// TODO(stevenzzzz): revisit the handling of deleted scopes here, per @htuch, SRDS's SotW update API +// should be similar to RDS' on the wire, act in a quasi-incremental way. See related discussion +// https://github.com/cncf/udpa-wg or +// https://blog.envoyproxy.io/the-universal-data-plane-api-d15cec7a. +// For now, we make this a quasi-incremental API, i.e., no removal of scopes(RouteConfigurations). void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { @@ -196,22 +201,16 @@ void ScopedRdsConfigSubscription::onConfigUpdate( scope_name_by_key_hash[key_fingerprint], scope_name)); } } - ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; Protobuf::RepeatedPtrField to_add_repeated; - Protobuf::RepeatedPtrField to_remove_repeated; for (auto& iter : scoped_routes) { const std::string& scope_name = iter.first; - scoped_routes_to_remove.erase(scope_name); auto* to_add = to_add_repeated.Add(); to_add->set_name(scope_name); to_add->set_version(version_info); to_add->mutable_resource()->PackFrom(iter.second); } - for (const auto& scoped_route : scoped_routes_to_remove) { - *to_remove_repeated.Add() = scoped_route.first; - } - onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); + onConfigUpdate(to_add_repeated, {}, version_info); } ScopedRdsConfigProvider::ScopedRdsConfigProvider( diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index f250ce4d06ba..cb15dfca2474 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -211,6 +211,43 @@ route_configuration_name: foo_routes EXPECT_EQ( 2UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .size(), + 2); + + // Deletion happens on delta API only. + resources.RemoveLast(); + + EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "3")); + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .size(), + 2); + EXPECT_EQ( + 3UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + Protobuf::RepeatedPtrField deletes; + *deletes.Add() = "foo_scope2"; + EXPECT_NO_THROW( + subscription_callbacks_->onConfigUpdate(anyToResource(resources, "4"), deletes, "4")); + // foo_scope2 is deleted. + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .size(), + 1); + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .count("foo_scope"), + 1); + EXPECT_EQ( + 4UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); } // Tests that conflict resources are detected. @@ -455,16 +492,23 @@ route_configuration_name: dynamic-foo-route-config nanos: 234000000 dynamic_scoped_route_configs: - name: foo-dynamic-scoped-routes + scoped_route_configs: + - name: dynamic-foo + route_configuration_name: dynamic-foo-route-config + key: + fragments: { string_key: "172.30.30.10" } last_updated: seconds: 1234567891 nanos: 567000000 - version_info: "2" + version_info: "1" )EOF", expected_config_dump); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump4 = MessageUtil::downcastAndValidate( *message_ptr); + // The StoW update API acts in a quasi-incremental way, there is no deletion, and no change, so no + // version flip. EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } From 5c8a5464b1efffa44fc025a49c059be656df57ce Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 12 Jul 2019 14:42:40 -0400 Subject: [PATCH 19/53] fix clang-tidy bug Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.cc | 2 +- source/common/router/scoped_rds.cc | 8 ++++---- test/common/router/scoped_rds_test.cc | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index d10a74b96f0b..61a0f1d7c7e1 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -103,7 +103,7 @@ void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( ConfigSharedPtr config = typed_provider->getConfig(); update_fn(config); }, - std::move(complete_cb)); + complete_cb); } } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 43470add6e85..13d772ed0185 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -171,9 +171,9 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // TODO(stevenzzzz): see issue #7508, consider generalizing this function as it overlaps with // CdsApiImpl::onConfigUpdate. -// TODO(stevenzzzz): revisit the handling of deleted scopes here, per @htuch, SRDS's SotW update API -// should be similar to RDS' on the wire, act in a quasi-incremental way. See related discussion -// https://github.com/cncf/udpa-wg or +// TODO(stevenzzzz): revisit the handling of deleted scopes here, per @htuch, the SRDS state of the +// world update API should be similar to RDS' on the wire, act in a quasi-incremental way. See +// related discussion https://github.com/cncf/udpa-wg or // https://blog.envoyproxy.io/the-universal-data-plane-api-d15cec7a. // For now, we make this a quasi-incremental API, i.e., no removal of scopes(RouteConfigurations). void ScopedRdsConfigSubscription::onConfigUpdate( @@ -190,7 +190,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( auto scope_config_inserted = scoped_routes.try_emplace(scope_name, std::move(scoped_route)); if (!scope_config_inserted.second) { throw EnvoyException( - fmt::format("duplicate scoped route configuration '{}' found", scoped_route.name())); + fmt::format("duplicate scoped route configuration '{}' found", scope_name)); } const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config = scope_config_inserted.first->second; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index cb15dfca2474..7f29a1f4ae08 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -77,7 +77,8 @@ class ScopedRoutesTestBase : public testing::Test { // The delta style API helper. Protobuf::RepeatedPtrField - anyToResource(Protobuf::RepeatedPtrField& resources, std::string version) { + anyToResource(Protobuf::RepeatedPtrField& resources, + const std::string& version) { Protobuf::RepeatedPtrField added_resources; for (const auto& resource_any : resources) { auto config = TestUtility::anyConvert(resource_any); @@ -507,8 +508,8 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump4 = MessageUtil::downcastAndValidate( *message_ptr); - // The StoW update API acts in a quasi-incremental way, there is no deletion, and no change, so no - // version flip. + // The delta update API acts in a quasi-incremental way, there is no deletion, and no change, so + // no version flip. EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } From 4a1fb01bdc1e6da73a8af2c31e6d8dadba3b15b9 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 15 Jul 2019 10:54:53 -0400 Subject: [PATCH 20/53] revert the quasi-incremental delta onConfigUpdate change after offline discussion Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 13 ++++++------ test/common/router/scoped_rds_test.cc | 30 +++++++++++---------------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 13d772ed0185..3be47136056b 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -171,11 +171,6 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // TODO(stevenzzzz): see issue #7508, consider generalizing this function as it overlaps with // CdsApiImpl::onConfigUpdate. -// TODO(stevenzzzz): revisit the handling of deleted scopes here, per @htuch, the SRDS state of the -// world update API should be similar to RDS' on the wire, act in a quasi-incremental way. See -// related discussion https://github.com/cncf/udpa-wg or -// https://blog.envoyproxy.io/the-universal-data-plane-api-d15cec7a. -// For now, we make this a quasi-incremental API, i.e., no removal of scopes(RouteConfigurations). void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { @@ -201,16 +196,22 @@ void ScopedRdsConfigSubscription::onConfigUpdate( scope_name_by_key_hash[key_fingerprint], scope_name)); } } + ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; Protobuf::RepeatedPtrField to_add_repeated; + Protobuf::RepeatedPtrField to_remove_repeated; for (auto& iter : scoped_routes) { const std::string& scope_name = iter.first; + scoped_routes_to_remove.erase(scope_name); auto* to_add = to_add_repeated.Add(); to_add->set_name(scope_name); to_add->set_version(version_info); to_add->mutable_resource()->PackFrom(iter.second); } - onConfigUpdate(to_add_repeated, {}, version_info); + for (const auto& scoped_route : scoped_routes_to_remove) { + *to_remove_repeated.Add() = scoped_route.first; + } + onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); } ScopedRdsConfigProvider::ScopedRdsConfigProvider( diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 7f29a1f4ae08..323100f264fb 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -219,7 +219,7 @@ route_configuration_name: foo_routes .size(), 2); - // Deletion happens on delta API only. + // Delete foo_scope2. resources.RemoveLast(); EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "3")); @@ -227,25 +227,26 @@ route_configuration_name: foo_routes ->subscription() .scopedRouteMap() .size(), - 2); + 1); + EXPECT_EQ(dynamic_cast(provider_.get()) + ->subscription() + .scopedRouteMap() + .count("foo_scope"), + 1); EXPECT_EQ( 3UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // Delete foo_scope via Delta API. Protobuf::RepeatedPtrField deletes; - *deletes.Add() = "foo_scope2"; + *deletes.Add() = "foo_scope"; + resources.RemoveLast(); EXPECT_NO_THROW( subscription_callbacks_->onConfigUpdate(anyToResource(resources, "4"), deletes, "4")); - // foo_scope2 is deleted. EXPECT_EQ(dynamic_cast(provider_.get()) ->subscription() .scopedRouteMap() .size(), - 1); - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .count("foo_scope"), - 1); + 0); EXPECT_EQ( 4UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); @@ -493,23 +494,16 @@ route_configuration_name: dynamic-foo-route-config nanos: 234000000 dynamic_scoped_route_configs: - name: foo-dynamic-scoped-routes - scoped_route_configs: - - name: dynamic-foo - route_configuration_name: dynamic-foo-route-config - key: - fragments: { string_key: "172.30.30.10" } last_updated: seconds: 1234567891 nanos: 567000000 - version_info: "1" + version_info: "2" )EOF", expected_config_dump); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump4 = MessageUtil::downcastAndValidate( *message_ptr); - // The delta update API acts in a quasi-incremental way, there is no deletion, and no change, so - // no version flip. EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } From 467a6a198dadb4cae8a7661106feb06183eb3f6e Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 23 Jul 2019 22:46:06 -0400 Subject: [PATCH 21/53] refactor config provider framework Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.cc | 61 +--- source/common/config/config_provider_impl.h | 289 +++++++----------- source/common/router/BUILD | 1 + source/common/router/scoped_config_impl.h | 2 +- source/common/router/scoped_rds.cc | 59 ++-- source/common/router/scoped_rds.h | 24 +- .../config/config_provider_impl_test.cc | 206 ++++++------- test/mocks/config/mocks.cc | 7 - test/mocks/config/mocks.h | 14 - 9 files changed, 238 insertions(+), 425 deletions(-) diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index ed209e8427ad..8deac8cae62e 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -23,24 +23,6 @@ ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } - -void ConfigSubscriptionCommonBase::bindConfigProvider(MutableConfigProviderCommonBase* provider) { - // All config providers bound to a ConfigSubscriptionCommonBase must be of the same concrete - // type; this is assumed by ConfigSubscriptionInstance::checkAndApplyConfigUpdate() and is - // verified by the assertion below. NOTE: an inlined statement ASSERT() triggers a potentially - // evaluated expression warning from clang due to `typeid(**mutable_config_providers_.begin())`. - // To avoid this, we use a lambda to separate the first mutable provider dereference from the - // typeid() statement. - ASSERT([&]() { - if (!mutable_config_providers_.empty()) { - const auto& first_provider = **mutable_config_providers_.begin(); - return typeid(*provider) == typeid(first_provider); - } - return true; - }()); - mutable_config_providers_.insert(provider); -} - bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info) { @@ -55,49 +37,12 @@ bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Messa config_info_ = {new_hash, version_info}; ENVOY_LOG(debug, "{}: loading new configuration: config_name={} hash={}", name_, config_name, new_hash); - - ASSERT(!mutable_config_providers_.empty()); - ConfigProvider::ConfigConstSharedPtr new_config; - for (auto* provider : mutable_config_providers_) { - // All bound mutable config providers must be of the same type (see the ASSERT... in - // bindConfigProvider()). - // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config - // impl, which can then be passed to all providers. - auto* typed_provider = static_cast(provider); - if (new_config == nullptr) { - if ((new_config = typed_provider->onConfigProtoUpdate(config_proto)) == nullptr) { - return false; - } - } - typed_provider->onConfigUpdate(new_config); - } - + auto new_config_impl = onConfigProtoUpdate(config_proto); + applyConfigUpdate([new_config_impl](ConfigProvider::ConfigConstSharedPtr) + -> ConfigProvider::ConfigConstSharedPtr { return new_config_impl; }); return true; } -void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( - const std::function& update_fn) { - // The Config implementation is assumed to be shared across the config providers bound to this - // subscription, therefore, simply propagating the update to all worker threads for a single bound - // provider will be sufficient. - if (mutable_config_providers_.size() > 1) { - ASSERT(static_cast(*mutable_config_providers_.begin()) - ->getConfig() == static_cast( - *std::next(mutable_config_providers_.begin())) - ->getConfig()); - } - - // TODO(AndresGuedez): currently, the caller has to compute the differences in resources between - // DS API config updates and passes a granular update_fn() that adds/modifies/removes resources as - // needed. Such logic could be generalized as part of this framework such that this function owns - // the diffing and issues the corresponding call to add/modify/remove a resource according to a - // vector of functions passed by the caller. - auto* typed_provider = - static_cast(getAnyBoundMutableConfigProvider()); - ConfigSharedPtr config = typed_provider->getConfig(); - typed_provider->onConfigUpdate([config, update_fn]() { update_fn(config); }); -} - ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name) { config_tracker_entry_ = diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 2553b3976e59..de914b0df7df 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -20,11 +20,11 @@ namespace Envoy { namespace Config { // This file provides a set of base classes, (ImmutableConfigProviderBase, -// MutableConfigProviderCommonBase, MutableConfigProviderBase, DeltaMutableConfigProviderBase, -// ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, ConfigSubscriptionInstance, -// DeltaConfigSubscriptionInstance), conforming to the ConfigProvider/ConfigProviderManager -// interfaces, which in tandem provide a framework for implementing statically defined (i.e., -// immutable) and dynamic (mutable via subscriptions) configuration for Envoy. +// MutableConfigProviderCommonBase, ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, +// ConfigSubscriptionInstance, DeltaConfigSubscriptionInstance), conforming to the +// ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a framework for +// implementing statically defined (i.e., immutable) and dynamic (mutable via subscriptions) +// configuration for Envoy. // // The mutability property applies to the ConfigProvider itself and _not_ the underlying config // proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto @@ -58,11 +58,12 @@ namespace Config { // interface. // // For mutable (xDS) providers: -// 1) According to the API type, create a class derived from MutableConfigProviderBase or -// DeltaMutableConfigProviderBase and implement the required interface. -// 2) According to the API type, create a class derived from ConfigSubscriptionInstance or -// DeltaConfigSubscriptionInstance; this is the entity responsible for owning and managing the -// Envoy::Config::Subscription that provides the underlying config subscription. +// 1) According to the API type, create a class derived from MutableConfigProviderCommonBase and +// implement the required interface. +// 2) According to the API type, create a class derived from +// ConfigSubscriptionInstance or DeltaConfigSubscriptionInstance; this is the entity responsible +// for owning and managing the Envoy::Config::Subscription that provides the +// underlying config subscription, and the Config implemention shared by associated providers. // a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a // ConfigSubscriptionInstance child): // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the @@ -78,8 +79,8 @@ namespace Config { // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the // underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called // as well. -// - On a successful config update, applyConfigUpdate() should be called to propagate the config -// updates to all bound config providers and worker threads. +// - On a successful config update, applyConfigUpdate() should be called to propagate the +// config updates to all bound config providers and worker threads. class ConfigProviderManagerImplBase; @@ -130,12 +131,13 @@ class ImmutableConfigProviderBase : public ConfigProvider { class MutableConfigProviderCommonBase; /** - * Provides common DS API subscription functionality required by the ConfigProvider::ApiType - * specific base classes (see ConfigSubscriptionInstance and DeltaConfigSubscriptionInstance). + * Provides common DS API subscription functionality required by the ConfigProvider::ApiType. * - * To do so, this class keeps track of a set of MutableConfigProviderCommonBase instances associated - * with an underlying subscription; providers are bound/unbound as needed as they are created and - * destroyed. + * This class can not be instantiated directly; instead, it provides the foundation for + * config subscription implementations which derive from it. + * + * A subscription is supposed to be co-owned by config providers with the same config source, it's + * designed to be created/destructed on admin thread only. * * xDS config providers and subscriptions are split to avoid lifetime issues with arguments * required by the config providers. An example is the Server::Configuration::FactoryContext, which @@ -143,10 +145,10 @@ class MutableConfigProviderCommonBase; * in use (see #3960). This split enables single ownership of the config providers, while enabling * shared ownership of the underlying subscription. * - * This class can not be instantiated directly; instead, it provides the foundation for - * config subscription implementations which derive from it. */ -class ConfigSubscriptionCommonBase : protected Logger::Loggable { +class ConfigSubscriptionCommonBase + : protected Logger::Loggable, + public std::enable_shared_from_this { public: struct LastConfigInfo { absl::optional last_config_hash_; @@ -166,6 +168,10 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable& configInfo() const { return config_info_; } + ConfigProvider::ConfigConstSharedPtr getConfig() const { + return tls_->getTyped().config_; + } + /** * Must be called by derived classes when the onConfigUpdate() callback associated with the * underlying subscription is issued. @@ -184,25 +190,45 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable& update_fn, + Event::PostCb complete_cb = []() {}) { + // It is safe to call shared_from_this here as this is in main thread, and destruction of a + // ConfigSubscriptionCommonBase owner (i.e., a provider) happens in main thread as well. + auto shared_this = shared_from_this(); + tls_->runOnAllThreads( + [this, update_fn]() { + tls_->getTyped().config_ = update_fn(this->getConfig()); + }, + /*Make sure this subscription will not be teared down during the update propagation.*/ + [shared_this, complete_cb]() { complete_cb(); }); } void setLastUpdated() { last_updated_ = time_source_.systemTime(); } @@ -212,16 +238,12 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable mutable_config_providers_; absl::optional config_info_; + // This slot holds a Config implementation in each thread, which is shared between + // bound providers. + ThreadLocal::SlotPtr tls_; private: - void bindConfigProvider(MutableConfigProviderCommonBase* provider); - - void unbindConfigProvider(MutableConfigProviderCommonBase* provider) { - mutable_config_providers_.erase(provider); - } - Init::TargetImpl init_target_; const uint64_t manager_identifier_; ConfigProviderManagerImplBase& config_provider_manager_; @@ -235,28 +257,34 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggables and // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit // reference counting would be more maintainable. - friend class MutableConfigProviderCommonBase; - friend class MutableConfigProviderBase; - friend class DeltaMutableConfigProviderBase; friend class ConfigProviderManagerImplBase; - friend class MockMutableConfigProviderBase; }; using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr; /** * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs. + * A single Config instance is shared across all providers and all workers associated with this + * subscription. */ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { -protected: +public: ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager, - TimeSource& time_source, const SystemTime& last_updated, - const LocalInfo::LocalInfo& local_info) - : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, - last_updated, local_info) {} + Server::Configuration::FactoryContext& factory_context) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, + factory_context) {} - ~ConfigSubscriptionInstance() override = default; + /** + * Must be called by the derived class' constructor. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with + * the underlying subscription, shared across all providers and workers. + */ + void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) { + tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(initial_config); + }); + } /** * Determines whether a configuration proto is a new update, and if so, propagates it to all @@ -268,43 +296,52 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { */ bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info); -}; -using ConfigSharedPtr = std::shared_ptr; +protected: + /** + * Called when a new config proto is received via an xDS subscription. + * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config + * implementation that will be propagated to all mutable config providers sharing the + * subscription. + * Note that this function is called _once_ across all shared config providers per xDS + * subscription config update. + * @param config_proto supplies the configuration proto. + * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. + */ + virtual ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; +}; /** * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs. */ class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { protected: - DeltaConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager, - TimeSource& time_source, const SystemTime& last_updated, - const LocalInfo::LocalInfo& local_info) - : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, - last_updated, local_info) {} - + using ConfigSubscriptionCommonBase::ConfigSubscriptionCommonBase; ~DeltaConfigSubscriptionInstance() override = default; /** - * Propagates a config update to the config providers and worker threads associated with the - * subscription. - * - * @param update_fn the callback to run on each worker thread. + * Must be called by the derived class' constructor. + * @param init_cb supplies an initial Envoy::Config::ConfigProvider::Config associated with the + * underlying subscription for each worker thread. */ - void applyDeltaConfigUpdate(const std::function& update_fn); + void initialize(const std::function& init_cb) { + tls_->set([init_cb](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(init_cb()); + }); + } }; /** * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config - * providers (see MutableConfigProviderBase and DeltaMutableConfigProviderBase). + * providers (see MutableConfigProviderBase for example). * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. */ class MutableConfigProviderCommonBase : public ConfigProvider { public: - ~MutableConfigProviderCommonBase() override { subscription_->unbindConfigProvider(this); } + ~MutableConfigProviderCommonBase() override = default; // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } @@ -312,126 +349,16 @@ class MutableConfigProviderCommonBase : public ConfigProvider { protected: MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, ApiType api_type) - : tls_(factory_context.threadLocal().allocateSlot()), subscription_(subscription), - api_type_(api_type) {} - - ThreadLocal::SlotPtr tls_; - ConfigSubscriptionCommonBaseSharedPtr subscription_; + : subscription_(subscription), api_type_(api_type) {} -private: - ApiType api_type_; -}; - -/** - * Provides common mutable (dynamic) config provider functionality required by - * ConfigProvider::ApiType::Full DS APIs. - */ -class MutableConfigProviderBase : public MutableConfigProviderCommonBase { -public: // Envoy::Config::ConfigProvider - // NOTE: This is being promoted to public for internal uses to avoid an unnecessary dynamic_cast - // in the public API (ConfigProvider::config()). - ConfigConstSharedPtr getConfig() const override { - return tls_->getTyped().config_; - } - - /** - * Called when a new config proto is received via an xDS subscription. - * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config - * implementation that will be propagated to all mutable config providers sharing the - * subscription. - * Note that this function is called _once_ across all shared config providers per xDS - * subscription config update. - * @param config_proto supplies the configuration proto. - * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. - */ - virtual ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; - - /** - * Must be called by the derived class' constructor. - * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with - * the underlying subscription. - */ - void initialize(const ConfigConstSharedPtr& initial_config) { - subscription_->bindConfigProvider(this); - tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(initial_config); - }); - } - - /** - * Propagates a newly instantiated Envoy::Config::ConfigProvider::Config to all workers. - * @param config supplies the newly instantiated config. - */ - void onConfigUpdate(const ConfigConstSharedPtr& config) { - if (getConfig() == config) { - return; - } - tls_->runOnAllThreads( - [this, config]() -> void { tls_->getTyped().config_ = config; }); - } + ConfigConstSharedPtr getConfig() const override { return subscription_->getConfig(); } -protected: - MutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - - ~MutableConfigProviderBase() override = default; + ConfigSubscriptionCommonBaseSharedPtr subscription_; private: - struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { - ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config) - : config_(std::move(initial_config)) {} - - ConfigProvider::ConfigConstSharedPtr config_; - }; -}; - -/** - * Provides common mutable (dynamic) config provider functionality required by - * ConfigProvider::ApiType::Delta DS APIs. - */ -class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { -public: - // Envoy::Config::ConfigProvider - // This promotes getConfig() to public so that internal uses can avoid an unnecessary dynamic_cast - // in the public API (ConfigProvider::config()). - ConfigConstSharedPtr getConfig() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - - /** - * Non-const overload for use within the framework. - * @return ConfigSharedPtr the config implementation associated with the provider. - */ - virtual ConfigSharedPtr getConfig() PURE; - - /** - * Propagates a delta config update to all workers. - * @param updateCb the callback to run on each worker. - */ - void onConfigUpdate(Envoy::Event::PostCb update_cb) { - tls_->runOnAllThreads(std::move(update_cb)); - } - -protected: - DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - - ~DeltaMutableConfigProviderBase() override = default; - - /** - * Must be called by the derived class' constructor. - * @param initializeCb supplies the initialization callback to be issued for each worker - * thread. - */ - void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { - subscription_->bindConfigProvider(this); - tls_->set(std::move(initializeCb)); - } + ApiType api_type_; }; /** diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 445cc2232673..b08e7c763fbb 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -175,6 +175,7 @@ envoy_cc_library( hdrs = ["scoped_rds.h"], deps = [ ":scoped_config_lib", + "//include/envoy/config:config_provider_interface", "//include/envoy/config:subscription_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 808b442838c4..9f7a02b75f99 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -141,7 +141,7 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { * ConnectionManagerImpl::refreshCachedRoute() will call getRouterConfig() to obtain the * Router::ConfigConstSharedPtr to use for route selection. */ -class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { +class ThreadLocalScopedConfigImpl : public ScopedConfig { public: ThreadLocalScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) : scope_key_builder_(std::move(scope_key_builder)) {} diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 86e1d1dc88b8..9aa94a915a29 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -76,12 +76,13 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager) - : DeltaConfigSubscriptionInstance( - "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()), - name_(name), + : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, + factory_context), + name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), validation_visitor_(factory_context.messageValidationVisitor()) { @@ -91,6 +92,12 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), *scope_, *this); + + initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { + return std::make_shared( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( + scope_key_builder)); + }); } void ScopedRdsConfigSubscription::onConfigUpdate( @@ -124,10 +131,12 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ScopedRouteInfoConstSharedPtr scoped_route_info = scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - applyDeltaConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { + applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) + -> ConfigProvider::ConfigConstSharedPtr { auto* thread_local_scoped_config = const_cast( static_cast(config.get())); thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; }); } @@ -135,35 +144,28 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const std::string scoped_route_name = scoped_route.first; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); scoped_config_manager_.removeRoutingScope(scoped_route_name); - applyDeltaConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { + applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) + -> ConfigProvider::ConfigConstSharedPtr { + // In place update. auto* thread_local_scoped_config = const_cast( static_cast(config.get())); thread_local_scoped_config->removeRoutingScope(scoped_route_name); + return config; }); } - ConfigSubscriptionCommonBase::onConfigUpdate(); + DeltaConfigSubscriptionInstance::onConfigUpdate(); setLastConfigInfo(absl::optional({absl::nullopt, version_info})); stats_.config_reload_.inc(); } ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - envoy::api::v2::core::ConfigSource rds_config_source, - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder) - : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), + envoy::api::v2::core::ConfigSource rds_config_source) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), subscription_(static_cast( MutableConfigProviderCommonBase::subscription_.get())), - rds_config_source_(std::move(rds_config_source)) { - initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared( - envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( - scope_key_builder)); - }); -} + rds_config_source_(std::move(rds_config_source)) {} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); @@ -207,28 +209,25 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, const ConfigProviderManager::OptionalArg& optarg) { + const auto& typed_optarg = static_cast(optarg); ScopedRdsConfigSubscriptionSharedPtr subscription = ConfigProviderManagerImplBase::getSubscription( config_source_proto, factory_context.initManager(), [&config_source_proto, &factory_context, &stat_prefix, - &optarg](const uint64_t manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager) + &typed_optarg](const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) -> Envoy::Config::ConfigSubscriptionCommonBaseSharedPtr { const auto& scoped_rds_config_source = dynamic_cast< const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( config_source_proto); return std::make_shared( - scoped_rds_config_source, manager_identifier, - static_cast(optarg) - .scoped_routes_name_, - factory_context, stat_prefix, + scoped_rds_config_source, manager_identifier, typed_optarg.scoped_routes_name_, + typed_optarg.scope_key_builder_, factory_context, stat_prefix, static_cast(config_provider_manager)); }); - const auto& typed_optarg = static_cast(optarg); - return std::make_unique(std::move(subscription), factory_context, - typed_optarg.rds_config_source_, - typed_optarg.scope_key_builder_); + return std::make_unique(std::move(subscription), + typed_optarg.rds_config_source_); } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index f0cc72f71c15..725c3ec755ea 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -86,6 +86,8 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager); @@ -98,7 +100,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio } private: - // Envoy::Config::ConfigSubscriptionCommonBase + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks @@ -109,7 +111,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void onConfigUpdateFailed(const EnvoyException*) override { - ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + DeltaConfigSubscriptionInstance::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, @@ -119,6 +121,8 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const std::string name_; std::unique_ptr subscription_; + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder_; Stats::ScopePtr scope_; ScopedRdsStats stats_; ScopedConfigManager scoped_config_manager_; @@ -129,25 +133,13 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr(tls_->get()); - } - private: ScopedRdsConfigSubscription* subscription_; const envoy::api::v2::core::ConfigSource rds_config_source_; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 2b3e320a7a62..a7d631188a55 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -4,7 +4,6 @@ #include "common/protobuf/utility.h" #include "test/common/config/dummy_config.pb.h" -#include "test/mocks/config/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -20,6 +19,22 @@ using testing::InSequence; class DummyConfigProviderManager; +class DummyConfig : public Envoy::Config::ConfigProvider::Config { +public: + DummyConfig() {} + DummyConfig(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + void addProto(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + + uint32_t numProtos() const { return protos_.size(); } + +private: + std::vector protos_; +}; + class StaticDummyConfigProvider : public ImmutableConfigProviderBase { public: StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, @@ -48,12 +63,18 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, DummyConfigSubscription(const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager); - ~DummyConfigSubscription() override = default; // Envoy::Config::ConfigSubscriptionCommonBase void start() override {} + // Envoy::Config::ConfigSubscriptionInstance + ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config_proto) override { + return std::make_shared( + static_cast(config_proto)); + } + // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { @@ -84,33 +105,17 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, }; using DummyConfigSubscriptionSharedPtr = std::shared_ptr; -class DummyConfig : public ConfigProvider::Config { +class DummyDynamicConfigProvider : public MutableConfigProviderCommonBase { public: - DummyConfig(const test::common::config::DummyConfig&) {} -}; - -class DummyDynamicConfigProvider : public MutableConfigProviderBase { -public: - DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, - const ConfigConstSharedPtr& initial_config, - Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), + DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ApiType::Full), subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())) { - initialize(initial_config); - } + MutableConfigProviderCommonBase::subscription_.get())) {} ~DummyDynamicConfigProvider() override = default; DummyConfigSubscription& subscription() { return *subscription_; } - // Envoy::Config::MutableConfigProviderBase - ConfigProvider::ConfigConstSharedPtr - onConfigProtoUpdate(const Protobuf::Message& config) override { - return std::make_shared( - static_cast(config)); - } - // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { if (!subscription_->config_proto().has_value()) { @@ -177,14 +182,7 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { static_cast(config_provider_manager)); }); - ConfigProvider::ConfigConstSharedPtr initial_config; - const auto* provider = static_cast( - subscription->getAnyBoundMutableConfigProvider()); - if (provider) { - initial_config = provider->getConfig(); - } - return std::make_unique(std::move(subscription), initial_config, - factory_context); + return std::make_unique(std::move(subscription)); } // Envoy::Config::ConfigProviderManager @@ -204,6 +202,15 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { } }; +DummyConfigSubscription::DummyConfigSubscription( + const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager) + : ConfigSubscriptionInstance("DummyDS", manager_identifier, config_provider_manager, + factory_context) { + // Returns a null value. + initialize(nullptr); +} + StaticDummyConfigProvider::StaticDummyConfigProvider( const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, @@ -212,13 +219,6 @@ StaticDummyConfigProvider::StaticDummyConfigProvider( ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} -DummyConfigSubscription::DummyConfigSubscription( - const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, - DummyConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstance( - "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()) {} - class ConfigProviderImplTest : public testing::Test { public: void initialize() { @@ -338,15 +338,14 @@ class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderM manager_identifier, factory_context, static_cast(config_provider_manager)); }); - return std::make_unique(std::move(subscription), nullptr, - factory_context); + return std::make_unique(std::move(subscription)); } }; // Test that duplicate config updates will not trigger creation of a new ConfigProvider::Config. TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { InSequence sequence; - // This provider manager returns a MockMutableConfigProviderBase. + // This provider manager returns a DummyDynamicConfigProvider. auto provider_manager = std::make_unique(factory_context_.admin_); envoy::api::v2::core::ApiConfigSource config_source_proto; @@ -354,18 +353,22 @@ TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { ConfigProviderPtr provider = provider_manager->createXdsConfigProvider( config_source_proto, factory_context_, "dummy_prefix", ConfigProviderManager::NullOptionalArg()); - auto* typed_provider = static_cast(provider.get()); + auto* typed_provider = static_cast(provider.get()); DummyConfigSubscription& subscription = static_cast(typed_provider->subscription()); + EXPECT_EQ(subscription.getConfig(), nullptr); // First time issuing a configUpdate(). A new ConfigProvider::Config should be created. - EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(1); Protobuf::RepeatedPtrField untyped_dummy_configs; untyped_dummy_configs.Add()->PackFrom(parseDummyConfigFromYaml("a: a dynamic dummy config")); subscription.onConfigUpdate(untyped_dummy_configs, "1"); + EXPECT_NE(subscription.getConfig(), nullptr); + auto config_ptr = subscription.getConfig(); + EXPECT_EQ(typed_provider->config().get(), config_ptr.get()); // Second time issuing the configUpdate(), this time with a duplicate proto. A new // ConfigProvider::Config _should not_ be created. - EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(0); - subscription.onConfigUpdate(untyped_dummy_configs, "1"); + subscription.onConfigUpdate(untyped_dummy_configs, "2"); + EXPECT_EQ(config_ptr, subscription.getConfig()); + EXPECT_EQ(typed_provider->config().get(), config_ptr.get()); } // An empty config provider tests on base class' constructor. @@ -518,7 +521,31 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) override; + const std::string& version_info) override { + if (resources.empty()) { + return; + } + + // For simplicity, there is no logic here to track updates and/or removals to the existing + // config proto set (i.e., this is append only). Real xDS APIs will need to track additions, + // updates and removals to the config set and apply the diffs to the underlying config + // implementations. + for (const auto& resource_any : resources) { + auto dummy_config = TestUtility::anyConvert(resource_any); + proto_map_[version_info] = dummy_config; + // Propagate the new config proto to all worker threads. + applyConfigUpdate([&dummy_config](ConfigProvider::ConfigConstSharedPtr prev_config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* config = const_cast(static_cast(prev_config.get())); + // Per above, append only for now. + config->addProto(dummy_config); + return prev_config; + }); + } + + ConfigSubscriptionCommonBase::onConfigUpdate(); + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; @@ -537,32 +564,12 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, }; using DeltaDummyConfigSubscriptionSharedPtr = std::shared_ptr; -class ThreadLocalDummyConfig : public ThreadLocal::ThreadLocalObject, - public Envoy::Config::ConfigProvider::Config { +class DeltaDummyDynamicConfigProvider : public Envoy::Config::MutableConfigProviderCommonBase { public: - void addProto(const test::common::config::DummyConfig& config_proto) { - protos_.push_back(config_proto); - } - - uint32_t numProtos() const { return protos_.size(); } - -private: - std::vector protos_; -}; - -class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfigProviderBase { -public: - DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - std::shared_ptr dummy_config) - : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), + DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())) { - initialize([&dummy_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return (dummy_config != nullptr) ? dummy_config : std::make_shared(); - }); - } + MutableConfigProviderCommonBase::subscription_.get())) {} DeltaDummyConfigSubscription& subscription() { return *subscription_; } @@ -574,23 +581,12 @@ class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfig } return proto_vector; } + std::string getConfigVersion() const override { return (subscription_->configInfo().has_value()) ? subscription_->configInfo().value().last_config_version_ : ""; } - ConfigConstSharedPtr getConfig() const override { - return std::dynamic_pointer_cast(tls_->get()); - } - - // Envoy::Config::DeltaMutableConfigProviderBase - ConfigSharedPtr getConfig() override { - return std::dynamic_pointer_cast(tls_->get()); - } - - std::shared_ptr getThreadLocalDummyConfig() { - return std::dynamic_pointer_cast(tls_->get()); - } private: DeltaDummyConfigSubscription* subscription_; @@ -632,6 +628,7 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { const std::string&, const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DeltaDummyConfigSubscriptionSharedPtr subscription = + getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, @@ -642,44 +639,17 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { static_cast(config_provider_manager)); }); - auto* existing_provider = static_cast( - subscription->getAnyBoundMutableConfigProvider()); - return std::make_unique( - std::move(subscription), factory_context, - (existing_provider != nullptr) ? existing_provider->getThreadLocalDummyConfig() : nullptr); + return std::make_unique(std::move(subscription)); } }; DeltaDummyConfigSubscription::DeltaDummyConfigSubscription( const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DeltaDummyConfigProviderManager& config_provider_manager) - : DeltaConfigSubscriptionInstance( - "Dummy", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()) {} - -void DeltaDummyConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) { - if (resources.empty()) { - return; - } - - // For simplicity, there is no logic here to track updates and/or removals to the existing config - // proto set (i.e., this is append only). Real xDS APIs will need to track additions, updates and - // removals to the config set and apply the diffs to the underlying config implementations. - for (const auto& resource_any : resources) { - auto dummy_config = TestUtility::anyConvert(resource_any); - proto_map_[version_info] = dummy_config; - // Propagate the new config proto to all worker threads. - applyDeltaConfigUpdate([&dummy_config](const ConfigSharedPtr& config) { - auto* thread_local_dummy_config = static_cast(config.get()); - // Per above, append only for now. - thread_local_dummy_config->addProto(dummy_config); - }); - } - - ConfigSubscriptionCommonBase::onConfigUpdate(); - setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + : DeltaConfigSubscriptionInstance("Dummy", manager_identifier, config_provider_manager, + factory_context) { + initialize( + []() -> ConfigProvider::ConfigConstSharedPtr { return std::make_shared(); }); } class DeltaConfigProviderImplTest : public testing::Test { @@ -721,7 +691,7 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { config_source_proto, factory_context_, "dummy_prefix", ConfigProviderManager::NullOptionalArg()); - // Providers, config implementations (i.e., the ThreadLocalDummyConfig) and config protos are + // Providers, config implementations (i.e., the DummyConfig) and config protos are // expected to be shared for a given subscription. EXPECT_EQ(&dynamic_cast(*provider1).subscription(), &dynamic_cast(*provider2).subscription()); @@ -729,17 +699,17 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { EXPECT_EQ( provider1->configProtoInfoVector().value().config_protos_, provider2->configProtoInfoVector().value().config_protos_); - EXPECT_EQ(provider1->config().get(), - provider2->config().get()); + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); // Validate that the config protos are propagated to the thread local config implementation. - EXPECT_EQ(provider1->config()->numProtos(), 2); + EXPECT_EQ(provider1->config()->numProtos(), 2); // Issue a second config update to validate that having multiple providers bound to the // subscription causes a single update to the underlying shared config implementation. subscription.onConfigUpdate(untyped_dummy_configs, "2"); // NOTE: the config implementation is append only and _does not_ track updates/removals to the // config proto set, so the expectation is to double the size of the set. - EXPECT_EQ(provider1->config()->numProtos(), 4); + EXPECT_EQ(provider1->config()->numProtos(), 4); EXPECT_EQ(provider1->configProtoInfoVector().value().version_, "2"); } diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index c5bee0ccb920..4a3e3099e0c1 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -47,12 +47,5 @@ MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { MockGrpcMuxCallbacks::~MockGrpcMuxCallbacks() = default; -MockMutableConfigProviderBase::MockMutableConfigProviderBase( - std::shared_ptr&& subscription, - ConfigProvider::ConfigConstSharedPtr, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full) { - subscription_->bindConfigProvider(this); -} - } // namespace Config } // namespace Envoy diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 8396fff4fc31..fbcd624a1e1a 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -109,20 +109,6 @@ class MockGrpcStreamCallbacks : public GrpcStreamCallbacks&& subscription, - ConfigProvider::ConfigConstSharedPtr initial_config, - Server::Configuration::FactoryContext& factory_context); - - MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); - MOCK_METHOD1(onConfigProtoUpdate, ConfigConstSharedPtr(const Protobuf::Message& config_proto)); - MOCK_METHOD1(initialize, void(const ConfigConstSharedPtr& initial_config)); - MOCK_METHOD1(onConfigUpdate, void(const ConfigConstSharedPtr& config)); - - ConfigSubscriptionCommonBase& subscription() { return *subscription_.get(); } -}; - class MockConfigProviderManager : public ConfigProviderManager { public: MockConfigProviderManager() = default; From 7dea1a079610b5483fd76c93951a470db046c9d7 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 24 Jul 2019 09:39:35 -0400 Subject: [PATCH 22/53] some renames and stale comment fix Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.h | 2 +- source/common/router/scoped_config_impl.cc | 7 +++---- source/common/router/scoped_config_impl.h | 4 ++-- source/common/router/scoped_rds.cc | 12 ++++++------ test/common/config/config_provider_impl_test.cc | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index de914b0df7df..d60b4cd83e80 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -334,7 +334,7 @@ class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { /** * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config - * providers (see MutableConfigProviderBase for example). + * providers. * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index ee3ce1318568..f5c8007f9e74 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -104,12 +104,11 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { return std::make_unique(std::move(key)); } -void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} +void ScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} -void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} +void ScopedConfigImpl::removeRoutingScope(const std::string&) {} -Router::ConfigConstSharedPtr -ThreadLocalScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { +Router::ConfigConstSharedPtr ScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { return std::make_shared(); } diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 9f7a02b75f99..184929f94664 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -141,9 +141,9 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { * ConnectionManagerImpl::refreshCachedRoute() will call getRouterConfig() to obtain the * Router::ConfigConstSharedPtr to use for route selection. */ -class ThreadLocalScopedConfigImpl : public ScopedConfig { +class ScopedConfigImpl : public ScopedConfig { public: - ThreadLocalScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) + ScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) : scope_key_builder_(std::move(scope_key_builder)) {} void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9aa94a915a29..e30ea4296b7c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -68,7 +68,7 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ConfigProviderInstanceType::Inline, ConfigProvider::ApiType::Delta), name_(std::move(name)), - config_(std::make_shared(std::move(scope_key_builder))), + config_(std::make_shared(std::move(scope_key_builder))), config_protos_(std::make_move_iterator(config_protos.begin()), std::make_move_iterator(config_protos.end())), rds_config_source_(std::move(rds_config_source)) {} @@ -94,7 +94,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( *scope_, *this); initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { - return std::make_shared( + return std::make_shared( envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( scope_key_builder)); }); @@ -133,8 +133,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) -> ConfigProvider::ConfigConstSharedPtr { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); return config; }); @@ -147,8 +147,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) -> ConfigProvider::ConfigConstSharedPtr { // In place update. - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); thread_local_scoped_config->removeRoutingScope(scoped_route_name); return config; }); diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index a7d631188a55..b887c3787b83 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -318,7 +318,7 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { .size()); } -// A ConfigProviderManager that returns a mock ConfigProvider. +// A ConfigProviderManager that returns a dummy ConfigProvider. class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderManager { public: DummyConfigProviderManagerMockConfigProvider(Server::Admin& admin) From c354a1e32b879f8df9ed4ce07ccec1b7fbe8d2d9 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 25 Jul 2019 09:54:08 -0400 Subject: [PATCH 23/53] merge PR #7704, add init_manager injection when creating RdsRouteConfigProvider Signed-off-by: Xin Zhuang --- .../router/route_config_provider_manager.h | 5 +- source/common/config/config_provider_impl.cc | 70 +--- source/common/config/config_provider_impl.h | 298 +++++++----------- source/common/router/BUILD | 3 + source/common/router/rds_impl.cc | 12 +- source/common/router/rds_impl.h | 4 +- source/common/router/scoped_config_impl.cc | 6 +- source/common/router/scoped_config_impl.h | 5 +- source/common/router/scoped_rds.cc | 119 ++++--- source/common/router/scoped_rds.h | 33 +- .../config/config_provider_impl_test.cc | 216 ++++++------- test/common/router/rds_impl_test.cc | 8 +- test/common/router/scoped_config_impl_test.cc | 14 +- test/common/router/scoped_rds_test.cc | 6 +- test/mocks/config/mocks.cc | 7 - test/mocks/config/mocks.h | 14 - test/mocks/router/mocks.h | 4 +- 17 files changed, 321 insertions(+), 503 deletions(-) diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index ffd9922bd1a7..9912aa460356 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -33,10 +33,13 @@ class RouteConfigProviderManager { * @param rds supplies the proto configuration of an RDS-configured RouteConfigProvider. * @param factory_context is the context to use for the route config provider. * @param stat_prefix supplies the stat_prefix to use for the provider stats. + * @param init_manager the Init::Manager used to coordinate initialization of a the underlying RDS + * subscription. */ virtual RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) PURE; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) PURE; /** * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index 61a0f1d7c7e1..8deac8cae62e 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -23,24 +23,6 @@ ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } - -void ConfigSubscriptionCommonBase::bindConfigProvider(MutableConfigProviderCommonBase* provider) { - // All config providers bound to a ConfigSubscriptionCommonBase must be of the same concrete - // type; this is assumed by ConfigSubscriptionInstance::checkAndApplyConfigUpdate() and is - // verified by the assertion below. NOTE: an inlined statement ASSERT() triggers a potentially - // evaluated expression warning from clang due to `typeid(**mutable_config_providers_.begin())`. - // To avoid this, we use a lambda to separate the first mutable provider dereference from the - // typeid() statement. - ASSERT([&]() { - if (!mutable_config_providers_.empty()) { - const auto& first_provider = **mutable_config_providers_.begin(); - return typeid(*provider) == typeid(first_provider); - } - return true; - }()); - mutable_config_providers_.insert(provider); -} - bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info) { @@ -55,58 +37,12 @@ bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Messa config_info_ = {new_hash, version_info}; ENVOY_LOG(debug, "{}: loading new configuration: config_name={} hash={}", name_, config_name, new_hash); - - ASSERT(!mutable_config_providers_.empty()); - ConfigProvider::ConfigConstSharedPtr new_config; - for (auto* provider : mutable_config_providers_) { - // All bound mutable config providers must be of the same type (see the ASSERT... in - // bindConfigProvider()). - // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config - // impl, which can then be passed to all providers. - auto* typed_provider = static_cast(provider); - if (new_config == nullptr) { - if ((new_config = typed_provider->onConfigProtoUpdate(config_proto)) == nullptr) { - return false; - } - } - typed_provider->onConfigUpdate(new_config); - } - + auto new_config_impl = onConfigProtoUpdate(config_proto); + applyConfigUpdate([new_config_impl](ConfigProvider::ConfigConstSharedPtr) + -> ConfigProvider::ConfigConstSharedPtr { return new_config_impl; }); return true; } -void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( - const std::function& update_fn, Event::PostCb complete_cb) { - // The Config implementation is assumed to be shared across the config providers bound to this - // subscription, therefore, simply propagating the update to all worker threads for a single bound - // provider will be sufficient. - if (mutable_config_providers_.size() > 1) { - ASSERT(static_cast(*mutable_config_providers_.begin()) - ->getConfig() == static_cast( - *std::next(mutable_config_providers_.begin())) - ->getConfig()); - } - - // TODO(AndresGuedez): currently, the caller has to compute the differences in resources between - // DS API config updates and passes a granular update_fn() that adds/modifies/removes resources as - // needed. Such logic could be generalized as part of this framework such that this function owns - // the diffing and issues the corresponding call to add/modify/remove a resource according to a - // vector of functions passed by the caller. - // For now each config provider has its own copy of config, we need to propagate the update to - // every provider. - for (auto* provider : mutable_config_providers_) { - auto* typed_provider = static_cast(provider); - typed_provider->onConfigUpdate( - [update_fn, typed_provider]() { - // Note: this lambda is run on every worker thread, getting the config from within the - // lambda ensures us getting the per-worker config. - ConfigSharedPtr config = typed_provider->getConfig(); - update_fn(config); - }, - complete_cb); - } -} - ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name) { config_tracker_entry_ = diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 29c4df358ec2..d60b4cd83e80 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -20,11 +20,11 @@ namespace Envoy { namespace Config { // This file provides a set of base classes, (ImmutableConfigProviderBase, -// MutableConfigProviderCommonBase, MutableConfigProviderBase, DeltaMutableConfigProviderBase, -// ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, ConfigSubscriptionInstance, -// DeltaConfigSubscriptionInstance), conforming to the ConfigProvider/ConfigProviderManager -// interfaces, which in tandem provide a framework for implementing statically defined (i.e., -// immutable) and dynamic (mutable via subscriptions) configuration for Envoy. +// MutableConfigProviderCommonBase, ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, +// ConfigSubscriptionInstance, DeltaConfigSubscriptionInstance), conforming to the +// ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a framework for +// implementing statically defined (i.e., immutable) and dynamic (mutable via subscriptions) +// configuration for Envoy. // // The mutability property applies to the ConfigProvider itself and _not_ the underlying config // proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto @@ -58,11 +58,12 @@ namespace Config { // interface. // // For mutable (xDS) providers: -// 1) According to the API type, create a class derived from MutableConfigProviderBase or -// DeltaMutableConfigProviderBase and implement the required interface. -// 2) According to the API type, create a class derived from ConfigSubscriptionInstance or -// DeltaConfigSubscriptionInstance; this is the entity responsible for owning and managing the -// Envoy::Config::Subscription that provides the underlying config subscription. +// 1) According to the API type, create a class derived from MutableConfigProviderCommonBase and +// implement the required interface. +// 2) According to the API type, create a class derived from +// ConfigSubscriptionInstance or DeltaConfigSubscriptionInstance; this is the entity responsible +// for owning and managing the Envoy::Config::Subscription that provides the +// underlying config subscription, and the Config implemention shared by associated providers. // a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a // ConfigSubscriptionInstance child): // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the @@ -78,8 +79,8 @@ namespace Config { // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the // underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called // as well. -// - On a successful config update, applyConfigUpdate() should be called to propagate the config -// updates to all bound config providers and worker threads. +// - On a successful config update, applyConfigUpdate() should be called to propagate the +// config updates to all bound config providers and worker threads. class ConfigProviderManagerImplBase; @@ -130,12 +131,13 @@ class ImmutableConfigProviderBase : public ConfigProvider { class MutableConfigProviderCommonBase; /** - * Provides common DS API subscription functionality required by the ConfigProvider::ApiType - * specific base classes (see ConfigSubscriptionInstance and DeltaConfigSubscriptionInstance). + * Provides common DS API subscription functionality required by the ConfigProvider::ApiType. * - * To do so, this class keeps track of a set of MutableConfigProviderCommonBase instances associated - * with an underlying subscription; providers are bound/unbound as needed as they are created and - * destroyed. + * This class can not be instantiated directly; instead, it provides the foundation for + * config subscription implementations which derive from it. + * + * A subscription is supposed to be co-owned by config providers with the same config source, it's + * designed to be created/destructed on admin thread only. * * xDS config providers and subscriptions are split to avoid lifetime issues with arguments * required by the config providers. An example is the Server::Configuration::FactoryContext, which @@ -143,10 +145,10 @@ class MutableConfigProviderCommonBase; * in use (see #3960). This split enables single ownership of the config providers, while enabling * shared ownership of the underlying subscription. * - * This class can not be instantiated directly; instead, it provides the foundation for - * config subscription implementations which derive from it. */ -class ConfigSubscriptionCommonBase : protected Logger::Loggable { +class ConfigSubscriptionCommonBase + : protected Logger::Loggable, + public std::enable_shared_from_this { public: struct LastConfigInfo { absl::optional last_config_hash_; @@ -166,6 +168,10 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable& configInfo() const { return config_info_; } + ConfigProvider::ConfigConstSharedPtr getConfig() const { + return tls_->getTyped().config_; + } + /** * Must be called by derived classes when the onConfigUpdate() callback associated with the * underlying subscription is issued. @@ -184,25 +190,45 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable& update_fn, + Event::PostCb complete_cb = []() {}) { + // It is safe to call shared_from_this here as this is in main thread, and destruction of a + // ConfigSubscriptionCommonBase owner (i.e., a provider) happens in main thread as well. + auto shared_this = shared_from_this(); + tls_->runOnAllThreads( + [this, update_fn]() { + tls_->getTyped().config_ = update_fn(this->getConfig()); + }, + /*Make sure this subscription will not be teared down during the update propagation.*/ + [shared_this, complete_cb]() { complete_cb(); }); } void setLastUpdated() { last_updated_ = time_source_.systemTime(); } @@ -212,16 +238,12 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable mutable_config_providers_; absl::optional config_info_; + // This slot holds a Config implementation in each thread, which is shared between + // bound providers. + ThreadLocal::SlotPtr tls_; private: - void bindConfigProvider(MutableConfigProviderCommonBase* provider); - - void unbindConfigProvider(MutableConfigProviderCommonBase* provider) { - mutable_config_providers_.erase(provider); - } - Init::TargetImpl init_target_; const uint64_t manager_identifier_; ConfigProviderManagerImplBase& config_provider_manager_; @@ -235,28 +257,34 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggables and // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit // reference counting would be more maintainable. - friend class MutableConfigProviderCommonBase; - friend class MutableConfigProviderBase; - friend class DeltaMutableConfigProviderBase; friend class ConfigProviderManagerImplBase; - friend class MockMutableConfigProviderBase; }; using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr; /** * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs. + * A single Config instance is shared across all providers and all workers associated with this + * subscription. */ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { -protected: +public: ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager, - TimeSource& time_source, const SystemTime& last_updated, - const LocalInfo::LocalInfo& local_info) - : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, - last_updated, local_info) {} + Server::Configuration::FactoryContext& factory_context) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, + factory_context) {} - ~ConfigSubscriptionInstance() override = default; + /** + * Must be called by the derived class' constructor. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with + * the underlying subscription, shared across all providers and workers. + */ + void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) { + tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(initial_config); + }); + } /** * Determines whether a configuration proto is a new update, and if so, propagates it to all @@ -268,46 +296,52 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { */ bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info); -}; -using ConfigSharedPtr = std::shared_ptr; +protected: + /** + * Called when a new config proto is received via an xDS subscription. + * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config + * implementation that will be propagated to all mutable config providers sharing the + * subscription. + * Note that this function is called _once_ across all shared config providers per xDS + * subscription config update. + * @param config_proto supplies the configuration proto. + * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. + */ + virtual ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; +}; /** * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs. */ class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { protected: - DeltaConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager, - TimeSource& time_source, const SystemTime& last_updated, - const LocalInfo::LocalInfo& local_info) - : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, - last_updated, local_info) {} - + using ConfigSubscriptionCommonBase::ConfigSubscriptionCommonBase; ~DeltaConfigSubscriptionInstance() override = default; /** - * Propagates a config update to the config providers and worker threads associated with the - * subscription. - * - * @param update_fn the callback to run on each worker thread. - * @param complete_cb the callback to run on each worker thread. NOTE it's called each time a - * registered provider's update propagation finishes. + * Must be called by the derived class' constructor. + * @param init_cb supplies an initial Envoy::Config::ConfigProvider::Config associated with the + * underlying subscription for each worker thread. */ - void applyDeltaConfigUpdate(const std::function& update_fn, - Event::PostCb complete_cb = Event::PostCb()); + void initialize(const std::function& init_cb) { + tls_->set([init_cb](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(init_cb()); + }); + } }; /** * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config - * providers (see MutableConfigProviderBase and DeltaMutableConfigProviderBase). + * providers. * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. */ class MutableConfigProviderCommonBase : public ConfigProvider { public: - ~MutableConfigProviderCommonBase() override { subscription_->unbindConfigProvider(this); } + ~MutableConfigProviderCommonBase() override = default; // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } @@ -315,132 +349,16 @@ class MutableConfigProviderCommonBase : public ConfigProvider { protected: MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, ApiType api_type) - : tls_(factory_context.threadLocal().allocateSlot()), subscription_(subscription), - api_type_(api_type) {} - - ThreadLocal::SlotPtr tls_; - ConfigSubscriptionCommonBaseSharedPtr subscription_; - -private: - ApiType api_type_; -}; + : subscription_(subscription), api_type_(api_type) {} -/** - * Provides common mutable (dynamic) config provider functionality required by - * ConfigProvider::ApiType::Full DS APIs. - */ -class MutableConfigProviderBase : public MutableConfigProviderCommonBase { -public: // Envoy::Config::ConfigProvider - // NOTE: This is being promoted to public for internal uses to avoid an unnecessary dynamic_cast - // in the public API (ConfigProvider::config()). - ConfigConstSharedPtr getConfig() const override { - return tls_->getTyped().config_; - } - - /** - * Called when a new config proto is received via an xDS subscription. - * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config - * implementation that will be propagated to all mutable config providers sharing the - * subscription. - * Note that this function is called _once_ across all shared config providers per xDS - * subscription config update. - * @param config_proto supplies the configuration proto. - * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. - */ - virtual ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; - - /** - * Must be called by the derived class' constructor. - * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with - * the underlying subscription. - */ - void initialize(const ConfigConstSharedPtr& initial_config) { - subscription_->bindConfigProvider(this); - tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(initial_config); - }); - } - - /** - * Propagates a newly instantiated Envoy::Config::ConfigProvider::Config to all workers. - * @param config supplies the newly instantiated config. - */ - void onConfigUpdate(const ConfigConstSharedPtr& config) { - if (getConfig() == config) { - return; - } - tls_->runOnAllThreads( - [this, config]() -> void { tls_->getTyped().config_ = config; }); - } - -protected: - MutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} + ConfigConstSharedPtr getConfig() const override { return subscription_->getConfig(); } - ~MutableConfigProviderBase() override = default; + ConfigSubscriptionCommonBaseSharedPtr subscription_; private: - struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { - ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config) - : config_(std::move(initial_config)) {} - - ConfigProvider::ConfigConstSharedPtr config_; - }; -}; - -/** - * Provides common mutable (dynamic) config provider functionality required by - * ConfigProvider::ApiType::Delta DS APIs. - */ -class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { -public: - // Envoy::Config::ConfigProvider - // This promotes getConfig() to public so that internal uses can avoid an unnecessary dynamic_cast - // in the public API (ConfigProvider::config()). - ConfigConstSharedPtr getConfig() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - - /** - * Non-const overload for use within the framework. - * @return ConfigSharedPtr the config implementation associated with the provider. - */ - virtual ConfigSharedPtr getConfig() PURE; - - /** - * Propagates a delta config update to all workers. - * @param update_cb the callback to run on each worker. - * @param complete_cb the callback to run in main thread after the update propagation is done on - * every worker thread. - */ - void onConfigUpdate(Envoy::Event::PostCb update_cb, Event::PostCb complete_cb) { - if (complete_cb) { - tls_->runOnAllThreads(std::move(update_cb), std::move(complete_cb)); - } else { - tls_->runOnAllThreads(std::move(update_cb)); - } - } - -protected: - DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - - ~DeltaMutableConfigProviderBase() override = default; - - /** - * Must be called by the derived class' constructor. - * @param initializeCb supplies the initialization callback to be issued for each worker - * thread. - */ - void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { - subscription_->bindConfigProvider(this); - tls_->set(std::move(initializeCb)); - } + ApiType api_type_; }; /** diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 3a499d27fae0..d6b5ee2bf957 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -169,12 +169,15 @@ envoy_cc_library( hdrs = ["scoped_rds.h"], deps = [ ":scoped_config_lib", + "//include/envoy/config:config_provider_interface", "//include/envoy/config:subscription_interface", "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:config_provider_lib", + "//source/common/init:manager_lib", + "//source/common/init:watcher_lib", "@envoy_api//envoy/admin/v2alpha:config_dump_cc", "@envoy_api//envoy/api/v2:srds_cc", ], diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 781ce9db1521..eb40e8d5e9e1 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -30,8 +30,8 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( return route_config_provider_manager.createStaticRouteConfigProvider(config.route_config(), factory_context); case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds: - return route_config_provider_manager.createRdsRouteConfigProvider(config.rds(), factory_context, - stat_prefix); + return route_config_provider_manager.createRdsRouteConfigProvider( + config.rds(), factory_context, stat_prefix, factory_context.initManager()); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -207,8 +207,8 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& ad Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) { - + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) { // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. const uint64_t manager_identifier = MessageUtil::hash(rds); @@ -221,9 +221,7 @@ Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteCon // of simplicity. subscription.reset(new RdsRouteConfigSubscription(rds, manager_identifier, factory_context, stat_prefix, *this)); - - factory_context.initManager().add(subscription->init_target_); - + init_manager.add(subscription->init_target_); route_config_subscriptions_.insert({manager_identifier, subscription}); } else { // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index ee311b63a163..09baa58fc96d 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -194,8 +194,8 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, // RouteConfigProviderManager RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) override; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::api::v2::RouteConfiguration& route_config, diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index d324db781198..d0a27686d8f2 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -98,13 +98,13 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { return std::make_unique(std::move(key)); } -void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope( +void ScopedConfigImpl::addOrUpdateRoutingScope( const ScopedRouteInfoConstSharedPtr& scoped_route_info) { scoped_route_info_by_name_.try_emplace(scoped_route_info->scopeName(), scoped_route_info); scoped_route_info_by_key_.try_emplace(scoped_route_info->scopeKey().hash(), scoped_route_info); } -void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { +void ScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { const auto iter = scoped_route_info_by_name_.find(scope_name); if (iter != scoped_route_info_by_name_.end()) { ASSERT(scoped_route_info_by_key_.count(iter->second->scopeKey().hash()) == 1); @@ -114,7 +114,7 @@ void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string& scope_na } Router::ConfigConstSharedPtr -ThreadLocalScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { +ScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { std::unique_ptr scope_key = scope_key_builder_.computeScopeKey(headers); if (scope_key == nullptr) { return nullptr; diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 8e5fff1582f8..23cdfee0d76e 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -167,7 +167,6 @@ class ScopedRouteInfo { } } } - ~ScopedRouteInfo() = default; Router::ConfigConstSharedPtr routeConfig() const { return route_provider_->config(); } const ScopeKey& scopeKey() const { return scope_key_; } @@ -190,9 +189,9 @@ using ScopedRouteMap = std::map; * ConnectionManagerImpl::refreshCachedRoute() will call getRouterConfig() to obtain the * Router::ConfigConstSharedPtr to use for route selection. */ -class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { +class ScopedConfigImpl : public ScopedConfig { public: - ThreadLocalScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) + ScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) : scope_key_builder_(std::move(scope_key_builder)) {} void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 3be47136056b..2a850df370ec 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -8,6 +8,9 @@ #include "common/common/assert.h" #include "common/common/logger.h" #include "common/common/utility.h" +#include "common/config/resources.h" +#include "common/init/manager_impl.h" +#include "common/init/watcher_impl.h" // Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all // ConfigProvider related types for consistency. @@ -67,7 +70,7 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ConfigProviderInstanceType::Inline, ConfigProvider::ApiType::Delta), name_(std::move(name)), - config_(std::make_shared(std::move(scope_key_builder))), + config_(std::make_shared(std::move(scope_key_builder))), config_protos_(std::make_move_iterator(config_protos.begin()), std::make_move_iterator(config_protos.end())), rds_config_source_(std::move(rds_config_source)) {} @@ -75,24 +78,32 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, envoy::api::v2::core::ConfigSource rds_config_source, + RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager) - : DeltaConfigSubscriptionInstance( - "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()), - factory_context_(factory_context), name_(name), + : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, + factory_context), + factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), rds_config_source_(std::move(rds_config_source)), validation_visitor_(factory_context.messageValidationVisitor()), stat_prefix_(stat_prefix), - srds_config_provider_manager_(config_provider_manager) { + route_config_provider_manager_(route_config_provider_manager) { subscription_ = factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( scoped_rds.scoped_rds_config_source(), Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), *scope_, *this); + + initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { + return std::make_shared( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( + scope_key_builder)); + }); } void ScopedRdsConfigSubscription::onConfigUpdate( @@ -104,6 +115,16 @@ void ScopedRdsConfigSubscription::onConfigUpdate( absl::flat_hash_set unique_resource_names; envoy::config::filter::network::http_connection_manager::v2::Rds rds; rds.mutable_config_source()->MergeFrom(rds_config_source_); + + std::unique_ptr overriding_init_manager; + if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { + // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. + factory_context_.clusterManager().adsMux().pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + overriding_init_manager = + std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); + } + for (const auto& resource : added_resources) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; try { @@ -116,8 +137,11 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } rds.set_route_config_name(scoped_route_config.route_configuration_name()); ScopedRouteInfoConstSharedPtr scoped_route_info = std::make_shared( - std::move(scoped_route_config), srds_config_provider_manager_.createRouteConfigProvider( - factory_context_, rds, stat_prefix_)); + std::move(scoped_route_config), + route_config_provider_manager_.createRdsRouteConfigProvider( + rds, factory_context_, stat_prefix_, + (overriding_init_manager == nullptr ? factory_context_.initManager() + : *overriding_init_manager))); // Detect if there is key conflict between two scopes. auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); if (iter != scope_name_by_hash_.end() && iter->second != scoped_route_info->scopeName()) { @@ -127,30 +151,46 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; - applyDeltaConfigUpdate( - [scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - }); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; + }); any_applied = true; ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_info->scopeName()); } catch (const EnvoyException& e) { exception_msgs.emplace_back(fmt::format("{}", e.what())); } } + if (overriding_init_manager != nullptr) { + // For new RDS subscriptions created after listener warming up, we don't wait for them to warm + // up. + Init::WatcherImpl noop_watcher( + // Note: we just throw it away. + fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), + []() { ENVOY_LOG_MISC(trace, ""); }); + overriding_init_manager->initialize(noop_watcher); + // New subscriptions should be created, now lift the floodgate. + factory_context_.clusterManager().adsMux().resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } + for (const auto& scope_name : removed_resources) { auto iter = scoped_route_map_.find(scope_name); if (iter != scoped_route_map_.end()) { ScopedRouteInfoConstSharedPtr to_be_deleted = iter->second; scope_name_by_hash_.erase(iter->second->scopeKey().hash()); scoped_route_map_.erase(iter); - applyDeltaConfigUpdate( - [scope_name](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + applyConfigUpdate( + [scope_name]( + ConfigProvider::ConfigConstSharedPtr config) -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); thread_local_scoped_config->removeRoutingScope(scope_name); + return config; }, // We need to delete the associated RouteConfigProvider in main thread. [to_be_deleted]() { /*to_be_deleted is destructed in main thread.*/ }); @@ -212,27 +252,14 @@ void ScopedRdsConfigSubscription::onConfigUpdate( *to_remove_repeated.Add() = scoped_route.first; } onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); -} +} // namespace Router ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - envoy::api::v2::core::ConfigSource rds_config_source, - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder) - : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), - rds_config_source_(std::move(rds_config_source)) { - initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared( - envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( - scope_key_builder)); - }); -} + envoy::api::v2::core::ConfigSource rds_config_source) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), + rds_config_source_(std::move(rds_config_source)) {} -Envoy::Config::ConfigSharedPtr ScopedRdsConfigProvider::getConfig() { - return std::dynamic_pointer_cast(tls_->get()); -} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); for (const auto& element : configSubscriptions()) { @@ -274,30 +301,28 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, const ConfigProviderManager::OptionalArg& optarg) { + const auto& typed_optarg = static_cast(optarg); ScopedRdsConfigSubscriptionSharedPtr subscription = ConfigProviderManagerImplBase::getSubscription( config_source_proto, factory_context.initManager(), [&config_source_proto, &factory_context, &stat_prefix, - &optarg](const uint64_t manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager) + &typed_optarg](const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) -> Envoy::Config::ConfigSubscriptionCommonBaseSharedPtr { const auto& scoped_rds_config_source = dynamic_cast< const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( config_source_proto); return std::make_shared( - scoped_rds_config_source, manager_identifier, - static_cast(optarg) - .scoped_routes_name_, - factory_context, stat_prefix, - static_cast(optarg) - .rds_config_source_, + scoped_rds_config_source, manager_identifier, typed_optarg.scoped_routes_name_, + typed_optarg.scope_key_builder_, factory_context, stat_prefix, + typed_optarg.rds_config_source_, + static_cast(config_provider_manager) + .route_config_provider_manager(), static_cast(config_provider_manager)); }); - const auto& typed_optarg = static_cast(optarg); - return std::make_unique(std::move(subscription), factory_context, - typed_optarg.rds_config_source_, - typed_optarg.scope_key_builder_); + return std::make_unique(std::move(subscription), + typed_optarg.rds_config_source_); } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 45948d0ffbed..3ee10c05bd63 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -88,8 +88,11 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, envoy::api::v2::core::ConfigSource rds_config_source, + RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); ~ScopedRdsConfigSubscription() override = default; @@ -99,7 +102,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: - // Envoy::Config::ConfigSubscriptionCommonBase + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks @@ -109,7 +112,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) override; void onConfigUpdateFailed(const EnvoyException*) override { - ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + DeltaConfigSubscriptionInstance::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, @@ -125,37 +128,29 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio Server::Configuration::FactoryContext& factory_context_; const std::string name_; std::unique_ptr subscription_; + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder_; Stats::ScopePtr scope_; ScopedRdsStats stats_; const envoy::api::v2::core::ConfigSource rds_config_source_; ProtobufMessage::ValidationVisitor& validation_visitor_; const std::string stat_prefix_; - ScopedRoutesConfigProviderManager& srds_config_provider_manager_; + RouteConfigProviderManager& route_config_provider_manager_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; // A ConfigProvider for scoped RDS that dynamically fetches scoped routing configuration via a // subscription. -class ScopedRdsConfigProvider : public Envoy::Config::DeltaMutableConfigProviderBase { +class ScopedRdsConfigProvider : public Envoy::Config::MutableConfigProviderCommonBase { public: ScopedRdsConfigProvider(ScopedRdsConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - envoy::api::v2::core::ConfigSource rds_config_source, - const envoy::config::filter::network::http_connection_manager::v2:: - ScopedRoutes::ScopeKeyBuilder& scope_key_builder); + envoy::api::v2::core::ConfigSource rds_config_source); ScopedRdsConfigSubscription& subscription() { return *static_cast(subscription_.get()); } - // getConfig() is overloaded (const/non-const only). Make all base getConfig()s visible to avoid - // compiler warnings. - using DeltaMutableConfigProviderBase::getConfig; - - // Envoy::Config::DeltaMutableConfigProviderBase - Envoy::Config::ConfigSharedPtr getConfig() override; - private: const envoy::api::v2::core::ConfigSource rds_config_source_; }; @@ -197,14 +192,6 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa return route_config_provider_manager_; } - std::shared_ptr createRouteConfigProvider( - Server::Configuration::FactoryContext& factory_context, - const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - const std::string& stat_name) { - return route_config_provider_manager_.createRdsRouteConfigProvider(rds, factory_context, - stat_name); - } - private: RouteConfigProviderManager& route_config_provider_manager_; }; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 8dcf22c39527..4a852615552a 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -4,7 +4,6 @@ #include "common/protobuf/utility.h" #include "test/common/config/dummy_config.pb.h" -#include "test/mocks/config/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -20,6 +19,22 @@ using testing::InSequence; class DummyConfigProviderManager; +class DummyConfig : public Envoy::Config::ConfigProvider::Config { +public: + DummyConfig() {} + DummyConfig(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + void addProto(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + + uint32_t numProtos() const { return protos_.size(); } + +private: + std::vector protos_; +}; + class StaticDummyConfigProvider : public ImmutableConfigProviderBase { public: StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, @@ -48,12 +63,18 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, DummyConfigSubscription(const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager); - ~DummyConfigSubscription() override = default; // Envoy::Config::ConfigSubscriptionCommonBase void start() override {} + // Envoy::Config::ConfigSubscriptionInstance + ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config_proto) override { + return std::make_shared( + static_cast(config_proto)); + } + // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { @@ -84,33 +105,17 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, }; using DummyConfigSubscriptionSharedPtr = std::shared_ptr; -class DummyConfig : public ConfigProvider::Config { +class DummyDynamicConfigProvider : public MutableConfigProviderCommonBase { public: - DummyConfig(const test::common::config::DummyConfig&) {} -}; - -class DummyDynamicConfigProvider : public MutableConfigProviderBase { -public: - DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, - const ConfigConstSharedPtr& initial_config, - Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), + DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ApiType::Full), subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())) { - initialize(initial_config); - } + MutableConfigProviderCommonBase::subscription_.get())) {} ~DummyDynamicConfigProvider() override = default; DummyConfigSubscription& subscription() { return *subscription_; } - // Envoy::Config::MutableConfigProviderBase - ConfigProvider::ConfigConstSharedPtr - onConfigProtoUpdate(const Protobuf::Message& config) override { - return std::make_shared( - static_cast(config)); - } - // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { if (!subscription_->config_proto().has_value()) { @@ -177,14 +182,7 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { static_cast(config_provider_manager)); }); - ConfigProvider::ConfigConstSharedPtr initial_config; - const auto* provider = static_cast( - subscription->getAnyBoundMutableConfigProvider()); - if (provider) { - initial_config = provider->getConfig(); - } - return std::make_unique(std::move(subscription), initial_config, - factory_context); + return std::make_unique(std::move(subscription)); } // Envoy::Config::ConfigProviderManager @@ -204,6 +202,15 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { } }; +DummyConfigSubscription::DummyConfigSubscription( + const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager) + : ConfigSubscriptionInstance("DummyDS", manager_identifier, config_provider_manager, + factory_context) { + // Returns a null value. + initialize(nullptr); +} + StaticDummyConfigProvider::StaticDummyConfigProvider( const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, @@ -212,13 +219,6 @@ StaticDummyConfigProvider::StaticDummyConfigProvider( ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} -DummyConfigSubscription::DummyConfigSubscription( - const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, - DummyConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstance( - "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()) {} - class ConfigProviderImplTest : public testing::Test { public: void initialize() { @@ -318,7 +318,7 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { .size()); } -// A ConfigProviderManager that returns a mock ConfigProvider. +// A ConfigProviderManager that returns a dummy ConfigProvider. class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderManager { public: DummyConfigProviderManagerMockConfigProvider(Server::Admin& admin) @@ -338,15 +338,14 @@ class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderM manager_identifier, factory_context, static_cast(config_provider_manager)); }); - return std::make_unique(std::move(subscription), nullptr, - factory_context); + return std::make_unique(std::move(subscription)); } }; // Test that duplicate config updates will not trigger creation of a new ConfigProvider::Config. TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { InSequence sequence; - // This provider manager returns a MockMutableConfigProviderBase. + // This provider manager returns a DummyDynamicConfigProvider. auto provider_manager = std::make_unique(factory_context_.admin_); envoy::api::v2::core::ApiConfigSource config_source_proto; @@ -354,18 +353,22 @@ TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { ConfigProviderPtr provider = provider_manager->createXdsConfigProvider( config_source_proto, factory_context_, "dummy_prefix", ConfigProviderManager::NullOptionalArg()); - auto* typed_provider = static_cast(provider.get()); + auto* typed_provider = static_cast(provider.get()); DummyConfigSubscription& subscription = static_cast(typed_provider->subscription()); + EXPECT_EQ(subscription.getConfig(), nullptr); // First time issuing a configUpdate(). A new ConfigProvider::Config should be created. - EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(1); Protobuf::RepeatedPtrField untyped_dummy_configs; untyped_dummy_configs.Add()->PackFrom(parseDummyConfigFromYaml("a: a dynamic dummy config")); subscription.onConfigUpdate(untyped_dummy_configs, "1"); + EXPECT_NE(subscription.getConfig(), nullptr); + auto config_ptr = subscription.getConfig(); + EXPECT_EQ(typed_provider->config().get(), config_ptr.get()); // Second time issuing the configUpdate(), this time with a duplicate proto. A new // ConfigProvider::Config _should not_ be created. - EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(0); - subscription.onConfigUpdate(untyped_dummy_configs, "1"); + subscription.onConfigUpdate(untyped_dummy_configs, "2"); + EXPECT_EQ(config_ptr, subscription.getConfig()); + EXPECT_EQ(typed_provider->config().get(), config_ptr.get()); } // An empty config provider tests on base class' constructor. @@ -518,7 +521,31 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) override; + const std::string& version_info) override { + if (resources.empty()) { + return; + } + + // For simplicity, there is no logic here to track updates and/or removals to the existing + // config proto set (i.e., this is append only). Real xDS APIs will need to track additions, + // updates and removals to the config set and apply the diffs to the underlying config + // implementations. + for (const auto& resource_any : resources) { + auto dummy_config = TestUtility::anyConvert(resource_any); + proto_map_[version_info] = dummy_config; + // Propagate the new config proto to all worker threads. + applyConfigUpdate([&dummy_config](ConfigProvider::ConfigConstSharedPtr prev_config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* config = const_cast(static_cast(prev_config.get())); + // Per above, append only for now. + config->addProto(dummy_config); + return prev_config; + }); + } + + ConfigSubscriptionCommonBase::onConfigUpdate(); + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; @@ -537,32 +564,12 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, }; using DeltaDummyConfigSubscriptionSharedPtr = std::shared_ptr; -class ThreadLocalDummyConfig : public ThreadLocal::ThreadLocalObject, - public Envoy::Config::ConfigProvider::Config { -public: - void addProto(const test::common::config::DummyConfig& config_proto) { - protos_.push_back(config_proto); - } - - uint32_t numProtos() const { return protos_.size(); } - -private: - std::vector protos_; -}; - -class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfigProviderBase { +class DeltaDummyDynamicConfigProvider : public Envoy::Config::MutableConfigProviderCommonBase { public: - DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - std::shared_ptr dummy_config) - : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), + DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())) { - initialize([&dummy_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return (dummy_config != nullptr) ? dummy_config : std::make_shared(); - }); - } + MutableConfigProviderCommonBase::subscription_.get())) {} DeltaDummyConfigSubscription& subscription() { return *subscription_; } @@ -574,23 +581,12 @@ class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfig } return proto_vector; } + std::string getConfigVersion() const override { return (subscription_->configInfo().has_value()) ? subscription_->configInfo().value().last_config_version_ : ""; } - ConfigConstSharedPtr getConfig() const override { - return std::dynamic_pointer_cast(tls_->get()); - } - - // Envoy::Config::DeltaMutableConfigProviderBase - ConfigSharedPtr getConfig() override { - return std::dynamic_pointer_cast(tls_->get()); - } - - std::shared_ptr getThreadLocalDummyConfig() { - return std::dynamic_pointer_cast(tls_->get()); - } private: DeltaDummyConfigSubscription* subscription_; @@ -632,6 +628,7 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { const std::string&, const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DeltaDummyConfigSubscriptionSharedPtr subscription = + getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, @@ -642,44 +639,17 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { static_cast(config_provider_manager)); }); - auto* existing_provider = static_cast( - subscription->getAnyBoundMutableConfigProvider()); - return std::make_unique( - std::move(subscription), factory_context, - (existing_provider != nullptr) ? existing_provider->getThreadLocalDummyConfig() : nullptr); + return std::make_unique(std::move(subscription)); } }; DeltaDummyConfigSubscription::DeltaDummyConfigSubscription( const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DeltaDummyConfigProviderManager& config_provider_manager) - : DeltaConfigSubscriptionInstance( - "Dummy", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()) {} - -void DeltaDummyConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) { - if (resources.empty()) { - return; - } - - // For simplicity, there is no logic here to track updates and/or removals to the existing config - // proto set (i.e., this is append only). Real xDS APIs will need to track additions, updates and - // removals to the config set and apply the diffs to the underlying config implementations. - for (const auto& resource_any : resources) { - auto dummy_config = TestUtility::anyConvert(resource_any); - proto_map_[version_info] = dummy_config; - // Propagate the new config proto to all worker threads. - applyDeltaConfigUpdate([&dummy_config](const ConfigSharedPtr& config) { - auto* thread_local_dummy_config = static_cast(config.get()); - // Per above, append only for now. - thread_local_dummy_config->addProto(dummy_config); - }); - } - - ConfigSubscriptionCommonBase::onConfigUpdate(); - setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + : DeltaConfigSubscriptionInstance("Dummy", manager_identifier, config_provider_manager, + factory_context) { + initialize( + []() -> ConfigProvider::ConfigConstSharedPtr { return std::make_shared(); }); } class DeltaConfigProviderImplTest : public testing::Test { @@ -721,7 +691,7 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { config_source_proto, factory_context_, "dummy_prefix", ConfigProviderManager::NullOptionalArg()); - // Providers, config implementations (i.e., the ThreadLocalDummyConfig) and config protos are + // Providers, config implementations (i.e., the DummyConfig) and config protos are // expected to be shared for a given subscription. EXPECT_EQ(&dynamic_cast(*provider1).subscription(), &dynamic_cast(*provider2).subscription()); @@ -729,18 +699,20 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { EXPECT_EQ( provider1->configProtoInfoVector().value().config_protos_, provider2->configProtoInfoVector().value().config_protos_); - EXPECT_EQ(provider1->config().get(), - provider2->config().get()); + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); // Validate that the config protos are propagated to the thread local config implementation. - EXPECT_EQ(provider1->config()->numProtos(), 2); + EXPECT_EQ(provider1->config()->numProtos(), 2); // Issue a second config update to validate that having multiple providers bound to the // subscription causes a single update to the underlying shared config implementation. subscription.onConfigUpdate(untyped_dummy_configs, "2"); - // NOTE: the two providers share the same config, each config update propagation would add 2 - // protos to the same config's proto vector, so the expectation is 6 here. - EXPECT_EQ(provider1->config()->numProtos(), 6); - EXPECT_EQ(provider1->configProtoInfoVector().value().version_, + // NOTE: the config implementation is append only and _does not_ track updates/removals to the + // config proto set, so the expectation is to double the size of the set. + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); + EXPECT_EQ(provider1->config()->numProtos(), 4); + EXPECT_EQ(provider2->configProtoInfoVector().value().version_, "2"); } diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index daa93436b370..34b68537c6f4 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -265,8 +265,8 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { // Get a RouteConfigProvider. This one should create an entry in the RouteConfigProviderManager. rds_.set_route_config_name("foo_route_config"); rds_.mutable_config_source()->set_path("foo_path"); - provider_ = route_config_provider_manager_->createRdsRouteConfigProvider(rds_, factory_context_, - "foo_prefix."); + provider_ = route_config_provider_manager_->createRdsRouteConfigProvider( + rds_, factory_context_, "foo_prefix.", factory_context_.initManager()); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -416,7 +416,7 @@ name: foo_route_config "1"); RouteConfigProviderPtr provider2 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix"); + rds_, factory_context_, "foo_prefix", factory_context_.initManager()); // provider2 should have route config immediately after create EXPECT_TRUE(provider2->configInfo().has_value()); @@ -430,7 +430,7 @@ name: foo_route_config rds2.set_route_config_name("foo_route_config"); rds2.mutable_config_source()->set_path("bar_path"); RouteConfigProviderPtr provider3 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds2, factory_context_, "foo_prefix"); + rds2, factory_context_, "foo_prefix", factory_context_.initManager()); EXPECT_NE(provider3, provider_); factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(route_configs, "provider3"); diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index b4bfe4f434a3..5e0f20ce644a 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -394,7 +394,7 @@ TEST_F(ScopedRouteInfoDeathTest, AssertFailure) { "RouteConfigProvider's name 'bar_route' doesn't match route_configuration_name 'foo_route'."); } -class ThreadLocalScopedConfigImplTest : public testing::Test { +class ScopedConfigImplTest : public testing::Test { public: void SetUp() override { std::string yaml_plain = R"EOF( @@ -445,12 +445,11 @@ class ThreadLocalScopedConfigImplTest : public testing::Test { std::shared_ptr scope_info_a_; std::shared_ptr scope_info_b_; ScopedRoutes::ScopeKeyBuilder key_builder_config_; - std::unique_ptr scoped_config_impl_; + std::unique_ptr scoped_config_impl_; }; -TEST_F(ThreadLocalScopedConfigImplTest, PickRoute) { - scoped_config_impl_ = - std::make_unique(std::move(key_builder_config_)); +TEST_F(ScopedConfigImplTest, PickRoute) { + scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); @@ -476,9 +475,8 @@ TEST_F(ThreadLocalScopedConfigImplTest, PickRoute) { EXPECT_EQ(route_config, nullptr); } -TEST_F(ThreadLocalScopedConfigImplTest, Update) { - scoped_config_impl_ = - std::make_unique(std::move(key_builder_config_)); +TEST_F(ScopedConfigImplTest, Update) { + scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); TestHeaderMapImpl headers{ {"foo_header", ",,key=value,bar=foo,"}, diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 323100f264fb..a5b767f8d873 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -53,11 +53,11 @@ class ScopedRoutesTestBase : public testing::Test { EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); config_provider_manager_ = std::make_unique( factory_context_.admin_, route_config_provider_manager_); - EXPECT_CALL(route_config_provider_manager_, createRdsRouteConfigProvider(_, _, _)) + EXPECT_CALL(route_config_provider_manager_, createRdsRouteConfigProvider(_, _, _, _)) .WillRepeatedly(Invoke( [this](const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext&, - const std::string&) -> RouteConfigProviderPtr { + Server::Configuration::FactoryContext&, const std::string&, + Init::Manager&) -> RouteConfigProviderPtr { auto iter = cached_route_configs_.find(rds.route_config_name()); if (iter == cached_route_configs_.end()) { cached_route_configs_[rds.route_config_name()] = std::make_shared(); diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index c5bee0ccb920..4a3e3099e0c1 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -47,12 +47,5 @@ MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { MockGrpcMuxCallbacks::~MockGrpcMuxCallbacks() = default; -MockMutableConfigProviderBase::MockMutableConfigProviderBase( - std::shared_ptr&& subscription, - ConfigProvider::ConfigConstSharedPtr, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full) { - subscription_->bindConfigProvider(this); -} - } // namespace Config } // namespace Envoy diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 8396fff4fc31..fbcd624a1e1a 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -109,20 +109,6 @@ class MockGrpcStreamCallbacks : public GrpcStreamCallbacks&& subscription, - ConfigProvider::ConfigConstSharedPtr initial_config, - Server::Configuration::FactoryContext& factory_context); - - MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); - MOCK_METHOD1(onConfigProtoUpdate, ConfigConstSharedPtr(const Protobuf::Message& config_proto)); - MOCK_METHOD1(initialize, void(const ConfigConstSharedPtr& initial_config)); - MOCK_METHOD1(onConfigUpdate, void(const ConfigConstSharedPtr& config)); - - ConfigSubscriptionCommonBase& subscription() { return *subscription_.get(); } -}; - class MockConfigProviderManager : public ConfigProviderManager { public: MockConfigProviderManager() = default; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 0c9113c13788..ffd1d375f86f 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -400,11 +400,11 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { MockRouteConfigProviderManager(); ~MockRouteConfigProviderManager(); - MOCK_METHOD3(createRdsRouteConfigProvider, + MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix)); + const std::string& stat_prefix, Init::Manager& init_manager)); MOCK_METHOD2(createStaticRouteConfigProvider, RouteConfigProviderPtr(const envoy::api::v2::RouteConfiguration& route_config, Server::Configuration::FactoryContext& factory_context)); From 9dd42146af3116b87e5ff3ccfe3d03075ba41e66 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 29 Jul 2019 18:11:10 -0400 Subject: [PATCH 24/53] fix a bug in update-propagation: new version should overwrite old version in per-thread config impl, and make sure the old copy always gets destructed in main thread Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 4 +-- source/common/router/scoped_config_impl.h | 4 +-- source/common/router/scoped_rds.cc | 30 ++++++++++++++-------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index d0a27686d8f2..392b83cb3574 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -100,8 +100,8 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { void ScopedConfigImpl::addOrUpdateRoutingScope( const ScopedRouteInfoConstSharedPtr& scoped_route_info) { - scoped_route_info_by_name_.try_emplace(scoped_route_info->scopeName(), scoped_route_info); - scoped_route_info_by_key_.try_emplace(scoped_route_info->scopeKey().hash(), scoped_route_info); + scoped_route_info_by_name_[scoped_route_info->scopeName()] = scoped_route_info; + scoped_route_info_by_key_[scoped_route_info->scopeKey().hash()] = scoped_route_info; } void ScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 23cdfee0d76e..a049cbb17bd5 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -203,9 +203,9 @@ class ScopedConfigImpl : public ScopedConfig { private: ScopeKeyBuilderImpl scope_key_builder_; // From scope name to cached ScopedRouteInfo. - absl::flat_hash_map scoped_route_info_by_name_; + absl::flat_hash_map scoped_route_info_by_name_; // Hash by ScopeKey hash to lookup in constant time. - absl::flat_hash_map scoped_route_info_by_key_; + absl::flat_hash_map scoped_route_info_by_key_; }; /** diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 2a850df370ec..adb3a479e41c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -143,22 +143,30 @@ void ScopedRdsConfigSubscription::onConfigUpdate( (overriding_init_manager == nullptr ? factory_context_.initManager() : *overriding_init_manager))); // Detect if there is key conflict between two scopes. + ScopedRouteInfoConstSharedPtr prev_scoped_route_info; auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); - if (iter != scope_name_by_hash_.end() && iter->second != scoped_route_info->scopeName()) { - throw EnvoyException( - fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", - iter->second, scoped_route_info->scopeName())); + if (iter != scope_name_by_hash_.end()) { + if (iter->second != scoped_route_info->scopeName()) { + throw EnvoyException( + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + iter->second, scoped_route_info->scopeName())); + } + prev_scoped_route_info = scoped_route_map_[scoped_route_info->scopeName()]; } scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; - applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) - -> ConfigProvider::ConfigConstSharedPtr { - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); + applyConfigUpdate( + [scoped_route_info]( + ConfigProvider::ConfigConstSharedPtr config) -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - return config; - }); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; + }, + [prev_scoped_route_info]() { + /*Make sure previous route_config_info is destructed in main thread.*/ + }); any_applied = true; ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_info->scopeName()); } catch (const EnvoyException& e) { From 5d2f8a244e0f8577a40fe94a2634908d6517688a Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 29 Jul 2019 18:21:44 -0400 Subject: [PATCH 25/53] make a ScopeRouteInfo owns a RouteConfigProvider, as per the config provider framework refactor, we will likely not have shared RouteConfigProvider instance but shared RouteConfigSubscription/Route ConfigImpl instance instead Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.h | 4 ++-- test/common/router/scoped_config_impl_test.cc | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index a049cbb17bd5..560c90327762 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -147,7 +147,7 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { class ScopedRouteInfo { public: ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, - std::shared_ptr&& route_provider) + std::unique_ptr&& route_provider) : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); ASSERT( @@ -176,7 +176,7 @@ class ScopedRouteInfo { private: const envoy::api::v2::ScopedRouteConfiguration config_proto_; ScopeKey scope_key_; - std::shared_ptr route_provider_; + std::unique_ptr route_provider_; }; using ScopedRouteInfoConstSharedPtr = std::shared_ptr; // Ordered map for consistent config dumping. diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 5e0f20ce644a..68faf60a6356 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -359,7 +359,7 @@ class ScopedRouteInfoTest : public testing::Test { route_config_ = std::make_shared>(); route_config_->name_ = "foo_route"; - route_config_provider_ = std::make_shared(); + route_config_provider_ = absl::make_unique(); EXPECT_CALL(*route_config_provider_, config()).WillRepeatedly(Return(route_config_)); EXPECT_CALL(*route_config_provider_, configInfo()) .WillRepeatedly(Return(RouteConfigProvider::ConfigInfo{route_configuration_, ""})); @@ -369,13 +369,13 @@ class ScopedRouteInfoTest : public testing::Test { envoy::api::v2::ScopedRouteConfiguration scoped_route_config_; std::shared_ptr route_config_; std::unique_ptr info_; - std::shared_ptr route_config_provider_; + std::unique_ptr route_config_provider_; }; TEST_F(ScopedRouteInfoTest, Creation) { envoy::api::v2::ScopedRouteConfiguration config_copy = scoped_route_config_; - info_ = - std::make_unique(std::move(scoped_route_config_), route_config_provider_); + info_ = std::make_unique(std::move(scoped_route_config_), + std::move(route_config_provider_)); EXPECT_EQ(info_->routeConfig().get(), route_config_.get()); EXPECT_TRUE(TestUtility::protoEqual(info_->configProto(), config_copy)); EXPECT_EQ(info_->scopeName(), "foo_scope"); @@ -390,7 +390,7 @@ TEST_F(ScopedRouteInfoDeathTest, AssertFailure) { route_config_->name_ = "bar_route"; EXPECT_DEBUG_DEATH( - ScopedRouteInfo(std::move(scoped_route_config_), route_config_provider_), + ScopedRouteInfo(std::move(scoped_route_config_), std::move(route_config_provider_)), "RouteConfigProvider's name 'bar_route' doesn't match route_configuration_name 'foo_route'."); } @@ -436,10 +436,11 @@ class ScopedConfigImplTest : public testing::Test { std::shared_ptr route_config = std::make_shared>(); route_config->name_ = scoped_route_config.route_configuration_name(); - std::shared_ptr route_config_provider = - std::make_shared>(); + std::unique_ptr route_config_provider = + absl::make_unique>(); EXPECT_CALL(*route_config_provider, config()).WillRepeatedly(Return(route_config)); - return std::make_shared(std::move(scoped_route_config), route_config_provider); + return std::make_shared(std::move(scoped_route_config), + std::move(route_config_provider)); } std::shared_ptr scope_info_a_; From 86c455e94b70f6a228a6bc85862f847bedbc8584 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 29 Jul 2019 23:45:04 -0400 Subject: [PATCH 26/53] fixes for review feedbacks Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.cc | 2 +- source/common/config/config_provider_impl.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index 8deac8cae62e..11cbf993e51c 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -37,7 +37,7 @@ bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Messa config_info_ = {new_hash, version_info}; ENVOY_LOG(debug, "{}: loading new configuration: config_name={} hash={}", name_, config_name, new_hash); - auto new_config_impl = onConfigProtoUpdate(config_proto); + ConfigProvider::ConfigConstSharedPtr new_config_impl = onConfigProtoUpdate(config_proto); applyConfigUpdate([new_config_impl](ConfigProvider::ConfigConstSharedPtr) -> ConfigProvider::ConfigConstSharedPtr { return new_config_impl; }); return true; diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index d60b4cd83e80..5e808aff067f 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -136,7 +136,7 @@ class MutableConfigProviderCommonBase; * This class can not be instantiated directly; instead, it provides the foundation for * config subscription implementations which derive from it. * - * A subscription is supposed to be co-owned by config providers with the same config source, it's + * A subscription is intended to be co-owned by config providers with the same config source, it's * designed to be created/destructed on admin thread only. * * xDS config providers and subscriptions are split to avoid lifetime issues with arguments @@ -239,8 +239,8 @@ class ConfigSubscriptionCommonBase const std::string name_; absl::optional config_info_; - // This slot holds a Config implementation in each thread, which is shared between - // bound providers. + // This slot holds a Config implementation in each thread, which is intended to be shared between + // config providers from the same config source. ThreadLocal::SlotPtr tls_; private: From 60dd4336b3fab95dee4617d28281e87c36e3af27 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 30 Jul 2019 09:53:50 -0400 Subject: [PATCH 27/53] fix comment Signed-off-by: Xin Zhuang --- test/common/config/config_provider_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index b887c3787b83..f75e1b06dadc 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -207,7 +207,7 @@ DummyConfigSubscription::DummyConfigSubscription( DummyConfigProviderManager& config_provider_manager) : ConfigSubscriptionInstance("DummyDS", manager_identifier, config_provider_manager, factory_context) { - // Returns a null value. + // A nullptr is shared as the initial value. initialize(nullptr); } From 6c1a6722a5602b6ead12e08ec386ae98fbd667e4 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 30 Jul 2019 10:18:37 -0400 Subject: [PATCH 28/53] snowp comments Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.h | 7 +++++-- test/common/config/config_provider_impl_test.cc | 8 +++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 5e808aff067f..561170388eed 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -192,7 +192,7 @@ class ConfigSubscriptionCommonBase protected: struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { - ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config) + explicit ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config) : config_(std::move(initial_config)) {} ConfigProvider::ConfigConstSharedPtr config_; @@ -227,7 +227,10 @@ class ConfigSubscriptionCommonBase [this, update_fn]() { tls_->getTyped().config_ = update_fn(this->getConfig()); }, - /*Make sure this subscription will not be teared down during the update propagation.*/ + /*During the update propagation, a subscription may get teared down in main thread due to + all owners/providers destructed in a xDS update (e.g. LDS demolishes RouteConfigProvider and + its subscription). Hold a reference to the shared subscription instance to make sure the + update can be safely pushed to workers in such an event.*/ [shared_this, complete_cb]() { complete_cb(); }); } diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index f75e1b06dadc..46a614c1f23d 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -21,8 +21,7 @@ class DummyConfigProviderManager; class DummyConfig : public Envoy::Config::ConfigProvider::Config { public: - DummyConfig() {} - DummyConfig(const test::common::config::DummyConfig& config_proto) { + explicit DummyConfig(const test::common::config::DummyConfig& config_proto) { protos_.push_back(config_proto); } void addProto(const test::common::config::DummyConfig& config_proto) { @@ -107,7 +106,7 @@ using DummyConfigSubscriptionSharedPtr = std::shared_ptr( MutableConfigProviderCommonBase::subscription_.get())) {} @@ -132,7 +131,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderCommonBase { class DummyConfigProviderManager : public ConfigProviderManagerImplBase { public: - DummyConfigProviderManager(Server::Admin& admin) + explicit DummyConfigProviderManager(Server::Admin& admin) : ConfigProviderManagerImplBase(admin, "dummy") {} ~DummyConfigProviderManager() override = default; @@ -628,7 +627,6 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { const std::string&, const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DeltaDummyConfigSubscriptionSharedPtr subscription = - getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, From 5f357bd67bd13996914a783ea5557ef39b365f85 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 30 Jul 2019 12:06:49 -0400 Subject: [PATCH 29/53] some cleanup around noop watcher Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index c323c7d4effe..ca1ac710a6ef 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -179,9 +179,9 @@ void ScopedRdsConfigSubscription::onConfigUpdate( Init::WatcherImpl noop_watcher( // Note: we just throw it away. fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), - []() { ENVOY_LOG_MISC(trace, ""); }); + []() { /*Do nothing.*/ }); overriding_init_manager->initialize(noop_watcher); - // New subscriptions should be created, now lift the floodgate. + // New RDS subscriptions should be created, now lift the floodgate. factory_context_.clusterManager().adsMux().resume( Envoy::Config::TypeUrl::get().RouteConfiguration); } From dd3b97d859b4f5478b6abcb88cb3ad7ac63b2a22 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 30 Jul 2019 12:17:09 -0400 Subject: [PATCH 30/53] fix-format Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 68faf60a6356..77fba4fd02a5 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -359,7 +359,7 @@ class ScopedRouteInfoTest : public testing::Test { route_config_ = std::make_shared>(); route_config_->name_ = "foo_route"; - route_config_provider_ = absl::make_unique(); + route_config_provider_ = std::make_unique(); EXPECT_CALL(*route_config_provider_, config()).WillRepeatedly(Return(route_config_)); EXPECT_CALL(*route_config_provider_, configInfo()) .WillRepeatedly(Return(RouteConfigProvider::ConfigInfo{route_configuration_, ""})); @@ -437,7 +437,7 @@ class ScopedConfigImplTest : public testing::Test { route_config->name_ = scoped_route_config.route_configuration_name(); std::unique_ptr route_config_provider = - absl::make_unique>(); + std::make_unique>(); EXPECT_CALL(*route_config_provider, config()).WillRepeatedly(Return(route_config)); return std::make_shared(std::move(scoped_route_config), std::move(route_config_provider)); From ee09a5b1e87c1f47ca57248809c7dc1ba28b38f7 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 1 Aug 2019 09:18:12 -0400 Subject: [PATCH 31/53] add more comment Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index ca1ac710a6ef..161c07cecd68 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -116,6 +116,9 @@ void ScopedRdsConfigSubscription::onConfigUpdate( envoy::config::filter::network::http_connection_manager::v2::Rds rds; rds.mutable_config_source()->MergeFrom(rds_config_source_); + // If new route config sources come after the factory_context_.initManager()'s initialize() been + // called, that initManager can't accepct new targets. Instead we use a local override which will + // start new subscriptions but not wait on them to be ready (to not block on main thread). std::unique_ptr overriding_init_manager; if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. From 46ad9afb09fcce7fd01968a47293aa4fb4777253 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 1 Aug 2019 09:44:48 -0400 Subject: [PATCH 32/53] fix typo, add back missing constructor in config_provider_test Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 2 +- test/common/config/config_provider_impl_test.cc | 1 + test/mocks/router/mocks.cc | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 161c07cecd68..b2585fa564af 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -117,7 +117,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( rds.mutable_config_source()->MergeFrom(rds_config_source_); // If new route config sources come after the factory_context_.initManager()'s initialize() been - // called, that initManager can't accepct new targets. Instead we use a local override which will + // called, that initManager can't accept new targets. Instead we use a local override which will // start new subscriptions but not wait on them to be ready (to not block on main thread). std::unique_ptr overriding_init_manager; if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 22f4151d2cee..8623e728de1d 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -21,6 +21,7 @@ class DummyConfigProviderManager; class DummyConfig : public Envoy::Config::ConfigProvider::Config { public: + DummyConfig() {} explicit DummyConfig(const test::common::config::DummyConfig& config_proto) { protos_.push_back(config_proto); } diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 530ac9ad7ace..ef93c0dad8b0 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -126,6 +126,7 @@ MockRouteConfigProviderManager::~MockRouteConfigProviderManager() {} MockScopedConfig::MockScopedConfig() { ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); } +MockScopedConfig::~MockScopedConfig() = default; MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() : config_(std::make_shared()) { From b1acd70ab05bad1cafdafcc21486a5e250b6dc14 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 1 Aug 2019 17:24:32 -0400 Subject: [PATCH 33/53] expect death when a null fragment is added into scopekey Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 77fba4fd02a5..88be00b9dd52 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -239,7 +239,7 @@ ScopeKey makeKey(const std::vector& parts) { TEST(ScopeKeyDeathTest, AddNullFragment) { ScopeKey key; - EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); + EXPECT_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); } TEST(ScopeKeyTest, Unmatches) { From d08bdb259746be2c5a0de132c38222747a0a21af Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 2 Aug 2019 15:24:30 -0400 Subject: [PATCH 34/53] fix per andres comments Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 4 +++ source/common/router/scoped_rds.cc | 3 ++- source/common/router/scoped_rds.h | 5 ++++ test/common/router/scoped_config_impl_test.cc | 27 +++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 392b83cb3574..fd17d338221a 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -100,6 +100,10 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { void ScopedConfigImpl::addOrUpdateRoutingScope( const ScopedRouteInfoConstSharedPtr& scoped_route_info) { + const auto iter = scoped_route_info_by_name_.find(scoped_route_info->scopeName()); + if (iter != scoped_route_info_by_name_.end()) { + scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); + } scoped_route_info_by_name_[scoped_route_info->scopeName()] = scoped_route_info; scoped_route_info_by_key_[scoped_route_info->scopeKey().hash()] = scoped_route_info; } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index b2585fa564af..7d681b460277 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -168,7 +168,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( return config; }, [prev_scoped_route_info]() { - /*Make sure previous route_config_info is destructed in main thread.*/ + /*Make sure previous route_config_info is destructed in main thread, as it holds a + * RouteConfigProvider instance which is supposed to be destructed on main thread.*/ }); any_applied = true; ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_info->scopeName()); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 3ee10c05bd63..e1653aa10f3c 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -106,6 +106,11 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks + + // NOTE: StoW form onConfigUpdate(resources, version_info) will throw an EnvoyException on any + // error and essentially reject an update. While the Delta form onConfigUpdate(added_resources, + // removed_resources, version_info) by design will partially accept correct RouteConfiguration + // from management server. void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override; void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 88be00b9dd52..315af5a3aa82 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -419,6 +419,14 @@ class ScopedConfigImplTest : public testing::Test { fragments: - string_key: foo - string_key: bar +)EOF"); + scope_info_a_v2_ = makeScopedRouteInfo(R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: meh + - string_key: meh )EOF"); scope_info_b_ = makeScopedRouteInfo(R"EOF( name: bar_scope @@ -444,6 +452,7 @@ class ScopedConfigImplTest : public testing::Test { } std::shared_ptr scope_info_a_; + std::shared_ptr scope_info_a_v2_; std::shared_ptr scope_info_b_; ScopedRoutes::ScopeKeyBuilder key_builder_config_; std::unique_ptr scoped_config_impl_; @@ -489,16 +498,34 @@ TEST_F(ScopedConfigImplTest, Update) { // Add scope_key (bar, baz). scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + TestHeaderMapImpl{{"foo_header", ",,key=v,bar=bar,"}, {"bar_header", ";val1;baz"}}), + scope_info_b_->routeConfig()); // Add scope_key (foo, bar). scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); // Found scope_info_a_. EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), scope_info_a_->routeConfig()); + // Update scope foo_scope. + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_v2_); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // foo_scope now is keyed by (meh, meh). + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + TestHeaderMapImpl{{"foo_header", ",bar=meh,foo=bar"}, {"bar_header", ";;meh"}}), + scope_info_a_v2_->routeConfig()); + // Remove scope "foo_scope". scoped_config_impl_->removeRoutingScope("foo_scope"); // scope_info_a_ is gone. EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Now delete some non-existent scopes. + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("foo_scope1")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("base_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("bluh_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("meh_scope")); } } // namespace From eb65b89d688505d2b9a05baa9422a39b273df8b8 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 2 Aug 2019 15:32:16 -0400 Subject: [PATCH 35/53] fix pedant spelling check errors. Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.h | 8 ++++---- test/common/router/scoped_config_impl_test.cc | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index e1653aa10f3c..d79dcd0f354d 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -107,10 +107,10 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio // Envoy::Config::SubscriptionCallbacks - // NOTE: StoW form onConfigUpdate(resources, version_info) will throw an EnvoyException on any - // error and essentially reject an update. While the Delta form onConfigUpdate(added_resources, - // removed_resources, version_info) by design will partially accept correct RouteConfiguration - // from management server. + // NOTE: state-of-the-world form onConfigUpdate(resources, version_info) will throw an + // EnvoyException on any error and essentially reject an update. While the Delta form + // onConfigUpdate(added_resources, removed_resources, version_info) by design will partially + // accept correct RouteConfiguration from management server. void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override; void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 315af5a3aa82..72c28580dc66 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -425,8 +425,8 @@ class ScopedConfigImplTest : public testing::Test { route_configuration_name: foo_route key: fragments: - - string_key: meh - - string_key: meh + - string_key: xyz + - string_key: xyz )EOF"); scope_info_b_ = makeScopedRouteInfo(R"EOF( name: bar_scope @@ -511,9 +511,9 @@ TEST_F(ScopedConfigImplTest, Update) { scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_v2_); EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); - // foo_scope now is keyed by (meh, meh). + // foo_scope now is keyed by (xyz, xyz). EXPECT_EQ(scoped_config_impl_->getRouteConfig( - TestHeaderMapImpl{{"foo_header", ",bar=meh,foo=bar"}, {"bar_header", ";;meh"}}), + TestHeaderMapImpl{{"foo_header", ",bar=xyz,foo=bar"}, {"bar_header", ";;xyz"}}), scope_info_a_v2_->routeConfig()); // Remove scope "foo_scope". @@ -525,7 +525,7 @@ TEST_F(ScopedConfigImplTest, Update) { EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("foo_scope1")); EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("base_scope")); EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("bluh_scope")); - EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("meh_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("xyz_scope")); } } // namespace From 07bdfee54566f936fc098af22f32d43a6cbe7e0a Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 2 Aug 2019 16:14:32 -0400 Subject: [PATCH 36/53] fix failing add null fragment assertion test. expect_death ==> expect_debug_death Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 72c28580dc66..834b38b9e39b 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -239,7 +239,7 @@ ScopeKey makeKey(const std::vector& parts) { TEST(ScopeKeyDeathTest, AddNullFragment) { ScopeKey key; - EXPECT_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); + EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); } TEST(ScopeKeyTest, Unmatches) { From 40b214e8f026bccab95b6e000afd729389e70137 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Fri, 2 Aug 2019 16:49:26 -0400 Subject: [PATCH 37/53] only death test in debug mode for add null fragment to scopeKey Signed-off-by: Xin Zhuang --- test/common/router/scoped_config_impl_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 834b38b9e39b..d37c7894a6ab 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -239,7 +239,9 @@ ScopeKey makeKey(const std::vector& parts) { TEST(ScopeKeyDeathTest, AddNullFragment) { ScopeKey key; +#if !defined(NDEBUG) EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); +#endif } TEST(ScopeKeyTest, Unmatches) { From 4e57308d16e7c29aca9f95e6d3bb0e8962a6e9a2 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 6 Aug 2019 11:56:39 -0400 Subject: [PATCH 38/53] fix clang tidy errors around router/mock.h by removing empty destructors. Signed-off-by: Xin Zhuang --- test/mocks/router/mocks.cc | 27 --------------------------- test/mocks/router/mocks.h | 22 ++-------------------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index ef93c0dad8b0..d954c6a202c4 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -16,7 +16,6 @@ namespace Envoy { namespace Router { MockDirectResponseEntry::MockDirectResponseEntry() = default; -MockDirectResponseEntry::~MockDirectResponseEntry() = default; MockRetryState::MockRetryState() = default; @@ -35,44 +34,31 @@ void MockRetryState::expectResetRetry() { .WillOnce(DoAll(SaveArg<1>(&callback_), Return(RetryStatus::Yes))); } -MockRetryState::~MockRetryState() = default; - MockRateLimitPolicyEntry::MockRateLimitPolicyEntry() { ON_CALL(*this, disableKey()).WillByDefault(ReturnRef(disable_key_)); } -MockRateLimitPolicyEntry::~MockRateLimitPolicyEntry() = default; - MockRateLimitPolicy::MockRateLimitPolicy() { ON_CALL(*this, getApplicableRateLimit(_)).WillByDefault(ReturnRef(rate_limit_policy_entry_)); ON_CALL(*this, empty()).WillByDefault(Return(true)); } -MockRateLimitPolicy::~MockRateLimitPolicy() = default; - MockShadowWriter::MockShadowWriter() = default; -MockShadowWriter::~MockShadowWriter() = default; MockVirtualHost::MockVirtualHost() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, rateLimitPolicy()).WillByDefault(ReturnRef(rate_limit_policy_)); } -MockVirtualHost::~MockVirtualHost() = default; - MockHashPolicy::MockHashPolicy() = default; -MockHashPolicy::~MockHashPolicy() = default; MockMetadataMatchCriteria::MockMetadataMatchCriteria() = default; -MockMetadataMatchCriteria::~MockMetadataMatchCriteria() = default; MockPathMatchCriterion::MockPathMatchCriterion() { ON_CALL(*this, matchType()).WillByDefault(ReturnPointee(&type_)); ON_CALL(*this, matcher()).WillByDefault(ReturnPointee(&matcher_)); } -MockPathMatchCriterion::~MockPathMatchCriterion() = default; - MockRouteEntry::MockRouteEntry() { ON_CALL(*this, clusterName()).WillByDefault(ReturnRef(cluster_name_)); ON_CALL(*this, opaqueConfig()).WillByDefault(ReturnRef(opaque_config_)); @@ -90,8 +76,6 @@ MockRouteEntry::MockRouteEntry() { ON_CALL(*this, routeName()).WillByDefault(ReturnRef(route_name_)); } -MockRouteEntry::~MockRouteEntry() = default; - MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, route(_, _)).WillByDefault(Return(route_)); ON_CALL(*this, internalOnlyHeaders()).WillByDefault(ReturnRef(internal_only_headers_)); @@ -99,34 +83,23 @@ MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, usesVhds()).WillByDefault(Return(false)); } -MockConfig::~MockConfig() = default; - MockDecorator::MockDecorator() { ON_CALL(*this, getOperation()).WillByDefault(ReturnRef(operation_)); } -MockDecorator::~MockDecorator() = default; - -MockRouteTracing::MockRouteTracing() = default; -MockRouteTracing::~MockRouteTracing() = default; MockRoute::MockRoute() { ON_CALL(*this, routeEntry()).WillByDefault(Return(&route_entry_)); ON_CALL(*this, decorator()).WillByDefault(Return(&decorator_)); ON_CALL(*this, tracingConfig()).WillByDefault(Return(nullptr)); } -MockRoute::~MockRoute() = default; MockRouteConfigProvider::MockRouteConfigProvider() { ON_CALL(*this, config()).WillByDefault(Return(route_config_)); } -MockRouteConfigProviderManager::MockRouteConfigProviderManager() {} -MockRouteConfigProviderManager::~MockRouteConfigProviderManager() {} - MockScopedConfig::MockScopedConfig() { ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); } -MockScopedConfig::~MockScopedConfig() = default; MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() : config_(std::make_shared()) { diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index b69216a9c211..e5a5874a9f4b 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -37,7 +37,6 @@ using ::testing::NiceMock; class MockDirectResponseEntry : public DirectResponseEntry { public: MockDirectResponseEntry(); - ~MockDirectResponseEntry() override; // DirectResponseEntry MOCK_CONST_METHOD2(finalizeResponseHeaders, @@ -115,7 +114,6 @@ class TestRetryPolicy : public RetryPolicy { class MockRetryState : public RetryState { public: MockRetryState(); - ~MockRetryState() override; void expectHeadersRetry(); void expectHedgedPerTryTimeoutRetry(); @@ -141,7 +139,6 @@ class MockRetryState : public RetryState { class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { public: MockRateLimitPolicyEntry(); - ~MockRateLimitPolicyEntry() override; // Router::RateLimitPolicyEntry MOCK_CONST_METHOD0(stage, uint64_t()); @@ -159,7 +156,6 @@ class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { class MockRateLimitPolicy : public RateLimitPolicy { public: MockRateLimitPolicy(); - ~MockRateLimitPolicy() override; // Router::RateLimitPolicy MOCK_CONST_METHOD1( @@ -185,7 +181,6 @@ class TestShadowPolicy : public ShadowPolicy { class MockShadowWriter : public ShadowWriter { public: MockShadowWriter(); - ~MockShadowWriter() override; // Router::ShadowWriter void shadow(const std::string& cluster, Http::MessagePtr&& request, @@ -209,7 +204,6 @@ class TestVirtualCluster : public VirtualCluster { class MockVirtualHost : public VirtualHost { public: MockVirtualHost(); - ~MockVirtualHost() override; // Router::VirtualHost MOCK_CONST_METHOD0(name, const std::string&()); @@ -236,7 +230,6 @@ class MockVirtualHost : public VirtualHost { class MockHashPolicy : public HashPolicy { public: MockHashPolicy(); - ~MockHashPolicy() override; // Router::HashPolicy MOCK_CONST_METHOD3(generateHash, @@ -248,7 +241,6 @@ class MockHashPolicy : public HashPolicy { class MockMetadataMatchCriteria : public MetadataMatchCriteria { public: MockMetadataMatchCriteria(); - ~MockMetadataMatchCriteria() override; // Router::MetadataMatchCriteria MOCK_CONST_METHOD0(metadataMatchCriteria, @@ -259,7 +251,6 @@ class MockMetadataMatchCriteria : public MetadataMatchCriteria { class MockPathMatchCriterion : public PathMatchCriterion { public: MockPathMatchCriterion(); - ~MockPathMatchCriterion() override; // Router::PathMatchCriterion MOCK_CONST_METHOD0(matchType, PathMatchType()); @@ -272,7 +263,6 @@ class MockPathMatchCriterion : public PathMatchCriterion { class MockRouteEntry : public RouteEntry { public: MockRouteEntry(); - ~MockRouteEntry() override; // Router::Config MOCK_CONST_METHOD0(clusterName, const std::string&()); @@ -329,7 +319,6 @@ class MockRouteEntry : public RouteEntry { class MockDecorator : public Decorator { public: MockDecorator(); - ~MockDecorator() override; // Router::Decorator MOCK_CONST_METHOD0(getOperation, const std::string&()); @@ -340,8 +329,7 @@ class MockDecorator : public Decorator { class MockRouteTracing : public RouteTracing { public: - MockRouteTracing(); - ~MockRouteTracing() override; + MockRouteTracing() = default; // Router::RouteTracing MOCK_CONST_METHOD0(getClientSampling, const envoy::type::FractionalPercent&()); @@ -352,7 +340,6 @@ class MockRouteTracing : public RouteTracing { class MockRoute : public Route { public: MockRoute(); - ~MockRoute() override; // Router::Route MOCK_CONST_METHOD0(directResponseEntry, const DirectResponseEntry*()); @@ -369,7 +356,6 @@ class MockRoute : public Route { class MockConfig : public Config { public: MockConfig(); - ~MockConfig() override; // Router::Config MOCK_CONST_METHOD2(route, RouteConstSharedPtr(const Http::HeaderMap&, uint64_t random_value)); @@ -385,7 +371,6 @@ class MockConfig : public Config { class MockRouteConfigProvider : public RouteConfigProvider { public: MockRouteConfigProvider(); - ~MockRouteConfigProvider() = default; MOCK_METHOD0(config, ConfigConstSharedPtr()); MOCK_CONST_METHOD0(configInfo, absl::optional()); @@ -397,8 +382,7 @@ class MockRouteConfigProvider : public RouteConfigProvider { class MockRouteConfigProviderManager : public RouteConfigProviderManager { public: - MockRouteConfigProviderManager(); - ~MockRouteConfigProviderManager() override; + MockRouteConfigProviderManager() = default; MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( @@ -413,7 +397,6 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { class MockScopedConfig : public ScopedConfig { public: MockScopedConfig(); - ~MockScopedConfig() override; MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); @@ -423,7 +406,6 @@ class MockScopedConfig : public ScopedConfig { class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { public: MockScopedRouteConfigProvider(); - ~MockScopedRouteConfigProvider() = default; // Config::ConfigProvider MOCK_CONST_METHOD0(lastUpdated, SystemTime()); From 710c7e85836570c01df43708be201fd2dd7b4b6e Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 8 Aug 2019 11:06:31 -0400 Subject: [PATCH 39/53] move empty ctors,destructors back to cc file Signed-off-by: Xin Zhuang --- test/mocks/router/mocks.cc | 29 +++++++++++++++++++++++++++++ test/mocks/router/mocks.h | 22 ++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index d954c6a202c4..291bd3248e9d 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -16,6 +16,7 @@ namespace Envoy { namespace Router { MockDirectResponseEntry::MockDirectResponseEntry() = default; +MockDirectResponseEntry::~MockDirectResponseEntry() = default; MockRetryState::MockRetryState() = default; @@ -34,31 +35,44 @@ void MockRetryState::expectResetRetry() { .WillOnce(DoAll(SaveArg<1>(&callback_), Return(RetryStatus::Yes))); } +MockRetryState::~MockRetryState() = default; + MockRateLimitPolicyEntry::MockRateLimitPolicyEntry() { ON_CALL(*this, disableKey()).WillByDefault(ReturnRef(disable_key_)); } +MockRateLimitPolicyEntry::~MockRateLimitPolicyEntry() = default; + MockRateLimitPolicy::MockRateLimitPolicy() { ON_CALL(*this, getApplicableRateLimit(_)).WillByDefault(ReturnRef(rate_limit_policy_entry_)); ON_CALL(*this, empty()).WillByDefault(Return(true)); } +MockRateLimitPolicy::~MockRateLimitPolicy() = default; + MockShadowWriter::MockShadowWriter() = default; +MockShadowWriter::~MockShadowWriter() = default; MockVirtualHost::MockVirtualHost() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, rateLimitPolicy()).WillByDefault(ReturnRef(rate_limit_policy_)); } +MockVirtualHost::~MockVirtualHost() = default; + MockHashPolicy::MockHashPolicy() = default; +MockHashPolicy::~MockHashPolicy() = default; MockMetadataMatchCriteria::MockMetadataMatchCriteria() = default; +MockMetadataMatchCriteria::~MockMetadataMatchCriteria() = default; MockPathMatchCriterion::MockPathMatchCriterion() { ON_CALL(*this, matchType()).WillByDefault(ReturnPointee(&type_)); ON_CALL(*this, matcher()).WillByDefault(ReturnPointee(&matcher_)); } +MockPathMatchCriterion::~MockPathMatchCriterion() = default; + MockRouteEntry::MockRouteEntry() { ON_CALL(*this, clusterName()).WillByDefault(ReturnRef(cluster_name_)); ON_CALL(*this, opaqueConfig()).WillByDefault(ReturnRef(opaque_config_)); @@ -76,6 +90,8 @@ MockRouteEntry::MockRouteEntry() { ON_CALL(*this, routeName()).WillByDefault(ReturnRef(route_name_)); } +MockRouteEntry::~MockRouteEntry() = default; + MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, route(_, _)).WillByDefault(Return(route_)); ON_CALL(*this, internalOnlyHeaders()).WillByDefault(ReturnRef(internal_only_headers_)); @@ -83,29 +99,42 @@ MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, usesVhds()).WillByDefault(Return(false)); } +MockConfig::~MockConfig() = default; + MockDecorator::MockDecorator() { ON_CALL(*this, getOperation()).WillByDefault(ReturnRef(operation_)); } +MockDecorator::~MockDecorator() = default; + +MockRouteTracing::MockRouteTracing() = default; +MockRouteTracing::~MockRouteTracing() = default; MockRoute::MockRoute() { ON_CALL(*this, routeEntry()).WillByDefault(Return(&route_entry_)); ON_CALL(*this, decorator()).WillByDefault(Return(&decorator_)); ON_CALL(*this, tracingConfig()).WillByDefault(Return(nullptr)); } +MockRoute::~MockRoute() = default; MockRouteConfigProvider::MockRouteConfigProvider() { ON_CALL(*this, config()).WillByDefault(Return(route_config_)); } +MockRouteConfigProvider::~MockRouteConfigProvider() = default; + +MockRouteConfigProviderManager::MockRouteConfigProviderManager() = default; +MockRouteConfigProviderManager::~MockRouteConfigProviderManager() = default; MockScopedConfig::MockScopedConfig() { ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); } +MockScopedConfig::~MockScopedConfig() = default; MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() : config_(std::make_shared()) { ON_CALL(*this, getConfig()).WillByDefault(Return(config_)); ON_CALL(*this, apiType()).WillByDefault(Return(ApiType::Delta)); } +MockScopedRouteConfigProvider::~MockScopedRouteConfigProvider() = default; } // namespace Router } // namespace Envoy diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index e5a5874a9f4b..a5ae07f1c80b 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -37,6 +37,7 @@ using ::testing::NiceMock; class MockDirectResponseEntry : public DirectResponseEntry { public: MockDirectResponseEntry(); + ~MockDirectResponseEntry() override; // DirectResponseEntry MOCK_CONST_METHOD2(finalizeResponseHeaders, @@ -114,6 +115,7 @@ class TestRetryPolicy : public RetryPolicy { class MockRetryState : public RetryState { public: MockRetryState(); + ~MockRetryState() override; void expectHeadersRetry(); void expectHedgedPerTryTimeoutRetry(); @@ -139,6 +141,7 @@ class MockRetryState : public RetryState { class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { public: MockRateLimitPolicyEntry(); + ~MockRateLimitPolicyEntry() override; // Router::RateLimitPolicyEntry MOCK_CONST_METHOD0(stage, uint64_t()); @@ -156,6 +159,7 @@ class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { class MockRateLimitPolicy : public RateLimitPolicy { public: MockRateLimitPolicy(); + ~MockRateLimitPolicy() override; // Router::RateLimitPolicy MOCK_CONST_METHOD1( @@ -181,6 +185,7 @@ class TestShadowPolicy : public ShadowPolicy { class MockShadowWriter : public ShadowWriter { public: MockShadowWriter(); + ~MockShadowWriter() override; // Router::ShadowWriter void shadow(const std::string& cluster, Http::MessagePtr&& request, @@ -204,6 +209,7 @@ class TestVirtualCluster : public VirtualCluster { class MockVirtualHost : public VirtualHost { public: MockVirtualHost(); + ~MockVirtualHost() override; // Router::VirtualHost MOCK_CONST_METHOD0(name, const std::string&()); @@ -230,6 +236,7 @@ class MockVirtualHost : public VirtualHost { class MockHashPolicy : public HashPolicy { public: MockHashPolicy(); + ~MockHashPolicy() override; // Router::HashPolicy MOCK_CONST_METHOD3(generateHash, @@ -241,6 +248,7 @@ class MockHashPolicy : public HashPolicy { class MockMetadataMatchCriteria : public MetadataMatchCriteria { public: MockMetadataMatchCriteria(); + ~MockMetadataMatchCriteria() override; // Router::MetadataMatchCriteria MOCK_CONST_METHOD0(metadataMatchCriteria, @@ -251,6 +259,7 @@ class MockMetadataMatchCriteria : public MetadataMatchCriteria { class MockPathMatchCriterion : public PathMatchCriterion { public: MockPathMatchCriterion(); + ~MockPathMatchCriterion() override; // Router::PathMatchCriterion MOCK_CONST_METHOD0(matchType, PathMatchType()); @@ -263,6 +272,7 @@ class MockPathMatchCriterion : public PathMatchCriterion { class MockRouteEntry : public RouteEntry { public: MockRouteEntry(); + ~MockRouteEntry() override; // Router::Config MOCK_CONST_METHOD0(clusterName, const std::string&()); @@ -319,6 +329,7 @@ class MockRouteEntry : public RouteEntry { class MockDecorator : public Decorator { public: MockDecorator(); + ~MockDecorator() override; // Router::Decorator MOCK_CONST_METHOD0(getOperation, const std::string&()); @@ -329,7 +340,8 @@ class MockDecorator : public Decorator { class MockRouteTracing : public RouteTracing { public: - MockRouteTracing() = default; + MockRouteTracing(); + ~MockRouteTracing() override; // Router::RouteTracing MOCK_CONST_METHOD0(getClientSampling, const envoy::type::FractionalPercent&()); @@ -340,6 +352,7 @@ class MockRouteTracing : public RouteTracing { class MockRoute : public Route { public: MockRoute(); + ~MockRoute() override; // Router::Route MOCK_CONST_METHOD0(directResponseEntry, const DirectResponseEntry*()); @@ -356,6 +369,7 @@ class MockRoute : public Route { class MockConfig : public Config { public: MockConfig(); + ~MockConfig() override; // Router::Config MOCK_CONST_METHOD2(route, RouteConstSharedPtr(const Http::HeaderMap&, uint64_t random_value)); @@ -371,6 +385,7 @@ class MockConfig : public Config { class MockRouteConfigProvider : public RouteConfigProvider { public: MockRouteConfigProvider(); + ~MockRouteConfigProvider() override; MOCK_METHOD0(config, ConfigConstSharedPtr()); MOCK_CONST_METHOD0(configInfo, absl::optional()); @@ -382,7 +397,8 @@ class MockRouteConfigProvider : public RouteConfigProvider { class MockRouteConfigProviderManager : public RouteConfigProviderManager { public: - MockRouteConfigProviderManager() = default; + MockRouteConfigProviderManager(); + ~MockRouteConfigProviderManager() override; MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( @@ -397,6 +413,7 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { class MockScopedConfig : public ScopedConfig { public: MockScopedConfig(); + ~MockScopedConfig() override; MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); @@ -406,6 +423,7 @@ class MockScopedConfig : public ScopedConfig { class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { public: MockScopedRouteConfigProvider(); + ~MockScopedRouteConfigProvider() override; // Config::ConfigProvider MOCK_CONST_METHOD0(lastUpdated, SystemTime()); From 2dd671479ed57918ea0232d792c68fb0091ae863 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 12 Aug 2019 09:30:10 -0400 Subject: [PATCH 40/53] fix a race condition, in SRDS onConfigUpdate, when calling applyConfigUpdate, the complete_cb which holds a RouteConfigInfo shared_ptr may get destructed at last, after the complete_cb is posted back to main thread dispatcher queue and run immediately, which causes an assertion failure. Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 67 +++++++++------ source/common/router/scoped_rds.h | 10 +++ test/common/router/scoped_config_impl_test.cc | 2 + test/common/router/scoped_rds_test.cc | 86 ++++++++++++------- 4 files changed, 107 insertions(+), 58 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 7d681b460277..19454be3f62b 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -86,7 +86,9 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), - factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), + factory_context_(factory_context), cleanup_timer_(factory_context.dispatcher().createTimer( + [this]() -> void { cleanupDefferedDeletes(); })), + name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), rds_config_source_(std::move(rds_config_source)), @@ -106,6 +108,20 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( }); } +void ScopedRdsConfigSubscription::cleanupDefferedDeletes() { + for (auto iter = deffered_to_be_deleted_.begin(); iter != deffered_to_be_deleted_.end();) { + if (iter->use_count() == 1) { + iter = deffered_to_be_deleted_.erase(iter); + } else { + iter++; + } + } + if (!deffered_to_be_deleted_.empty()) { + // Cleanup as fast as possible. + cleanup_timer_->enableTimer(std::chrono::milliseconds(0)); + } +} + void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, @@ -128,6 +144,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); } + std::list to_be_deleted; for (const auto& resource : added_resources) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; try { @@ -146,7 +163,6 @@ void ScopedRdsConfigSubscription::onConfigUpdate( (overriding_init_manager == nullptr ? factory_context_.initManager() : *overriding_init_manager))); // Detect if there is key conflict between two scopes. - ScopedRouteInfoConstSharedPtr prev_scoped_route_info; auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); if (iter != scope_name_by_hash_.end()) { if (iter->second != scoped_route_info->scopeName()) { @@ -154,23 +170,17 @@ void ScopedRdsConfigSubscription::onConfigUpdate( fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", iter->second, scoped_route_info->scopeName())); } - prev_scoped_route_info = scoped_route_map_[scoped_route_info->scopeName()]; + to_be_deleted.push_back(scoped_route_map_[scoped_route_info->scopeName()]); } scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; - applyConfigUpdate( - [scoped_route_info]( - ConfigProvider::ConfigConstSharedPtr config) -> ConfigProvider::ConfigConstSharedPtr { - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); - - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - return config; - }, - [prev_scoped_route_info]() { - /*Make sure previous route_config_info is destructed in main thread, as it holds a - * RouteConfigProvider instance which is supposed to be destructed on main thread.*/ - }); + applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; + }); any_applied = true; ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_info->scopeName()); } catch (const EnvoyException& e) { @@ -193,18 +203,16 @@ void ScopedRdsConfigSubscription::onConfigUpdate( for (const auto& scope_name : removed_resources) { auto iter = scoped_route_map_.find(scope_name); if (iter != scoped_route_map_.end()) { - ScopedRouteInfoConstSharedPtr to_be_deleted = iter->second; + to_be_deleted.push_back(iter->second); scope_name_by_hash_.erase(iter->second->scopeKey().hash()); scoped_route_map_.erase(iter); - applyConfigUpdate( - [scope_name]( - ConfigProvider::ConfigConstSharedPtr config) -> ConfigProvider::ConfigConstSharedPtr { - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scope_name); - return config; - }, - [to_be_deleted]() { /*to_be_deleted will be destructed in main thread.*/ }); + applyConfigUpdate([scope_name](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scope_name); + return config; + }); any_applied = true; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scope_name); } @@ -213,6 +221,11 @@ void ScopedRdsConfigSubscription::onConfigUpdate( if (any_applied) { setLastConfigInfo(absl::optional({absl::nullopt, version_info})); } + if (!to_be_deleted.empty()) { + deffered_to_be_deleted_.insert(deffered_to_be_deleted_.begin(), to_be_deleted.begin(), + to_be_deleted.end()); + cleanup_timer_->enableTimer(std::chrono::milliseconds(0)); + } stats_.config_reload_.inc(); if (!exception_msgs.empty()) { throw EnvoyException(fmt::format("Error adding/updating scoped route(s): {}", @@ -240,7 +253,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config = scope_config_inserted.first->second; - uint64_t key_fingerprint = MessageUtil::hash(scoped_route_config.key()); + const uint64_t key_fingerprint = MessageUtil::hash(scoped_route_config.key()); if (!scope_name_by_key_hash.try_emplace(key_fingerprint, scope_name).second) { throw EnvoyException( fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 890af0c2d4e6..cbe0876aecf0 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -102,6 +102,9 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: + // Cleanup stale ScopedRouteInfo instances. + void cleanupDefferedDeletes(); + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } @@ -132,6 +135,13 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio absl::flat_hash_map scope_name_by_hash_; // For creating RDS subscriptions. Server::Configuration::FactoryContext& factory_context_; + // Deferred to be deleted ScopedRouteInfo. + // As RdsRouteConfigProvider instance (which contains a tls slot) can only be destructed in main + // thread, we need to make sure the shared ScopedRouteInfo is deleted in main thread. + std::list deffered_to_be_deleted_; + Envoy::Event::TimerPtr cleanup_timer_; + const std::chrono::milliseconds cleanup_interval_{3000}; // Clean up every 3s. + const std::string name_; std::unique_ptr subscription_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index d37c7894a6ab..f85d1bea61c3 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -460,6 +460,7 @@ class ScopedConfigImplTest : public testing::Test { std::unique_ptr scoped_config_impl_; }; +// Test a ScopedConfigImpl returns the correct route Config. TEST_F(ScopedConfigImplTest, PickRoute) { scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); @@ -487,6 +488,7 @@ TEST_F(ScopedConfigImplTest, PickRoute) { EXPECT_EQ(route_config, nullptr); } +// Test a ScopedConfigImpl returns the correct route Config before and after scope config update. TEST_F(ScopedConfigImplTest, Update) { scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 26ada1095670..12f5e61c718a 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -27,6 +27,8 @@ namespace Envoy { namespace Router { namespace { +using ::Envoy::Http::TestHeaderMapImpl; + envoy::api::v2::ScopedRouteConfiguration parseScopedRouteConfigurationFromYaml(const std::string& yaml) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; @@ -125,6 +127,14 @@ name: foo_scoped_routes subscription_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; } + ScopedRdsConfigProvider* getScopedRdsProvider() const { + return dynamic_cast(provider_.get()); + } + // Helper function which returns the ScopedRouteMap of the subscription. + const ScopedRouteMap& getScopedRouteMap() const { + return getScopedRdsProvider()->subscription().scopedRouteMap(); + } + Envoy::Config::SubscriptionCallbacks* subscription_callbacks_{}; Envoy::Config::ConfigProviderPtr provider_; }; @@ -212,44 +222,54 @@ route_configuration_name: foo_routes EXPECT_EQ( 2UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); - - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .size(), - 2); + EXPECT_EQ(getScopedRouteMap().size(), 2); + + // Verify the config is a ScopedConfigImpl instance, both scopes point to foo_routes. + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + "foo_routes"); // Delete foo_scope2. resources.RemoveLast(); EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "3")); - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .size(), - 1); - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .count("foo_scope"), - 1); + EXPECT_EQ(getScopedRouteMap().size(), 1); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); EXPECT_EQ( 3UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // now scope key "x-bar-key" points to nowhere. + EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); // Delete foo_scope via Delta API. Protobuf::RepeatedPtrField deletes; *deletes.Add() = "foo_scope"; resources.RemoveLast(); EXPECT_NO_THROW( subscription_callbacks_->onConfigUpdate(anyToResource(resources, "4"), deletes, "4")); - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .size(), - 0); + EXPECT_EQ(getScopedRouteMap().size(), 0); EXPECT_EQ( 4UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // Now scope key "x-foo-key" points to nowhere as well. + EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + nullptr); } // Tests that conflict resources are detected. @@ -280,6 +300,12 @@ route_configuration_name: foo_routes // Fully rejected. 0UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // Scope key "x-foo-key" points to nowhere. + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + nullptr); // Delta API. EXPECT_THROW_WITH_REGEX( @@ -291,16 +317,14 @@ route_configuration_name: foo_routes 1UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); // foo_scope update is applied. - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .size(), - 1UL); - EXPECT_EQ(dynamic_cast(provider_.get()) - ->subscription() - .scopedRouteMap() - .count("foo_scope"), - 1); + EXPECT_EQ(getScopedRouteMap().size(), 1UL); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + // Scope key "x-foo-key" points to foo_routes due to partial rejection. + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); } // Tests that only one resource is provided during a config update. From cef747dda8d5be256c0dc6121c1ec2a2351a828e Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Mon, 12 Aug 2019 10:19:56 -0400 Subject: [PATCH 41/53] add more review fixes Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 21 +++++++++++++++++++++ source/common/router/scoped_config_impl.h | 21 +-------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index fd17d338221a..ad2d907f7ae6 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -70,6 +70,27 @@ HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const return nullptr; } +ScopedRouteInfo::ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, + std::unique_ptr&& route_provider) + : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { + ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); + ASSERT(!route_provider_->configInfo().has_value() || + route_provider_->config()->name() == config_proto_.route_configuration_name(), + fmt::format("RouteConfigProvider's name '{}' doesn't match route_configuration_name '{}'.", + route_provider_->config()->name(), config_proto_.route_configuration_name())); + // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of + // Fragment. + for (const auto& fragment : config_proto_.key().fragments()) { + switch (fragment.type_case()) { + case envoy::api::v2::ScopedRouteConfiguration::Key::Fragment::kStringKey: + scope_key_.addFragment(std::make_unique(fragment.string_key())); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + } +} + ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config) : ScopeKeyBuilderBase(std::move(config)) { for (const auto& fragment_builder : config_.fragments()) { diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 560c90327762..015d1e955d0e 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -147,26 +147,7 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { class ScopedRouteInfo { public: ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, - std::unique_ptr&& route_provider) - : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { - ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); - ASSERT( - !route_provider_->configInfo().has_value() || - route_provider_->config()->name() == config_proto_.route_configuration_name(), - fmt::format("RouteConfigProvider's name '{}' doesn't match route_configuration_name '{}'.", - route_provider_->config()->name(), config_proto_.route_configuration_name())); - // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of - // Fragment. - for (const auto& fragment : config_proto_.key().fragments()) { - switch (fragment.type_case()) { - case envoy::api::v2::ScopedRouteConfiguration::Key::Fragment::kStringKey: - scope_key_.addFragment(std::make_unique(fragment.string_key())); - break; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } - } - } + std::unique_ptr&& route_provider); Router::ConfigConstSharedPtr routeConfig() const { return route_provider_->config(); } const ScopeKey& scopeKey() const { return scope_key_; } From bb0cdb8f80f77faa2725981ad00669179e8b33e2 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 13 Aug 2019 11:32:53 -0400 Subject: [PATCH 42/53] disable cleanup timer on SRDS subscription destruction Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 6 ++++-- source/common/router/scoped_rds.h | 2 +- test/integration/scoped_rds_integration_test.cc | 7 +++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 19454be3f62b..50ead7db58cc 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -108,6 +108,8 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( }); } +ScopedRdsConfigSubscription::~ScopedRdsConfigSubscription() { cleanup_timer_->disableTimer(); } + void ScopedRdsConfigSubscription::cleanupDefferedDeletes() { for (auto iter = deffered_to_be_deleted_.begin(); iter != deffered_to_be_deleted_.end();) { if (iter->use_count() == 1) { @@ -118,7 +120,7 @@ void ScopedRdsConfigSubscription::cleanupDefferedDeletes() { } if (!deffered_to_be_deleted_.empty()) { // Cleanup as fast as possible. - cleanup_timer_->enableTimer(std::chrono::milliseconds(0)); + cleanup_timer_->enableTimer(std::chrono::milliseconds(100)); } } @@ -224,7 +226,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( if (!to_be_deleted.empty()) { deffered_to_be_deleted_.insert(deffered_to_be_deleted_.begin(), to_be_deleted.begin(), to_be_deleted.end()); - cleanup_timer_->enableTimer(std::chrono::milliseconds(0)); + cleanup_timer_->enableTimer(std::chrono::milliseconds(100)); } stats_.config_reload_.inc(); if (!exception_msgs.empty()) { diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index cbe0876aecf0..9f919ce3380a 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -95,7 +95,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); - ~ScopedRdsConfigSubscription() override = default; + ~ScopedRdsConfigSubscription() override; const std::string& name() const { return name_; } diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 2c294366ce3b..11ff06916cc3 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -210,12 +210,15 @@ route_configuration_name: foo_route1 - string_key: x-baz-key )EOF"; sendScopedRdsResponse({scope_route3}, "2"); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 6927017134761466251UL); test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 3); sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_3"), "3"); - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 3); + // RDS updates won't affect SRDS. test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 6927017134761466251UL); - // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented // at this point. } From fa6ecb1c91b3353a34905bde83c2cb23a1a9201e Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 13 Aug 2019 11:50:15 -0400 Subject: [PATCH 43/53] update comment on initManager overriding Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 50ead7db58cc..9dbec6d4720b 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -136,7 +136,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // If new route config sources come after the factory_context_.initManager()'s initialize() been // called, that initManager can't accept new targets. Instead we use a local override which will - // start new subscriptions but not wait on them to be ready (to not block on main thread). + // start new subscriptions but not wait on them to be ready. std::unique_ptr overriding_init_manager; if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. From efaa891b7cb4526d08c9ddbe4bc557fb1712d548 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 20 Aug 2019 08:26:10 -0400 Subject: [PATCH 44/53] A bunch of update after several issues found: * The "measure of keeping tls_ owner(i.e., the SubscriptionCommonBase)" in ConfigSubscriptionCommonBase::applyConfigUpdate has the problem that on Envoy Server destruction, the complete_cb may leave the ownership of the ConfigSubscriptionCommonBase instance in ServerImpl's dispatcherImpl event queue, which breaks the dependency order of listener_manager-->dispatcherImpl and causes unknown behavior. * Distribution of ownership of RdsRouteConfigProvider in SRDS impl to workers has a similar life-circle problem: when the SRDS subscription is being teared down due to LDS update or Envoy server shutdown, the on-the-fly callback may holds the final share of the shared_ptr in an extreme race condition. Filed issue #7902 which will solve this problem. This update contains the following updates: > Remove the shared_from_this tweaking from ConfigSubscriptionCommonBase::applyConfigUpdate, as it doesn't work as expected. > Revoke distributing ownership of a RdsRouteConfigProvider to worker threads, but instead, have RDS updates its parent SRDS subscription's tls_ map of {scope_key: Route::ConfigImpl} dict instead (distribute "data", not provider). > Add a callback_manager to RDSSubscription, similar to SDS API impl, during RDS config update, it calls the registered callbacks to update dependent entities. > Adjust tests accordingly. Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.h | 38 +- source/common/router/BUILD | 3 + source/common/router/rds_impl.cc | 1 + source/common/router/rds_impl.h | 11 + source/common/router/scoped_config_impl.cc | 9 +- source/common/router/scoped_config_impl.h | 8 +- source/common/router/scoped_rds.cc | 133 ++++--- source/common/router/scoped_rds.h | 42 ++- test/common/router/BUILD | 3 + test/common/router/scoped_config_impl_test.cc | 27 +- test/common/router/scoped_rds_test.cc | 330 +++++++++++++----- 11 files changed, 387 insertions(+), 218 deletions(-) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index a1a7b02d71b7..d656840b6ed4 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -9,7 +9,6 @@ #include "envoy/server/config_tracker.h" #include "envoy/singleton/instance.h" #include "envoy/thread_local/thread_local.h" - #include "common/common/thread.h" #include "common/common/utility.h" #include "common/config/utility.h" @@ -224,21 +223,17 @@ class ConfigSubscriptionCommonBase */ void applyConfigUpdate( const ConfigUpdateCb& update_fn, const Event::PostCb& complete_cb = []() {}) { + // During the update propagation, a subscription may get teared down in main thread due to + // all owners/providers destructed in a xDS update (e.g. LDS demolishes a + // RouteConfigProvider and its subscription). Capture a weak reference of "*this" and only apply + // the update if "*this" is still available. // It is safe to call shared_from_this here as this is in main thread, and destruction of a // ConfigSubscriptionCommonBase owner (i.e., a provider) happens in main thread as well. - auto shared_this = shared_from_this(); tls_->runOnAllThreads( [this, update_fn]() { - tls_->getTyped().config_ = update_fn(this->getConfig()); + tls_->getTyped().config_ = update_fn(getConfig()); }, - // During the update propagation, a subscription may get teared down in main thread due to - // all owners/providers destructed in a xDS update (e.g. LDS demolishes a - // RouteConfigProvider and its subscription). - // If such a race condition happens, holding a reference to the "*this" subscription - // instance in this cb will ensure the shared "*this" gets posted back to main thread, after - // all the workers finish calling the update_fn, at which point it's safe to destruct - // "*this" instance. - [shared_this, complete_cb]() { complete_cb(); }); + complete_cb); } void setLastUpdated() { last_updated_ = time_source_.systemTime(); } @@ -287,8 +282,8 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { /** * Must be called by the derived class' constructor. - * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with - * the underlying subscription, shared across all providers and workers. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated + * with the underlying subscription, shared across all providers and workers. */ void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) { tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { @@ -302,7 +297,8 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { * @param config_proto supplies the newly received config proto. * @param config_name supplies the name associated with the config. * @param version_info supplies the version associated with the config. - * @return bool false when the config proto has no delta from the previous config, true otherwise. + * @return bool false when the config proto has no delta from the previous config, true + * otherwise. */ bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info); @@ -369,8 +365,8 @@ class MutableConfigProviderCommonBase : public ConfigProvider { }; /** - * Provides generic functionality required by all config provider managers, such as managing shared - * lifetime of subscriptions and dynamic config providers, along with determining which + * Provides generic functionality required by all config provider managers, such as managing + * shared lifetime of subscriptions and dynamic config providers, along with determining which * subscriptions should be associated with newly instantiated providers. * * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase @@ -379,9 +375,9 @@ class MutableConfigProviderCommonBase : public ConfigProvider { * * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care - * must be taken with regards to destruction of these objects, since it must also happen on the main - * thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they were - * created. + * must be taken with regards to destruction of these objects, since it must also happen on the + * main thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they + * were created. * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. @@ -415,8 +411,8 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl const ConfigProviderSet& immutableConfigProviders(ConfigProviderInstanceType type) const; /** - * Returns the subscription associated with the config_source_proto; if none exists, a new one is - * allocated according to the subscription_factory_fn. + * Returns the subscription associated with the config_source_proto; if none exists, a new one + * is allocated according to the subscription_factory_fn. * @param config_source_proto supplies the proto specifying the config subscription parameters. * @param init_manager supplies the init manager. * @param subscription_factory_fn supplies a function to be called when a new subscription needs diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 0327ee749ca2..21970908db10 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -132,6 +132,7 @@ envoy_cc_library( "//include/envoy/singleton:instance_interface", "//include/envoy/thread_local:thread_local_interface", "//source/common/common:assert_lib", + "//source/common/common:callback_impl_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:rds_json_lib", "//source/common/config:subscription_factory_lib", @@ -168,12 +169,14 @@ envoy_cc_library( srcs = ["scoped_rds.cc"], hdrs = ["scoped_rds.h"], deps = [ + ":rds_lib", ":scoped_config_lib", "//include/envoy/config:config_provider_interface", "//include/envoy/config:subscription_interface", "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", + "//source/common/common:cleanup_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:config_provider_lib", "//source/common/init:manager_lib", diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index e8be482b4a1b..2dff2f7c6eb6 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -120,6 +120,7 @@ void RdsRouteConfigSubscription::onConfigUpdate( } vhds_subscription_.release(); } + update_callback_manager_.runCallbacks(); } init_target_.ready(); diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 33e3796496fa..d461a7f9e272 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -13,6 +13,7 @@ #include "envoy/config/subscription.h" #include "envoy/http/codes.h" #include "envoy/local_info/local_info.h" +#include "common/common/callback_impl.h" #include "envoy/router/rds.h" #include "envoy/router/route_config_provider_manager.h" #include "envoy/router/route_config_update_receiver.h" @@ -31,6 +32,9 @@ namespace Envoy { namespace Router { +// For friend class declaration in RdsRouteConfigSubscription. +class ScopedRdsConfigSubscription; + /** * Route configuration provider utilities. */ @@ -122,6 +126,10 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, .name(); } + Common::CallbackHandle* addUpdateCallback(std::function callback) { + return update_callback_manager_.add(callback); + } + RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, @@ -143,8 +151,11 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, VhdsSubscriptionPtr vhds_subscription_; RouteConfigUpdatePtr config_update_info_; ProtobufMessage::ValidationVisitor& validation_visitor_; + Common::CallbackManager<> update_callback_manager_; friend class RouteConfigProviderManagerImpl; + // Access to addUpdateCallback + friend class ScopedRdsConfigSubscription; }; using RdsRouteConfigSubscriptionSharedPtr = std::shared_ptr; diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index ad2d907f7ae6..a9e030f9f7a8 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -71,13 +71,8 @@ HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const } ScopedRouteInfo::ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, - std::unique_ptr&& route_provider) - : config_proto_(std::move(config_proto)), route_provider_(std::move(route_provider)) { - ASSERT(route_provider_ != nullptr, "ScopedRouteInfo expects a valid RouteConfigProvider."); - ASSERT(!route_provider_->configInfo().has_value() || - route_provider_->config()->name() == config_proto_.route_configuration_name(), - fmt::format("RouteConfigProvider's name '{}' doesn't match route_configuration_name '{}'.", - route_provider_->config()->name(), config_proto_.route_configuration_name())); + ConfigConstSharedPtr&& route_config) + : config_proto_(std::move(config_proto)), route_config_(std::move(route_config)) { // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of // Fragment. for (const auto& fragment : config_proto_.key().fragments()) { diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 015d1e955d0e..6a4d849234df 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -147,17 +147,17 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { class ScopedRouteInfo { public: ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, - std::unique_ptr&& route_provider); + ConfigConstSharedPtr&& route_config); - Router::ConfigConstSharedPtr routeConfig() const { return route_provider_->config(); } + ConfigConstSharedPtr routeConfig() const { return route_config_; } const ScopeKey& scopeKey() const { return scope_key_; } const envoy::api::v2::ScopedRouteConfiguration& configProto() const { return config_proto_; } const std::string& scopeName() const { return config_proto_.name(); } private: - const envoy::api::v2::ScopedRouteConfiguration config_proto_; + envoy::api::v2::ScopedRouteConfiguration config_proto_; ScopeKey scope_key_; - std::unique_ptr route_provider_; + ConfigConstSharedPtr route_config_; }; using ScopedRouteInfoConstSharedPtr = std::shared_ptr; // Ordered map for consistent config dumping. diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9dbec6d4720b..2055cf72d7b1 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -7,6 +7,7 @@ #include "common/common/assert.h" #include "common/common/logger.h" +#include "common/common/cleanup.h" #include "common/common/utility.h" #include "common/config/resources.h" #include "common/init/manager_impl.h" @@ -86,9 +87,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), - factory_context_(factory_context), cleanup_timer_(factory_context.dispatcher().createTimer( - [this]() -> void { cleanupDefferedDeletes(); })), - name_(name), scope_key_builder_(scope_key_builder), + factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), rds_config_source_(std::move(rds_config_source)), @@ -108,21 +107,20 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( }); } -ScopedRdsConfigSubscription::~ScopedRdsConfigSubscription() { cleanup_timer_->disableTimer(); } - -void ScopedRdsConfigSubscription::cleanupDefferedDeletes() { - for (auto iter = deffered_to_be_deleted_.begin(); iter != deffered_to_be_deleted_.end();) { - if (iter->use_count() == 1) { - iter = deffered_to_be_deleted_.erase(iter); - } else { - iter++; - } - } - if (!deffered_to_be_deleted_.empty()) { - // Cleanup as fast as possible. - cleanup_timer_->enableTimer(std::chrono::milliseconds(100)); - } -} +ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::RdsRouteConfigProviderHelper( + ScopedRdsConfigSubscription& parent, std::string scope_name, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Init::Manager& init_manager) + : parent_(parent), scope_name_(scope_name), + route_provider_(static_cast( + parent_.route_config_provider_manager_ + .createRdsRouteConfigProvider(rds, parent_.factory_context_, parent_.stat_prefix_, + init_manager) + .release())), + rds_update_callback_handle_(route_provider_->subscription().addUpdateCallback([this]() { + // Subscribe to RDS update. + parent_.onRdsConfigUpdate(scope_name_, route_provider_->subscription()); + })) {} void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, @@ -138,32 +136,54 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // called, that initManager can't accept new targets. Instead we use a local override which will // start new subscriptions but not wait on them to be ready. std::unique_ptr overriding_init_manager; + // NOTE: This should be defined after overriding_init_manager as it depends on the + // overriding_init_manager. + std::unique_ptr resume_rds; if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { + overriding_init_manager = + std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. + // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either + // by Server init or LDS init. factory_context_.clusterManager().adsMux().pause( Envoy::Config::TypeUrl::get().RouteConfiguration); - overriding_init_manager = - std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); + resume_rds = std::make_unique([this, &overriding_init_manager, version_info] { + // For new RDS subscriptions created after listener warming up, we don't wait for them to warm + // up. + Init::WatcherImpl noop_watcher( + // Note: we just throw it away. + fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), + []() { /*Do nothing.*/ }); + overriding_init_manager->initialize(noop_watcher); + // New RDS subscriptions should have been created, now lift the floodgate. + // Note in the case of partial acceptance, accepted RDS subscriptions should be started + // dispite of any error. + factory_context_.clusterManager().adsMux().resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + }); } - std::list to_be_deleted; for (const auto& resource : added_resources) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; try { scoped_route_config = MessageUtil::anyConvert( resource.resource(), validation_visitor_); MessageUtil::validate(scoped_route_config); - if (!unique_resource_names.insert(scoped_route_config.name()).second) { - throw EnvoyException(fmt::format("duplicate scoped route configuration '{}' found", - scoped_route_config.name())); + const std::string scope_name = scoped_route_config.name(); + if (!unique_resource_names.insert(scope_name).second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration '{}' found", scope_name)); } + // TODO(stevenzzz): Creating a new RdsRouteConfigProvider likely expensive, migrate RDS to + // config-provider-framework to make it light weight. rds.set_route_config_name(scoped_route_config.route_configuration_name()); - ScopedRouteInfoConstSharedPtr scoped_route_info = std::make_shared( - std::move(scoped_route_config), - route_config_provider_manager_.createRdsRouteConfigProvider( - rds, factory_context_, stat_prefix_, - (overriding_init_manager == nullptr ? factory_context_.initManager() - : *overriding_init_manager))); + // Delete previous route provider if any. + auto rds_config_provider_helper = std::make_unique( + *this, scope_name, rds, + (overriding_init_manager == nullptr ? factory_context_.initManager() + : *overriding_init_manager)); + auto scoped_route_info = std::make_shared( + std::move(scoped_route_config), rds_config_provider_helper->routeConfig()); // Detect if there is key conflict between two scopes. auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); if (iter != scope_name_by_hash_.end()) { @@ -172,8 +192,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", iter->second, scoped_route_info->scopeName())); } - to_be_deleted.push_back(scoped_route_map_[scoped_route_info->scopeName()]); } + route_provider_by_scope_.insert({scope_name, std::move(rds_config_provider_helper)}); scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) @@ -184,28 +204,17 @@ void ScopedRdsConfigSubscription::onConfigUpdate( return config; }); any_applied = true; - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_info->scopeName()); + ENVOY_LOG(debug, "srds: add/update scoped_route '{}', version: {}", + scoped_route_info->scopeName(), version_info); } catch (const EnvoyException& e) { exception_msgs.emplace_back(fmt::format("{}", e.what())); } } - if (overriding_init_manager != nullptr) { - // For new RDS subscriptions created after listener warming up, we don't wait for them to warm - // up. - Init::WatcherImpl noop_watcher( - // Note: we just throw it away. - fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), - []() { /*Do nothing.*/ }); - overriding_init_manager->initialize(noop_watcher); - // New RDS subscriptions should be created, now lift the floodgate. - factory_context_.clusterManager().adsMux().resume( - Envoy::Config::TypeUrl::get().RouteConfiguration); - } for (const auto& scope_name : removed_resources) { auto iter = scoped_route_map_.find(scope_name); if (iter != scoped_route_map_.end()) { - to_be_deleted.push_back(iter->second); + route_provider_by_scope_.erase(scope_name); scope_name_by_hash_.erase(iter->second->scopeKey().hash()); scoped_route_map_.erase(iter); applyConfigUpdate([scope_name](ConfigProvider::ConfigConstSharedPtr config) @@ -216,18 +225,13 @@ void ScopedRdsConfigSubscription::onConfigUpdate( return config; }); any_applied = true; - ENVOY_LOG(debug, "srds: remove scoped route '{}'", scope_name); + ENVOY_LOG(debug, "srds: remove scoped route '{}', version: {}", scope_name, version_info); } } ConfigSubscriptionCommonBase::onConfigUpdate(); if (any_applied) { setLastConfigInfo(absl::optional({absl::nullopt, version_info})); } - if (!to_be_deleted.empty()) { - deffered_to_be_deleted_.insert(deffered_to_be_deleted_.begin(), to_be_deleted.begin(), - to_be_deleted.end()); - cleanup_timer_->enableTimer(std::chrono::milliseconds(100)); - } stats_.config_reload_.inc(); if (!exception_msgs.empty()) { throw EnvoyException(fmt::format("Error adding/updating scoped route(s): {}", @@ -235,6 +239,24 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } } +void ScopedRdsConfigSubscription::onRdsConfigUpdate(const std::string& scope_name, + RdsRouteConfigSubscription& rds_subscription) { + auto iter = scoped_route_map_.find(scope_name); + ASSERT(iter != scoped_route_map_.end(), + fmt::format("trying to update route config for non-existing scope {}", scope_name)); + auto new_scoped_route_info = std::make_shared( + envoy::api::v2::ScopedRouteConfiguration(iter->second->configProto()), + std::make_shared(rds_subscription.routeConfigUpdate()->routeConfiguration(), + factory_context_, false)); + applyConfigUpdate([new_scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(new_scoped_route_info); + return config; + }); +} + // TODO(stevenzzzz): see issue #7508, consider generalizing this function as it overlaps with // CdsApiImpl::onConfigUpdate. void ScopedRdsConfigSubscription::onConfigUpdate( @@ -281,10 +303,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } // namespace Router ScopedRdsConfigProvider::ScopedRdsConfigProvider( - ScopedRdsConfigSubscriptionSharedPtr&& subscription, - envoy::api::v2::core::ConfigSource rds_config_source) - : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), - rds_config_source_(std::move(rds_config_source)) {} + ScopedRdsConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta) {} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); @@ -347,8 +367,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( static_cast(config_provider_manager)); }); - return std::make_unique(std::move(subscription), - typed_optarg.rds_config_source_); + return std::make_unique(std::move(subscription)); } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 9f919ce3380a..212b1d616335 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -6,9 +6,11 @@ #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/config/subscription.h" #include "envoy/router/route_config_provider_manager.h" +#include "common/init/manager_impl.h" #include "envoy/stats/scope.h" - #include "common/config/config_provider_impl.h" +#include "envoy/common/callback.h" +#include "common/router/rds_impl.h" #include "common/router/scoped_config_impl.h" namespace Envoy { @@ -95,15 +97,28 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); - ~ScopedRdsConfigSubscription() override; + ~ScopedRdsConfigSubscription() override = default; const std::string& name() const { return name_; } const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: - // Cleanup stale ScopedRouteInfo instances. - void cleanupDefferedDeletes(); + // A helper class that takes care the life circle management of a RDS route provider and the + // update callback handle. + struct RdsRouteConfigProviderHelper { + RdsRouteConfigProviderHelper( + ScopedRdsConfigSubscription& parent, std::string scope_name, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Init::Manager& init_manager); + ~RdsRouteConfigProviderHelper() { rds_update_callback_handle_->remove(); } + ConfigConstSharedPtr routeConfig() { return route_provider_->config(); } + + ScopedRdsConfigSubscription& parent_; + std::string scope_name_; + std::unique_ptr route_provider_; + Common::CallbackHandle* rds_update_callback_handle_; + }; // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } @@ -128,20 +143,19 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio validation_visitor_) .name(); } + // Propagate RDS updates to ScopeConfigImpl in workers. + void onRdsConfigUpdate(const std::string& scope_name, + RdsRouteConfigSubscription& rds_subscription); // ScopedRouteInfo by scope name. ScopedRouteMap scoped_route_map_; + // RdsRouteConfigProvider by scope name. + absl::flat_hash_map> + route_provider_by_scope_; // A map of (hash, scope-name), used to detect the key conflict between scopes. absl::flat_hash_map scope_name_by_hash_; // For creating RDS subscriptions. Server::Configuration::FactoryContext& factory_context_; - // Deferred to be deleted ScopedRouteInfo. - // As RdsRouteConfigProvider instance (which contains a tls slot) can only be destructed in main - // thread, we need to make sure the shared ScopedRouteInfo is deleted in main thread. - std::list deffered_to_be_deleted_; - Envoy::Event::TimerPtr cleanup_timer_; - const std::chrono::milliseconds cleanup_interval_{3000}; // Clean up every 3s. - const std::string name_; std::unique_ptr subscription_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder @@ -160,15 +174,11 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr(subscription_.get()); } - -private: - const envoy::api::v2::core::ConfigSource rds_config_source_; }; // A ConfigProviderManager for scoped routing configuration that creates static/inline and dynamic diff --git a/test/common/router/BUILD b/test/common/router/BUILD index c47295ee6ae4..2ac71059b8bc 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -97,11 +97,14 @@ envoy_cc_test( "abseil_strings", ], deps = [ + "//include/envoy/config:subscription_interface", + "//include/envoy/init:manager_interface", "//source/common/config:utility_lib", "//source/common/http:message_lib", "//source/common/json:json_loader_lib", "//source/common/router:scoped_rds_lib", "//source/server/http:admin_lib", + "//test/mocks/config:config_mocks", "//test/mocks/init:init_mocks", "//test/mocks/router:router_mocks", "//test/mocks/server:server_mocks", diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index f85d1bea61c3..cf9bfd83055c 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -360,42 +360,23 @@ class ScopedRouteInfoTest : public testing::Test { route_config_ = std::make_shared>(); route_config_->name_ = "foo_route"; - - route_config_provider_ = std::make_unique(); - EXPECT_CALL(*route_config_provider_, config()).WillRepeatedly(Return(route_config_)); - EXPECT_CALL(*route_config_provider_, configInfo()) - .WillRepeatedly(Return(RouteConfigProvider::ConfigInfo{route_configuration_, ""})); } envoy::api::v2::RouteConfiguration route_configuration_; envoy::api::v2::ScopedRouteConfiguration scoped_route_config_; std::shared_ptr route_config_; std::unique_ptr info_; - std::unique_ptr route_config_provider_; }; TEST_F(ScopedRouteInfoTest, Creation) { envoy::api::v2::ScopedRouteConfiguration config_copy = scoped_route_config_; - info_ = std::make_unique(std::move(scoped_route_config_), - std::move(route_config_provider_)); + info_ = std::make_unique(std::move(scoped_route_config_), route_config_); EXPECT_EQ(info_->routeConfig().get(), route_config_.get()); EXPECT_TRUE(TestUtility::protoEqual(info_->configProto(), config_copy)); EXPECT_EQ(info_->scopeName(), "foo_scope"); EXPECT_EQ(info_->scopeKey(), makeKey({"foo", "bar"})); } -class ScopedRouteInfoDeathTest : public ScopedRouteInfoTest {}; - -TEST_F(ScopedRouteInfoDeathTest, AssertFailure) { - EXPECT_DEBUG_DEATH(ScopedRouteInfo(std::move(scoped_route_config_), nullptr), - "ScopedRouteInfo expects a valid RouteConfigProvider."); - - route_config_->name_ = "bar_route"; - EXPECT_DEBUG_DEATH( - ScopedRouteInfo(std::move(scoped_route_config_), std::move(route_config_provider_)), - "RouteConfigProvider's name 'bar_route' doesn't match route_configuration_name 'foo_route'."); -} - class ScopedConfigImplTest : public testing::Test { public: void SetUp() override { @@ -445,12 +426,8 @@ class ScopedConfigImplTest : public testing::Test { std::shared_ptr route_config = std::make_shared>(); route_config->name_ = scoped_route_config.route_configuration_name(); - - std::unique_ptr route_config_provider = - std::make_unique>(); - EXPECT_CALL(*route_config_provider, config()).WillRepeatedly(Return(route_config)); return std::make_shared(std::move(scoped_route_config), - std::move(route_config_provider)); + std::move(route_config)); } std::shared_ptr scope_info_a_; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 12f5e61c718a..5b07427f2f29 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -2,12 +2,13 @@ #include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/admin/v2alpha/config_dump.pb.validate.h" +#include "envoy/init/manager.h" #include "envoy/stats/scope.h" - +#include "envoy/config/subscription.h" #include "common/router/scoped_rds.h" - #include "test/mocks/router/mocks.h" #include "test/mocks/server/mocks.h" +#include "test/mocks/config/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -16,10 +17,13 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::AnyNumber; using testing::ByMove; using testing::DoAll; +using testing::Eq; using testing::InSequence; using testing::Invoke; +using testing::NiceMock; using testing::Return; using testing::ReturnRefOfCopy; @@ -52,27 +56,13 @@ parseHttpConnectionManagerFromYaml(const std::string& config_yaml) { class ScopedRoutesTestBase : public testing::Test { protected: ScopedRoutesTestBase() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("routes", _)); + route_config_provider_manager_ = + std::make_unique(factory_context_.admin_); + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); config_provider_manager_ = std::make_unique( - factory_context_.admin_, route_config_provider_manager_); - EXPECT_CALL(route_config_provider_manager_, createRdsRouteConfigProvider(_, _, _, _)) - .WillRepeatedly(Invoke( - [this](const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext&, const std::string&, - Init::Manager&) -> RouteConfigProviderPtr { - auto iter = cached_route_configs_.find(rds.route_config_name()); - if (iter == cached_route_configs_.end()) { - cached_route_configs_[rds.route_config_name()] = std::make_shared(); - EXPECT_CALL(*cached_route_configs_[rds.route_config_name()], name()) - .WillRepeatedly(ReturnRefOfCopy(rds.route_config_name())); - } - auto provider = std::make_unique>(); - EXPECT_CALL(*provider, config()) - .WillRepeatedly(Invoke([this, rds]() -> ConfigConstSharedPtr { - return cached_route_configs_[rds.route_config_name()]; - })); - return provider; - })); + factory_context_.admin_, *route_config_provider_manager_); } ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } @@ -95,11 +85,10 @@ class ScopedRoutesTestBase : public testing::Test { Event::SimulatedTimeSystem& timeSystem() { return time_system_; } NiceMock factory_context_; + std::unique_ptr route_config_provider_manager_; std::unique_ptr config_provider_manager_; - MockRouteConfigProviderManager route_config_provider_manager_; - absl::flat_hash_map> cached_route_configs_; + Event::SimulatedTimeSystem time_system_; - envoy::api::v2::core::ConfigSource rds_config_source_; }; class ScopedRdsTest : public ScopedRoutesTestBase { @@ -107,6 +96,47 @@ class ScopedRdsTest : public ScopedRoutesTestBase { void setup() { InSequence s; + // Since factory_context_.cluster_manager_.subscription_factory_.callbacks_ is taken by the SRDS + // subscription. We need to return a different MockSubscription here for each RDS subscription. + // To build the map from RDS route_config_name to the RDS subscription, we need to get the + // route_config_name by mocking start() on the Config::Subscription. + EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, + subscriptionFromConfigSource(_, _, _, _)) + .Times(AnyNumber()); + EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, + subscriptionFromConfigSource( + _, + Eq(Grpc::Common::typeUrl( + envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name())), + _, _)) + .Times(AnyNumber()) + .WillRepeatedly(Invoke([this](const envoy::api::v2::core::ConfigSource&, absl::string_view, + Stats::Scope&, + Envoy::Config::SubscriptionCallbacks& callbacks) { + auto ret = std::make_unique>(); + rds_subscription_by_config_subscription_[ret.get()] = &callbacks; + EXPECT_CALL(*ret, start(_)) + .WillOnce(Invoke( + [this, config_sub_addr = ret.get()](const std::set& resource_names) { + EXPECT_EQ(resource_names.size(), 1); + auto iter = rds_subscription_by_config_subscription_.find(config_sub_addr); + EXPECT_NE(iter, rds_subscription_by_config_subscription_.end()); + rds_subscription_by_name_[*resource_names.begin()] = iter->second; + })); + return ret; + })); + + ON_CALL(factory_context_.init_manager_, add(_)) + .WillByDefault(Invoke([this](const Init::Target& target) { + target_handles_.push_back(target.createHandle("test")); + })); + ON_CALL(factory_context_.init_manager_, initialize(_)) + .WillByDefault(Invoke([this](const Init::Watcher& watcher) { + for (auto& handle_ : target_handles_) { + handle_->initialize(watcher); + } + })); + const std::string config_yaml = R"EOF( name: foo_scoped_routes scope_key_builder: @@ -124,7 +154,25 @@ name: foo_scoped_routes ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), scoped_routes_config.rds_config_source(), scoped_routes_config.scope_key_builder())); - subscription_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; + srds_subscription_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; + } + + // Helper function which pushes an update to given RDS subscription, the start(_) of the + // subscription must have been called. + void pushRdsConfig(const std::string& route_config_name, const std::string& version) { + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: test + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: bluh }} +)EOF"; + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(TestUtility::parseYaml( + fmt::format(route_config_tmpl, route_config_name))); + rds_subscription_by_name_[route_config_name]->onConfigUpdate(resources, version); } ScopedRdsConfigProvider* getScopedRdsProvider() const { @@ -135,8 +183,15 @@ name: foo_scoped_routes return getScopedRdsProvider()->subscription().scopedRouteMap(); } - Envoy::Config::SubscriptionCallbacks* subscription_callbacks_{}; + Envoy::Config::SubscriptionCallbacks* srds_subscription_{}; Envoy::Config::ConfigProviderPtr provider_; + std::list target_handles_; + Init::ExpectableWatcherImpl init_watcher_; + + // RDS mocks. + absl::flat_hash_map + rds_subscription_by_config_subscription_; + absl::flat_hash_map rds_subscription_by_name_; }; TEST_F(ScopedRdsTest, ValidateFail) { @@ -152,11 +207,10 @@ route_configuration_name: foo_routes )EOF"; Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources, "1"), ProtoValidationException); EXPECT_THROW_WITH_REGEX( - subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), - EnvoyException, + srds_subscription_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'route_configuration_name' validation: value must be > 1 byte. @@ -169,10 +223,9 @@ name: foo_scope )EOF"; Protobuf::RepeatedPtrField resources2; parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources2, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources2, "1"), ProtoValidationException); EXPECT_THROW_WITH_REGEX( - subscription_callbacks_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), - EnvoyException, + srds_subscription_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), EnvoyException, "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'key' validation: must define at least 1 fragment. @@ -183,16 +236,15 @@ route_configuration_name: foo_routes )EOF"; Protobuf::RepeatedPtrField resources3; parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources3, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources3, "1"), ProtoValidationException); EXPECT_THROW_WITH_REGEX( - subscription_callbacks_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), - EnvoyException, + srds_subscription_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), EnvoyException, "Error adding/updating scoped route\\(s\\): Proto constraint validation failed .*value is " "required.*"); } // Tests that multiple uniquely named non-conflict resources are allowed in config updates. -TEST_F(ScopedRdsTest, MultipleResources) { +TEST_F(ScopedRdsTest, MultipleResourcesStow) { setup(); const std::string config_yaml = R"EOF( @@ -212,21 +264,29 @@ route_configuration_name: foo_routes - string_key: x-bar-key )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); - EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "1")); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(resources, "1")); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times(2); // SRDS and RDS "foo_routes" EXPECT_EQ( 1UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); - // Delta API. - EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(anyToResource(resources, "2"), {}, "2")); - EXPECT_EQ( - 2UL, - factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); - EXPECT_EQ(getScopedRouteMap().size(), 2); - - // Verify the config is a ScopedConfigImpl instance, both scopes point to foo_routes. + // Verify the config is a ScopedConfigImpl instance, both scopes point to "" as RDS hasn't kicked + // in yet(NullConfigImpl returned). EXPECT_NE(getScopedRdsProvider(), nullptr); EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + ""); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + ""); + // RDS updates foo_routes. + pushRdsConfig("foo_routes", "111"); EXPECT_EQ(getScopedRdsProvider() ->config() ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) @@ -240,12 +300,11 @@ route_configuration_name: foo_routes // Delete foo_scope2. resources.RemoveLast(); - - EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "3")); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(resources, "3")); EXPECT_EQ(getScopedRouteMap().size(), 1); EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); EXPECT_EQ( - 3UL, + 2UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); // now scope key "x-bar-key" points to nowhere. EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( @@ -256,20 +315,85 @@ route_configuration_name: foo_routes ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) ->name(), "foo_routes"); - // Delete foo_scope via Delta API. - Protobuf::RepeatedPtrField deletes; - *deletes.Add() = "foo_scope"; +} + +// Tests that multiple uniquely named non-conflict resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResourcesDelta) { + setup(); + init_watcher_.expectReady().Times(2); // SRDS and RDS "foo_routes" + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-bar-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + + // Delta API. + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(anyToResource(resources, "2"), {}, "1")); + factory_context_.init_manager_.initialize(init_watcher_); + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + EXPECT_EQ(getScopedRouteMap().size(), 2); + + // Verify the config is a ScopedConfigImpl instance, both scopes point to "" as RDS hasn't kicked + // in yet(NullConfigImpl returned). + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + ""); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + ""); + // RDS updates foo_routes. + pushRdsConfig("foo_routes", "111"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + "foo_routes"); + + // Delete foo_scope2. resources.RemoveLast(); - EXPECT_NO_THROW( - subscription_callbacks_->onConfigUpdate(anyToResource(resources, "4"), deletes, "4")); - EXPECT_EQ(getScopedRouteMap().size(), 0); + Protobuf::RepeatedPtrField deletes; + *deletes.Add() = "foo_scope2"; + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(anyToResource(resources, "4"), deletes, "2")); + EXPECT_EQ(getScopedRouteMap().size(), 1); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); EXPECT_EQ( - 4UL, + 2UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); - // Now scope key "x-foo-key" points to nowhere as well. + // now scope key "x-bar-key" points to nowhere. EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( - TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); } // Tests that conflict resources are detected. @@ -294,7 +418,7 @@ route_configuration_name: foo_routes )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); EXPECT_THROW_WITH_REGEX( - subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, + srds_subscription_->onConfigUpdate(resources, "1"), EnvoyException, ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); EXPECT_EQ( // Fully rejected. @@ -306,11 +430,16 @@ route_configuration_name: foo_routes EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), nullptr); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times( + 1); // Just SRDS, RDS "foo_routes" will initialized by the noop init-manager. + EXPECT_EQ(factory_context_.scope_.counter("foo.rds.foo_routes.config_reload").value(), 0UL); // Delta API. + EXPECT_CALL(factory_context_.init_manager_, state()) + .WillOnce(Return(Init::Manager::State::Initialized)); EXPECT_THROW_WITH_REGEX( - subscription_callbacks_->onConfigUpdate(anyToResource(resources, "2"), {}, "2"), - EnvoyException, + srds_subscription_->onConfigUpdate(anyToResource(resources, "2"), {}, "2"), EnvoyException, ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); EXPECT_EQ( // Partially reject. @@ -320,6 +449,8 @@ route_configuration_name: foo_routes EXPECT_EQ(getScopedRouteMap().size(), 1UL); EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); // Scope key "x-foo-key" points to foo_routes due to partial rejection. + pushRdsConfig("foo_routes", "111"); // Push some real route configuration. + EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.rds.foo_routes.config_reload").value()); EXPECT_EQ(getScopedRdsProvider() ->config() ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) @@ -328,8 +459,10 @@ route_configuration_name: foo_routes } // Tests that only one resource is provided during a config update. -TEST_F(ScopedRdsTest, InvalidDuplicateResource) { +TEST_F(ScopedRdsTest, InvalidDuplicateResourceSotw) { setup(); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times(0); const std::string config_yaml = R"EOF( name: foo_scope @@ -341,14 +474,42 @@ route_configuration_name: foo_routes Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); - EXPECT_THROW_WITH_MESSAGE(subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(srds_subscription_->onConfigUpdate(resources, "1"), EnvoyException, "duplicate scoped route configuration 'foo_scope' found"); +} + +// Tests that only one resource is provided during a config update. +TEST_F(ScopedRdsTest, InvalidDuplicateResourceDelta) { + setup(); + factory_context_.init_manager_.initialize(init_watcher_); + // After the above initialize, the default init_manager should return "Initialized". + EXPECT_CALL(factory_context_.init_manager_, state()) + .WillOnce(Return(Init::Manager::State::Initialized)); + init_watcher_.expectReady().Times( + 1); // SRDS onConfigUpdate breaks, but first foo_routes will + // kick start if it's initialized post-Server/LDS initialization. + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW_WITH_MESSAGE( - subscription_callbacks_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), - EnvoyException, + srds_subscription_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, "Error adding/updating scoped route(s): duplicate scoped route configuration 'foo_scope' " "found"); + EXPECT_EQ( + // Partially reject. + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // foo_scope update is applied. + EXPECT_EQ(getScopedRouteMap().size(), 1UL); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); } // Tests a config update failure. @@ -359,25 +520,30 @@ TEST_F(ScopedRdsTest, ConfigUpdateFailure) { timeSystem().setSystemTime(time); const EnvoyException ex(fmt::format("config failure")); // Verify the failure updates the lastUpdated() timestamp. - subscription_callbacks_->onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &ex); + srds_subscription_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, + &ex); EXPECT_EQ(std::chrono::time_point_cast(provider_->lastUpdated()) .time_since_epoch(), time); } -using ScopedRoutesConfigProviderManagerTest = ScopedRoutesTestBase; - // Tests that the /config_dump handler returns the corresponding scoped routing // config. -TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { +TEST_F(ScopedRdsTest, ConfigDump) { + setup(); + factory_context_.init_manager_.initialize(init_watcher_); + EXPECT_CALL(factory_context_.init_manager_, state()) + .Times(2) // There are two SRDS pushes. + .WillRepeatedly(Return(Init::Manager::State::Initialized)); + init_watcher_.expectReady().Times(1); // SRDS only, no RDS push. + auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump = MessageUtil::downcastAndValidate( *message_ptr); - // No routes at all, no last_updated timestamp + // No routes at all(no SRDS push yet), no last_updated timestamp envoy::admin::v2alpha::ScopedRoutesConfigDump expected_config_dump; TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: @@ -444,15 +610,7 @@ stat_prefix: foo expected_config_dump); EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump2)); - const std::string scoped_rds_config_yaml = R"EOF( - scoped_rds: - scoped_rds_config_source: -)EOF"; - Envoy::Config::ConfigProviderPtr dynamic_provider = ScopedRoutesConfigProviderUtil::create( - parseHttpConnectionManagerFromYaml(absl::Substitute( - hcm_base_config_yaml, "foo-dynamic-scoped-routes", scoped_rds_config_yaml)), - factory_context_, "foo.", *config_provider_manager_); - + // Now SRDS kicks off. Protobuf::RepeatedPtrField resources; resources.Add()->PackFrom(parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo @@ -462,8 +620,7 @@ route_configuration_name: dynamic-foo-route-config )EOF")); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891567)); - factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, - "1"); + srds_subscription_->onConfigUpdate(resources, "1"); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: @@ -481,7 +638,7 @@ route_configuration_name: dynamic-foo-route-config seconds: 1234567891 nanos: 234000000 dynamic_scoped_route_configs: - - name: foo-dynamic-scoped-routes + - name: foo_scoped_routes scoped_route_configs: - name: dynamic-foo route_configuration_name: dynamic-foo-route-config @@ -500,8 +657,7 @@ route_configuration_name: dynamic-foo-route-config EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump3)); resources.Clear(); - factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, - "2"); + srds_subscription_->onConfigUpdate(resources, "2"); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: - name: foo-scoped-routes @@ -518,7 +674,7 @@ route_configuration_name: dynamic-foo-route-config seconds: 1234567891 nanos: 234000000 dynamic_scoped_route_configs: - - name: foo-dynamic-scoped-routes + - name: foo_scoped_routes last_updated: seconds: 1234567891 nanos: 567000000 @@ -532,10 +688,8 @@ route_configuration_name: dynamic-foo-route-config EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } -using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderManagerTest; - // Tests that SRDS only allows creation of delta static config providers. -TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { +TEST_F(ScopedRdsTest, DeltaStaticConfigProviderOnly) { // Use match all regex due to lack of distinctive matchable output for // coverage test. EXPECT_DEATH(config_provider_manager_->createStaticConfigProvider( From bf4dc23a044e48ebde6738300c7b9cca7c1883b8 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Tue, 20 Aug 2019 10:05:18 -0400 Subject: [PATCH 45/53] fix-format, and fix update for htuch's comments around ScopeKey hash and overriding_init_manager. Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.h | 1 + source/common/router/rds_impl.h | 2 +- source/common/router/scoped_config_impl.h | 7 +++++-- source/common/router/scoped_rds.cc | 5 +++-- source/common/router/scoped_rds.h | 5 +++-- test/common/router/scoped_rds_test.cc | 6 ++++-- 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index d656840b6ed4..259246665c6f 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -9,6 +9,7 @@ #include "envoy/server/config_tracker.h" #include "envoy/singleton/instance.h" #include "envoy/thread_local/thread_local.h" + #include "common/common/thread.h" #include "common/common/utility.h" #include "common/config/utility.h" diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index d461a7f9e272..ff64639d41b1 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -13,7 +13,6 @@ #include "envoy/config/subscription.h" #include "envoy/http/codes.h" #include "envoy/local_info/local_info.h" -#include "common/common/callback_impl.h" #include "envoy/router/rds.h" #include "envoy/router/route_config_provider_manager.h" #include "envoy/router/route_config_update_receiver.h" @@ -23,6 +22,7 @@ #include "envoy/stats/scope.h" #include "envoy/thread_local/thread_local.h" +#include "common/common/callback_impl.h" #include "common/common/logger.h" #include "common/init/target_impl.h" #include "common/protobuf/utility.h" diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 6a4d849234df..f6197f09abdc 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -66,8 +66,11 @@ class ScopeKey { private: // Update the key's hash with the new fragment hash. void updateHash(const ScopeKeyFragmentBase& fragment) { - absl::uint128 buffer = absl::MakeUint128(hash_, fragment.hash()); - hash_ = HashUtil::xxHash64(absl::string_view(reinterpret_cast(&buffer), 16)); + std::stringbuf buffer; + buffer.sputn(reinterpret_cast(&hash_), sizeof(hash_)); + const auto& fragment_hash = fragment.hash(); + buffer.sputn(reinterpret_cast(&fragment_hash), sizeof(fragment_hash)); + hash_ = HashUtil::xxHash64(buffer.str()); } uint64_t hash_{0}; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 2055cf72d7b1..64ccabb6c333 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -6,8 +6,8 @@ #include "envoy/api/v2/srds.pb.validate.h" #include "common/common/assert.h" -#include "common/common/logger.h" #include "common/common/cleanup.h" +#include "common/common/logger.h" #include "common/common/utility.h" #include "common/config/resources.h" #include "common/init/manager_impl.h" @@ -184,7 +184,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( : *overriding_init_manager)); auto scoped_route_info = std::make_shared( std::move(scoped_route_config), rds_config_provider_helper->routeConfig()); - // Detect if there is key conflict between two scopes. + // Detect if there is key conflict between two scopes, in which case Envoy won't be able to + // tell which RouteConfiguration to use. Reject the second scope in the delta form API. auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); if (iter != scope_name_by_hash_.end()) { if (iter->second != scoped_route_info->scopeName()) { diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 212b1d616335..41598d8b801b 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -3,13 +3,14 @@ #include #include "envoy/api/v2/srds.pb.h" +#include "envoy/common/callback.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/config/subscription.h" #include "envoy/router/route_config_provider_manager.h" -#include "common/init/manager_impl.h" #include "envoy/stats/scope.h" + #include "common/config/config_provider_impl.h" -#include "envoy/common/callback.h" +#include "common/init/manager_impl.h" #include "common/router/rds_impl.h" #include "common/router/scoped_config_impl.h" diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 5b07427f2f29..dfe3c4634806 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -2,13 +2,15 @@ #include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/admin/v2alpha/config_dump.pb.validate.h" +#include "envoy/config/subscription.h" #include "envoy/init/manager.h" #include "envoy/stats/scope.h" -#include "envoy/config/subscription.h" + #include "common/router/scoped_rds.h" + +#include "test/mocks/config/mocks.h" #include "test/mocks/router/mocks.h" #include "test/mocks/server/mocks.h" -#include "test/mocks/config/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" From c4719d84014301ab6806ff827df5fc59493776e6 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 21 Aug 2019 11:39:00 -0400 Subject: [PATCH 46/53] split SRDS onConfigUpdate into smaller functions and some minor comment udpates Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.cc | 3 +- source/common/router/scoped_rds.cc | 102 ++++++++++++--------- source/common/router/scoped_rds.h | 11 +++ 3 files changed, 71 insertions(+), 45 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index a9e030f9f7a8..c6f00f58ea1e 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -118,6 +118,7 @@ void ScopedConfigImpl::addOrUpdateRoutingScope( const ScopedRouteInfoConstSharedPtr& scoped_route_info) { const auto iter = scoped_route_info_by_name_.find(scoped_route_info->scopeName()); if (iter != scoped_route_info_by_name_.end()) { + ASSERT(scoped_route_info_by_key_.contains(iter->second->scopeKey().hash())); scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); } scoped_route_info_by_name_[scoped_route_info->scopeName()] = scoped_route_info; @@ -127,7 +128,7 @@ void ScopedConfigImpl::addOrUpdateRoutingScope( void ScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { const auto iter = scoped_route_info_by_name_.find(scope_name); if (iter != scoped_route_info_by_name_.end()) { - ASSERT(scoped_route_info_by_key_.count(iter->second->scopeKey().hash()) == 1); + ASSERT(scoped_route_info_by_key_.contains(iter->second->scopeKey().hash())); scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); scoped_route_info_by_name_.erase(iter); } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 64ccabb6c333..3f84e82da7c1 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -122,48 +122,15 @@ ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::RdsRouteConfigProvide parent_.onRdsConfigUpdate(scope_name_, route_provider_->subscription()); })) {} -void ScopedRdsConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& version_info) { +bool ScopedRdsConfigSubscription::addOrUpdateScopes( + const Protobuf::RepeatedPtrField& resources, + Init::Manager& init_manager, const std::string& version_info, + std::vector& exception_msgs) { bool any_applied = false; - std::vector exception_msgs; - absl::flat_hash_set unique_resource_names; envoy::config::filter::network::http_connection_manager::v2::Rds rds; rds.mutable_config_source()->MergeFrom(rds_config_source_); - - // If new route config sources come after the factory_context_.initManager()'s initialize() been - // called, that initManager can't accept new targets. Instead we use a local override which will - // start new subscriptions but not wait on them to be ready. - std::unique_ptr overriding_init_manager; - // NOTE: This should be defined after overriding_init_manager as it depends on the - // overriding_init_manager. - std::unique_ptr resume_rds; - if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { - overriding_init_manager = - std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); - // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. - // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either - // by Server init or LDS init. - factory_context_.clusterManager().adsMux().pause( - Envoy::Config::TypeUrl::get().RouteConfiguration); - resume_rds = std::make_unique([this, &overriding_init_manager, version_info] { - // For new RDS subscriptions created after listener warming up, we don't wait for them to warm - // up. - Init::WatcherImpl noop_watcher( - // Note: we just throw it away. - fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), - []() { /*Do nothing.*/ }); - overriding_init_manager->initialize(noop_watcher); - // New RDS subscriptions should have been created, now lift the floodgate. - // Note in the case of partial acceptance, accepted RDS subscriptions should be started - // dispite of any error. - factory_context_.clusterManager().adsMux().resume( - Envoy::Config::TypeUrl::get().RouteConfiguration); - }); - } - - for (const auto& resource : added_resources) { + absl::flat_hash_set unique_resource_names; + for (const auto& resource : resources) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; try { scoped_route_config = MessageUtil::anyConvert( @@ -178,10 +145,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // config-provider-framework to make it light weight. rds.set_route_config_name(scoped_route_config.route_configuration_name()); // Delete previous route provider if any. - auto rds_config_provider_helper = std::make_unique( - *this, scope_name, rds, - (overriding_init_manager == nullptr ? factory_context_.initManager() - : *overriding_init_manager)); + auto rds_config_provider_helper = + std::make_unique(*this, scope_name, rds, init_manager); auto scoped_route_info = std::make_shared( std::move(scoped_route_config), rds_config_provider_helper->routeConfig()); // Detect if there is key conflict between two scopes, in which case Envoy won't be able to @@ -211,8 +176,13 @@ void ScopedRdsConfigSubscription::onConfigUpdate( exception_msgs.emplace_back(fmt::format("{}", e.what())); } } + return any_applied; +} - for (const auto& scope_name : removed_resources) { +bool ScopedRdsConfigSubscription::removeScopes( + const Protobuf::RepeatedPtrField& scope_names, const std::string& version_info) { + bool any_applied = false; + for (const auto& scope_name : scope_names) { auto iter = scoped_route_map_.find(scope_name); if (iter != scoped_route_map_.end()) { route_provider_by_scope_.erase(scope_name); @@ -229,6 +199,50 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ENVOY_LOG(debug, "srds: remove scoped route '{}', version: {}", scope_name, version_info); } } + return any_applied; +} + +void ScopedRdsConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info) { + // If new route config sources come after the factory_context_.initManager()'s initialize() been + // called, that initManager can't accept new targets. Instead we use a local override which will + // start new subscriptions but not wait on them to be ready. + std::unique_ptr overriding_init_manager; + // NOTE: This should be defined after overriding_init_manager as it depends on the + // overriding_init_manager. + std::unique_ptr resume_rds; + if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { + overriding_init_manager = + std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); + // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. + // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either + // by Server init or LDS init. + factory_context_.clusterManager().adsMux().pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + resume_rds = std::make_unique([this, &overriding_init_manager, version_info] { + // For new RDS subscriptions created after listener warming up, we don't wait for them to warm + // up. + Init::WatcherImpl noop_watcher( + // Note: we just throw it away. + fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), + []() { /*Do nothing.*/ }); + overriding_init_manager->initialize(noop_watcher); + // New RDS subscriptions should have been created, now lift the floodgate. + // Note in the case of partial acceptance, accepted RDS subscriptions should be started + // despite of any error. + factory_context_.clusterManager().adsMux().resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + }); + } + std::vector exception_msgs; + bool any_applied = + addOrUpdateScopes(added_resources, + (overriding_init_manager == nullptr ? factory_context_.initManager() + : *overriding_init_manager), + version_info, exception_msgs) || + removeScopes(removed_resources, version_info); ConfigSubscriptionCommonBase::onConfigUpdate(); if (any_applied) { setLastConfigInfo(absl::optional({absl::nullopt, version_info})); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 41598d8b801b..bde1b1969ea7 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -121,6 +121,17 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio Common::CallbackHandle* rds_update_callback_handle_; }; + // Adds or updates scopes, create a new RDS provider for each resource, if an exception is thrown + // during updating, the exception message is collected via the exception_msgs vector. + // Returns true if any scope updated, false otherwise. + bool addOrUpdateScopes(const Protobuf::RepeatedPtrField& resources, + Init::Manager& init_manager, const std::string& version_info, + std::vector& exception_msgs); + // Removes given scopes from the managed set of scopes. + // Returns true if any scope updated, false otherwise. + bool removeScopes(const Protobuf::RepeatedPtrField& scope_names, + const std::string& version_info); + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } From 56e2876326b81b2352b5c2c9cc42ae32dbe5e746 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 21 Aug 2019 14:49:48 -0400 Subject: [PATCH 47/53] cleanup comment and remove unused header. Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 259246665c6f..6d083c164a40 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -1,7 +1,5 @@ #pragma once -#include - #include "envoy/config/config_provider.h" #include "envoy/config/config_provider_manager.h" #include "envoy/init/manager.h" @@ -146,9 +144,7 @@ class MutableConfigProviderCommonBase; * shared ownership of the underlying subscription. * */ -class ConfigSubscriptionCommonBase - : protected Logger::Loggable, - public std::enable_shared_from_this { +class ConfigSubscriptionCommonBase : protected Logger::Loggable { public: // Callback for updating a Config implementation held in each worker thread, the callback is // called in applyConfigUpdate() with the current version Config, and is expected to return the @@ -224,14 +220,11 @@ class ConfigSubscriptionCommonBase */ void applyConfigUpdate( const ConfigUpdateCb& update_fn, const Event::PostCb& complete_cb = []() {}) { - // During the update propagation, a subscription may get teared down in main thread due to - // all owners/providers destructed in a xDS update (e.g. LDS demolishes a - // RouteConfigProvider and its subscription). Capture a weak reference of "*this" and only apply - // the update if "*this" is still available. - // It is safe to call shared_from_this here as this is in main thread, and destruction of a - // ConfigSubscriptionCommonBase owner (i.e., a provider) happens in main thread as well. tls_->runOnAllThreads( [this, update_fn]() { + // NOTE: there is a known race condition between *this* subscription be teared down in + // main thread and the posted callback been executed before the destruction. See more + // details in https://github.com/envoyproxy/envoy/issues/7902 tls_->getTyped().config_ = update_fn(getConfig()); }, complete_cb); From d26d9b79dbede2424a19647fd35a131583b8f1a8 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Wed, 21 Aug 2019 16:29:56 -0400 Subject: [PATCH 48/53] fix a bug introduced in the most recent function splitting, adjust the comment around overriding init-manager Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 3f84e82da7c1..9e3c8079c1a1 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -209,26 +209,28 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // If new route config sources come after the factory_context_.initManager()'s initialize() been // called, that initManager can't accept new targets. Instead we use a local override which will // start new subscriptions but not wait on them to be ready. - std::unique_ptr overriding_init_manager; - // NOTE: This should be defined after overriding_init_manager as it depends on the - // overriding_init_manager. + // NOTE: For now we use a local init-manager, in the future when Envoy supports on-demand xDS, we + // will probably make this init-manager as a member of the subscription. + std::unique_ptr noop_init_manager; + // NOTE: This should be defined after noop_init_manager as it depends on the + // noop_init_manager. std::unique_ptr resume_rds; if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { - overriding_init_manager = + noop_init_manager = std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either // by Server init or LDS init. factory_context_.clusterManager().adsMux().pause( Envoy::Config::TypeUrl::get().RouteConfiguration); - resume_rds = std::make_unique([this, &overriding_init_manager, version_info] { + resume_rds = std::make_unique([this, &noop_init_manager, version_info] { // For new RDS subscriptions created after listener warming up, we don't wait for them to warm // up. Init::WatcherImpl noop_watcher( // Note: we just throw it away. fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), []() { /*Do nothing.*/ }); - overriding_init_manager->initialize(noop_watcher); + noop_init_manager->initialize(noop_watcher); // New RDS subscriptions should have been created, now lift the floodgate. // Note in the case of partial acceptance, accepted RDS subscriptions should be started // despite of any error. @@ -237,12 +239,11 @@ void ScopedRdsConfigSubscription::onConfigUpdate( }); } std::vector exception_msgs; - bool any_applied = - addOrUpdateScopes(added_resources, - (overriding_init_manager == nullptr ? factory_context_.initManager() - : *overriding_init_manager), - version_info, exception_msgs) || - removeScopes(removed_resources, version_info); + bool any_applied = addOrUpdateScopes( + added_resources, + (noop_init_manager == nullptr ? factory_context_.initManager() : *noop_init_manager), + version_info, exception_msgs); + any_applied = removeScopes(removed_resources, version_info) || any_applied; ConfigSubscriptionCommonBase::onConfigUpdate(); if (any_applied) { setLastConfigInfo(absl::optional({absl::nullopt, version_info})); From 4ab7005fadeabc1ce924cb064d53cc0f6cce375e Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 22 Aug 2019 08:54:12 -0400 Subject: [PATCH 49/53] fixes feedbacks from Andres Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.h | 4 ++-- source/common/router/scoped_rds.h | 4 +++- test/common/router/scoped_rds_test.cc | 19 ++++++++++--------- .../scoped_rds_integration_test.cc | 2 -- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 6d083c164a40..4a7499c5abdb 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -222,8 +222,8 @@ class ConfigSubscriptionCommonBase : protected Logger::LoggablerunOnAllThreads( [this, update_fn]() { - // NOTE: there is a known race condition between *this* subscription be teared down in - // main thread and the posted callback been executed before the destruction. See more + // NOTE: there is a known race condition between *this* subscription being teared down in + // main thread and the posted callback being executed before the destruction. See more // details in https://github.com/envoyproxy/envoy/issues/7902 tls_->getTyped().config_ = update_fn(getConfig()); }, diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index bde1b1969ea7..107921c738ad 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -105,7 +105,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: - // A helper class that takes care the life circle management of a RDS route provider and the + // A helper class that takes care the life cycle management of a RDS route provider and the // update callback handle. struct RdsRouteConfigProviderHelper { RdsRouteConfigProviderHelper( @@ -118,6 +118,8 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio ScopedRdsConfigSubscription& parent_; std::string scope_name_; std::unique_ptr route_provider_; + // This handle_ is owned by the route config provider's RDS subscrption, when the helper + // destructs, the handle is deleted as well. Common::CallbackHandle* rds_update_callback_handle_; }; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index dfe3c4634806..a01a494a0b7e 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -25,6 +25,7 @@ using testing::DoAll; using testing::Eq; using testing::InSequence; using testing::Invoke; +using testing::IsNull; using testing::NiceMock; using testing::Return; using testing::ReturnRefOfCopy; @@ -309,9 +310,9 @@ route_configuration_name: foo_routes 2UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); // now scope key "x-bar-key" points to nowhere. - EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( - TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), - nullptr); + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + IsNull()); EXPECT_EQ(getScopedRdsProvider() ->config() ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) @@ -388,9 +389,9 @@ route_configuration_name: foo_routes 2UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); // now scope key "x-bar-key" points to nowhere. - EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( - TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), - nullptr); + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + IsNull()); EXPECT_EQ(getScopedRdsProvider() ->config() ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) @@ -429,9 +430,9 @@ route_configuration_name: foo_routes // Scope key "x-foo-key" points to nowhere. EXPECT_NE(getScopedRdsProvider(), nullptr); EXPECT_NE(getScopedRdsProvider()->config(), nullptr); - EXPECT_EQ(getScopedRdsProvider()->config()->getRouteConfig( - TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), - nullptr); + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + IsNull()); factory_context_.init_manager_.initialize(init_watcher_); init_watcher_.expectReady().Times( 1); // Just SRDS, RDS "foo_routes" will initialized by the noop init-manager. diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 11ff06916cc3..e14e87d08fa7 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -219,8 +219,6 @@ route_configuration_name: foo_route1 // RDS updates won't affect SRDS. test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 6927017134761466251UL); - // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented - // at this point. } // Test that a bad config update updates the corresponding stats. From bf703fc74610b44a7e412d5e85f4c4fa25b0dcb3 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 22 Aug 2019 09:10:19 -0400 Subject: [PATCH 50/53] fix typo Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 107921c738ad..4ad686d5d4ef 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -118,13 +118,13 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio ScopedRdsConfigSubscription& parent_; std::string scope_name_; std::unique_ptr route_provider_; - // This handle_ is owned by the route config provider's RDS subscrption, when the helper + // This handle_ is owned by the route config provider's RDS subscription, when the helper // destructs, the handle is deleted as well. Common::CallbackHandle* rds_update_callback_handle_; }; // Adds or updates scopes, create a new RDS provider for each resource, if an exception is thrown - // during updating, the exception message is collected via the exception_msgs vector. + // during updating, the exception message is collected via the exception messages vector. // Returns true if any scope updated, false otherwise. bool addOrUpdateScopes(const Protobuf::RepeatedPtrField& resources, Init::Manager& init_manager, const std::string& version_info, From 2b375d42c7541eb91c9472a605fa66e53eae1e77 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 22 Aug 2019 12:25:40 -0400 Subject: [PATCH 51/53] fix nits from htuch feedbacks Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9e3c8079c1a1..d1a36f1b6361 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -144,7 +144,6 @@ bool ScopedRdsConfigSubscription::addOrUpdateScopes( // TODO(stevenzzz): Creating a new RdsRouteConfigProvider likely expensive, migrate RDS to // config-provider-framework to make it light weight. rds.set_route_config_name(scoped_route_config.route_configuration_name()); - // Delete previous route provider if any. auto rds_config_provider_helper = std::make_unique(*this, scope_name, rds, init_manager); auto scoped_route_info = std::make_shared( @@ -159,6 +158,7 @@ bool ScopedRdsConfigSubscription::addOrUpdateScopes( iter->second, scoped_route_info->scopeName())); } } + // NOTE: delete previous route provider if any. route_provider_by_scope_.insert({scope_name, std::move(rds_config_provider_helper)}); scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; @@ -285,7 +285,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( auto scoped_route = MessageUtil::anyConvert( resource_any, validation_visitor_); MessageUtil::validate(scoped_route); - const std::string scope_name = scoped_route.name(); + const std::string& scope_name = scoped_route.name(); auto scope_config_inserted = scoped_routes.try_emplace(scope_name, std::move(scoped_route)); if (!scope_config_inserted.second) { throw EnvoyException( From c7a241cf60f1835d5bb9c028924c971c91d6a7f3 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 22 Aug 2019 13:32:04 -0400 Subject: [PATCH 52/53] change back scope_name assignment to value, as protobuf returns a refernce instead of a string value alreay Signed-off-by: Xin Zhuang --- source/common/router/scoped_rds.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index d1a36f1b6361..8d82c91a8654 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -285,7 +285,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( auto scoped_route = MessageUtil::anyConvert( resource_any, validation_visitor_); MessageUtil::validate(scoped_route); - const std::string& scope_name = scoped_route.name(); + const std::string scope_name = scoped_route.name(); auto scope_config_inserted = scoped_routes.try_emplace(scope_name, std::move(scoped_route)); if (!scope_config_inserted.second) { throw EnvoyException( From c2ff01596de58fdbf462db4feb5eff55cf956d48 Mon Sep 17 00:00:00 2001 From: Xin Zhuang Date: Thu, 22 Aug 2019 15:04:17 -0400 Subject: [PATCH 53/53] fix nits Signed-off-by: Xin Zhuang --- source/common/router/scoped_config_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index f6197f09abdc..91e67e841b14 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -152,7 +152,7 @@ class ScopedRouteInfo { ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, ConfigConstSharedPtr&& route_config); - ConfigConstSharedPtr routeConfig() const { return route_config_; } + const ConfigConstSharedPtr& routeConfig() const { return route_config_; } const ScopeKey& scopeKey() const { return scope_key_; } const envoy::api::v2::ScopedRouteConfiguration& configProto() const { return config_proto_; } const std::string& scopeName() const { return config_proto_.name(); }