Add back aws api cache

This commits reverts changes made on commit b5e3324
but gives the users the ability to enable the cache and set the cache TTL, by default
is turned off.

Related to: #1100

Author: jescarri

cmd/main.go

@@ -26,9 +26,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
+	"github.com/ticketmaster/aws-sdk-go-cache/cache"
+
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/aws"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/controller"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/metric"
+	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/metric/collectors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -83,13 +88,18 @@ func main() {
 	reg.MustRegister(prometheus.NewGoCollector())
 	reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
 
+	cc := cache.NewConfig(options.SdkCacheDuration)
+	cc.SetCacheTTL(resourcegroupstaggingapi.ServiceName, "GetResources", time.Hour)
+	cc.SetCacheTTL(ec2.ServiceName, "DescribeInstanceStatus", time.Minute)
+	reg.MustRegister(cc.NewCacheCollector(collectors.PrometheusNamespace))
+
 	mc, err := metric.NewCollector(reg, options.ingressCTLConfig.IngressClass)
 	if err != nil {
 		glog.Fatal(err)
 	}
 	mc.Start()
 
-	cloud, err := aws.New(options.cloudConfig, options.ingressCTLConfig.ClusterName, mc)
+	cloud, err := aws.New(options.cloudConfig, options.ingressCTLConfig.ClusterName, mc, options.EnableSdkCache, cc)
 	if err != nil {
 		glog.Fatal(err)
 	}

cmd/options.go

@@ -42,6 +42,8 @@ const (
 	defaultHealthCheckPeriod       = 1 * time.Minute
 	defaultHealthzPort             = 10254
 	defaultProfilingEnabled        = true
+	defaultEnableSdkCache          = false
+	defaultSdkCacheDuration        = 5 * time.Minute
 )
 
 // Options defines the commandline interface of this binary
@@ -66,6 +68,10 @@ type Options struct {
 
 	// ingress controller specific configuration
 	ingressCTLConfig config.Configuration
+
+	// aws sdk cache options
+	EnableSdkCache   bool
+	SdkCacheDuration time.Duration
 }
 
 func (options *Options) BindFlags(fs *pflag.FlagSet) {
@@ -95,6 +101,8 @@ func (options *Options) BindFlags(fs *pflag.FlagSet) {
 		`Port to use for the healthz endpoint.`)
 	fs.BoolVar(&options.ProfilingEnabled, "profiling", defaultProfilingEnabled,
 		`Enable profiling via web interface host:port/debug/pprof/`)
+	fs.BoolVar(&options.EnableSdkCache, "aws-cache-enable", defaultEnableSdkCache, "Enables AWS SDK Caching")
+	fs.DurationVar(&options.SdkCacheDuration, "aws-cache-duration", defaultSdkCacheDuration, "Duration of AWS SDK Cache entries, default 5m")
 	options.cloudConfig.BindFlags(fs)
 	options.ingressCTLConfig.BindFlags(fs)
 

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/prometheus/common v0.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	github.com/ticketmaster/aws-sdk-go-cache v0.0.0-20180926195306-58922816129c // indirect
+	github.com/ticketmaster/aws-sdk-go-cache v0.0.0-20200114210642-9a510f7c39db
 	golang.org/x/oauth2 v0.0.0-20190212230446-3e8b2be13635 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	k8s.io/api v0.0.0-20181213150558-05914d821849

go.sum

@@ -175,6 +175,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karlseguin/ccache v2.0.2+incompatible h1:MpSlLlHgG3vPWTAIJsSYlyAQsHwfQ2HzgUlbJFh9Ufk=
 github.com/karlseguin/ccache v2.0.2+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
 github.com/karlseguin/expect v1.0.1/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -295,6 +296,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/ticketmaster/aws-sdk-go-cache v0.0.0-20180926195306-58922816129c/go.mod h1:H9sbOivuFYIUAS9No3MxP7K6WXz8i8Xg4qRJ/nu3zM4=
+github.com/ticketmaster/aws-sdk-go-cache v0.0.0-20200114210642-9a510f7c39db h1:6Powh9OH1h71lc0723hF3TXX3GWoZpSg+DwPWarZDEI=
+github.com/ticketmaster/aws-sdk-go-cache v0.0.0-20200114210642-9a510f7c39db/go.mod h1:DrzEMzgFXDUd2FVZals3Pv9cvdE007ggKR1qnvWzTk8=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=

internal/aws/cloud.go

@@ -18,6 +18,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/wafregional"
 	"github.com/aws/aws-sdk-go/service/wafregional/wafregionaliface"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/metric"
+	"github.com/ticketmaster/aws-sdk-go-cache/cache"
 )
 
 type CloudAPI interface {
@@ -49,8 +50,8 @@ type Cloud struct {
 // But due to huge number of aws clients, it's best to have one container AWS client that embed these aws clients.
 // TODO: remove clusterName dependency
 // TODO: remove mc dependency like https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/aws/aws_metrics.go
-func New(cfg CloudConfig, clusterName string, mc metric.Collector) (CloudAPI, error) {
-	awsSession := NewSession(&aws.Config{MaxRetries: aws.Int(cfg.APIMaxRetries)}, cfg.APIDebug, mc)
+func New(cfg CloudConfig, clusterName string, mc metric.Collector, ce bool, cc *cache.Config) (CloudAPI, error) {
+	awsSession := NewSession(&aws.Config{MaxRetries: aws.Int(cfg.APIMaxRetries)}, cfg.APIDebug, mc, ce, cc)
 	metadata := ec2metadata.New(awsSession)
 
 	if len(cfg.VpcID) == 0 {

internal/aws/session.go

@@ -10,17 +10,21 @@ import (
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/metric"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/pkg/util/log"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/ticketmaster/aws-sdk-go-cache/cache"
 )
 
 // NewSession returns an AWS session based off of the provided AWS config
-func NewSession(awsconfig *aws.Config, AWSDebug bool, mc metric.Collector) *session.Session {
+func NewSession(awsconfig *aws.Config, AWSDebug bool, mc metric.Collector, ce bool, cc *cache.Config) *session.Session {
 	session, err := session.NewSession(awsconfig)
 	if err != nil {
 		mc.IncAPIErrorCount(prometheus.Labels{"service": "AWS", "request": "NewSession"})
 		glog.ErrorDepth(4, fmt.Sprintf("Failed to create AWS session: %s", err.Error()))
 		return nil
 	}
-
+	if ce {
+		// Adds caching to session if cache is enabled
+		cache.AddCaching(session, cc)
+	}
 	session.Handlers.Retry.PushFront(func(r *request.Request) {
 		mc.IncAPIRetryCount(prometheus.Labels{"service": r.ClientInfo.ServiceName, "operation": r.Operation.Name})
 	})

test/e2e/framework/framework.go

@@ -2,6 +2,7 @@ package framework
 
 import (
 	"context"
+	"time"
 
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/aws"
 	"github.com/kubernetes-sigs/aws-alb-ingress-controller/internal/ingress/metric"
@@ -10,6 +11,7 @@ import (
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/ticketmaster/aws-sdk-go-cache/cache"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -54,10 +56,11 @@ func (f *Framework) BeforeEach() {
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	}
 	if f.Cloud == nil {
+		cc := cache.NewConfig(0 * time.Millisecond)
 		reg := prometheus.NewRegistry()
 		mc, _ := metric.NewCollector(reg, "alb")
 		var err error
-		f.Cloud, err = aws.New(aws.CloudConfig{Region: f.Options.AWSRegion, VpcID: f.Options.AWSVPCID}, f.Options.ClusterName, mc)
+		f.Cloud, err = aws.New(aws.CloudConfig{Region: f.Options.AWSRegion, VpcID: f.Options.AWSVPCID}, f.Options.ClusterName, mc, false, cc)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	}