Add gateway route controller (#256)

* Add gateway route controller

Co-authored-by: Karan Vasnani <vasnk@amazon.com>

Author: Fawad Khaliq

README.md

@@ -7,7 +7,7 @@
 
 ## AWS App Mesh Controller For K8s
 
-AWS App Mesh Controller For K8s is a controller to help manage [App Mesh](https://aws.amazon.com/app-mesh/) resources for a Kubernetes cluster.  The controller watches custom resources for changes and reflects those changes into the [App Mesh API](https://docs.aws.amazon.com/app-mesh/latest/APIReference/Welcome.html). It is accompanied by the deployment of three custom resource definitions ([CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)): meshes, virtualnodes, and virtualservices.  These map to App Mesh API objects which the controller manages for you.
+AWS App Mesh Controller For K8s is a controller to help manage [App Mesh](https://aws.amazon.com/app-mesh/) resources for a Kubernetes cluster.  The controller watches custom resources for changes and reflects those changes into the [App Mesh API](https://docs.aws.amazon.com/app-mesh/latest/APIReference/Welcome.html). It is accompanied by the deployment of three custom resource definitions ([CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)): meshes, virtualnodes, virtualservices, virtualgateways and gatewayroutes.  These map to App Mesh API objects which the controller manages for you.
 
 ## Documentation
 Checkout our [Live Docs](https://aws.github.io/aws-app-mesh-controller-for-k8s/)!

config/crd/kustomization.yaml

@@ -7,7 +7,7 @@ resources:
 - bases/appmesh.k8s.aws_virtualnodes.yaml
 - bases/appmesh.k8s.aws_virtualrouters.yaml
 - bases/appmesh.k8s.aws_virtualgateways.yaml
-#- bases/appmesh.k8s.aws_gatewayroutes.yaml
+- bases/appmesh.k8s.aws_gatewayroutes.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/rbac/role.yaml

@@ -50,6 +50,26 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - appmesh.k8s.aws
+  resources:
+  - gatewayroutes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - appmesh.k8s.aws
+  resources:
+  - gatewayroutes/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - appmesh.k8s.aws
   resources:

config/webhook/manifests.yaml

@@ -6,6 +6,24 @@ metadata:
   creationTimestamp: null
   name: mutating-webhook-configuration
 webhooks:
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-appmesh-k8s-aws-v1beta2-gatewayroute
+  failurePolicy: Fail
+  name: mgatewayroute.appmesh.k8s.aws
+  rules:
+  - apiGroups:
+    - appmesh.k8s.aws
+    apiVersions:
+    - v1beta2
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - gatewayroutes
 - clientConfig:
     caBundle: Cg==
     service:
@@ -121,6 +139,24 @@ metadata:
   creationTimestamp: null
   name: validating-webhook-configuration
 webhooks:
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-appmesh-k8s-aws-v1beta2-gatewayroute
+  failurePolicy: Fail
+  name: vgatewayroute.appmesh.k8s.aws
+  rules:
+  - apiGroups:
+    - appmesh.k8s.aws
+    apiVersions:
+    - v1beta2
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - gatewayroutes
 - clientConfig:
     caBundle: Cg==
     service:

controllers/appmesh/gatewayroute_controller.go

@@ -18,36 +18,92 @@ package controllers
 
 import (
 	"context"
-
+	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/gatewayroute"
+	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/k8s"
+	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/runtime"
 	"github.com/go-logr/logr"
-	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
-	appmeshv1beta2 "github.com/aws/aws-app-mesh-controller-for-k8s/apis/appmesh/v1beta2"
+	appmesh "github.com/aws/aws-app-mesh-controller-for-k8s/apis/appmesh/v1beta2"
 )
 
-// GatewayRouteReconciler reconciles a GatewayRoute object
-type GatewayRouteReconciler struct {
-	client.Client
-	Log    logr.Logger
-	Scheme *runtime.Scheme
+// NewGatewayRouteReconciler constructs new gatewayRouteReconciler
+func NewGatewayRouteReconciler(
+	k8sClient client.Client,
+	finalizerManager k8s.FinalizerManager,
+	grResManager gatewayroute.ResourceManager,
+	log logr.Logger) *gatewayRouteReconciler {
+	return &gatewayRouteReconciler{
+		k8sClient:                              k8sClient,
+		finalizerManager:                       finalizerManager,
+		grResManager:                           grResManager,
+		enqueueRequestsForMeshEvents:           gatewayroute.NewEnqueueRequestsForMeshEvents(k8sClient, log),
+		enqueueRequestsForVirtualGatewayEvents: gatewayroute.NewEnqueueRequestsForVirtualGatewayEvents(k8sClient, log),
+		log:                                    log,
+	}
 }
 
-//// +kubebuilder:rbac:groups=appmesh.k8s.aws,resources=gatewayroutes,verbs=get;list;watch;create;update;patch;delete
-//// +kubebuilder:rbac:groups=appmesh.k8s.aws,resources=gatewayroutes/status,verbs=get;update;patch
+// gatewayRouteReconciler reconciles a GatewayRoute object
+type gatewayRouteReconciler struct {
+	k8sClient        client.Client
+	finalizerManager k8s.FinalizerManager
+	grResManager     gatewayroute.ResourceManager
 
-func (r *GatewayRouteReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
-	_ = context.Background()
-	_ = r.Log.WithValues("gatewayroute", req.NamespacedName)
+	enqueueRequestsForMeshEvents           handler.EventHandler
+	enqueueRequestsForVirtualGatewayEvents handler.EventHandler
+	log                                    logr.Logger
+}
 
-	// your logic here
+// +kubebuilder:rbac:groups=appmesh.k8s.aws,resources=gatewayroutes,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=appmesh.k8s.aws,resources=gatewayroutes/status,verbs=get;update;patch
 
-	return ctrl.Result{}, nil
+func (r *gatewayRouteReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	return runtime.HandleReconcileError(r.reconcile(req), r.log)
 }
 
-func (r *GatewayRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *gatewayRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&appmeshv1beta2.GatewayRoute{}).
+		For(&appmesh.GatewayRoute{}).
+		Watches(&source.Kind{Type: &appmesh.Mesh{}}, r.enqueueRequestsForMeshEvents).
+		Watches(&source.Kind{Type: &appmesh.VirtualGateway{}}, r.enqueueRequestsForVirtualGatewayEvents).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
 		Complete(r)
 }
+
+func (r *gatewayRouteReconciler) reconcile(req ctrl.Request) error {
+	ctx := context.Background()
+	gr := &appmesh.GatewayRoute{}
+	if err := r.k8sClient.Get(ctx, req.NamespacedName, gr); err != nil {
+		return client.IgnoreNotFound(err)
+	}
+	if !gr.DeletionTimestamp.IsZero() {
+		return r.cleanupGatewayRoute(ctx, gr)
+	}
+	return r.reconcileGatewayRoute(ctx, gr)
+}
+
+func (r *gatewayRouteReconciler) reconcileGatewayRoute(ctx context.Context, gr *appmesh.GatewayRoute) error {
+	if err := r.finalizerManager.AddFinalizers(ctx, gr, k8s.FinalizerAWSAppMeshResources); err != nil {
+		return err
+	}
+	if err := r.grResManager.Reconcile(ctx, gr); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *gatewayRouteReconciler) cleanupGatewayRoute(ctx context.Context, gr *appmesh.GatewayRoute) error {
+	if k8s.HasFinalizer(gr, k8s.FinalizerAWSAppMeshResources) {
+		if err := r.grResManager.Cleanup(ctx, gr); err != nil {
+			return err
+		}
+		if err := r.finalizerManager.RemoveFinalizers(ctx, gr, k8s.FinalizerAWSAppMeshResources); err != nil {
+			return err
+		}
+	}
+	return nil
+}

main.go

@@ -38,6 +38,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/gatewayroute"
 	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/inject"
 	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/mesh"
 	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/virtualgateway"
@@ -127,12 +128,14 @@ func main() {
 	cloudMapInstancesReconciler := cloudmap.NewDefaultInstancesReconciler(mgr.GetClient(), cloud.CloudMap(), ctrl.Log, stopChan)
 	meshResManager := mesh.NewDefaultResourceManager(mgr.GetClient(), cloud.AppMesh(), cloud.AccountID(), ctrl.Log)
 	vgResManager := virtualgateway.NewDefaultResourceManager(mgr.GetClient(), cloud.AppMesh(), referencesResolver, cloud.AccountID(), ctrl.Log)
+	grResManager := gatewayroute.NewDefaultResourceManager(mgr.GetClient(), cloud.AppMesh(), referencesResolver, cloud.AccountID(), ctrl.Log)
 	vnResManager := virtualnode.NewDefaultResourceManager(mgr.GetClient(), cloud.AppMesh(), referencesResolver, cloud.AccountID(), ctrl.Log)
 	vsResManager := virtualservice.NewDefaultResourceManager(mgr.GetClient(), cloud.AppMesh(), referencesResolver, cloud.AccountID(), ctrl.Log)
 	vrResManager := virtualrouter.NewDefaultResourceManager(mgr.GetClient(), cloud.AppMesh(), referencesResolver, cloud.AccountID(), ctrl.Log)
 	cloudMapResManager := cloudmap.NewDefaultResourceManager(mgr.GetClient(), cloud.CloudMap(), referencesResolver, virtualNodeEndpointResolver, cloudMapInstancesReconciler, enableCustomHealthCheck, ctrl.Log)
 	msReconciler := appmeshcontroller.NewMeshReconciler(mgr.GetClient(), finalizerManager, meshMembersFinalizer, meshResManager, ctrl.Log.WithName("controllers").WithName("Mesh"))
 	vgReconciler := appmeshcontroller.NewVirtualGatewayReconciler(mgr.GetClient(), finalizerManager, vgMembersFinalizer, vgResManager, ctrl.Log.WithName("controllers").WithName("VirtualGateway"))
+	grReconciler := appmeshcontroller.NewGatewayRouteReconciler(mgr.GetClient(), finalizerManager, grResManager, ctrl.Log.WithName("controllers").WithName("GatewayRoute"))
 	vnReconciler := appmeshcontroller.NewVirtualNodeReconciler(mgr.GetClient(), finalizerManager, vnResManager, ctrl.Log.WithName("controllers").WithName("VirtualNode"))
 	cloudMapReconciler := appmeshcontroller.NewCloudMapReconciler(mgr.GetClient(), finalizerManager, cloudMapResManager, ctrl.Log.WithName("controllers").WithName("CloudMap"))
 	vsReconciler := appmeshcontroller.NewVirtualServiceReconciler(mgr.GetClient(), finalizerManager, referencesIndexer, vsResManager, ctrl.Log.WithName("controllers").WithName("VirtualService"))
@@ -145,12 +148,14 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "VirtualService")
 		os.Exit(1)
 	}
-
 	if err = vgReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "VirtualGateway")
 		os.Exit(1)
 	}
-
+	if err = grReconciler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "GatewayRoute")
+		os.Exit(1)
+	}
 	if err = vnReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "VirtualNode")
 		os.Exit(1)
@@ -165,15 +170,15 @@ func main() {
 	}
 
 	meshMembershipDesignator := mesh.NewMembershipDesignator(mgr.GetClient())
-	//vgMembershipDesignator := virtualgateway.NewMembershipDesignator(mgr.GetClient())
+	vgMembershipDesignator := virtualgateway.NewMembershipDesignator(mgr.GetClient())
 	vnMembershipDesignator := virtualnode.NewMembershipDesignator(mgr.GetClient())
 	sidecarInjector := inject.NewSidecarInjector(injectConfig, cloud.Region(), mgr.GetClient(), referencesResolver, vnMembershipDesignator)
 	appmeshwebhook.NewMeshMutator().SetupWithManager(mgr)
 	appmeshwebhook.NewMeshValidator().SetupWithManager(mgr)
 	appmeshwebhook.NewVirtualGatewayMutator(meshMembershipDesignator).SetupWithManager(mgr)
 	appmeshwebhook.NewVirtualGatewayValidator().SetupWithManager(mgr)
-	//appmeshwebhook.NewGatewayRouteMutator(meshMembershipDesignator, vgMembershipDesignator).SetupWithManager(mgr)
-	//appmeshwebhook.NewGatewayRouteValidator().SetupWithManager(mgr)
+	appmeshwebhook.NewGatewayRouteMutator(meshMembershipDesignator, vgMembershipDesignator).SetupWithManager(mgr)
+	appmeshwebhook.NewGatewayRouteValidator().SetupWithManager(mgr)
 	appmeshwebhook.NewVirtualNodeMutator(meshMembershipDesignator).SetupWithManager(mgr)
 	appmeshwebhook.NewVirtualNodeValidator().SetupWithManager(mgr)
 	appmeshwebhook.NewVirtualServiceMutator(meshMembershipDesignator).SetupWithManager(mgr)

pkg/gatewayroute/enqueue_requests_for_mesh_events.go

@@ -0,0 +1,71 @@
+package gatewayroute
+
+import (
+	"context"
+	appmesh "github.com/aws/aws-app-mesh-controller-for-k8s/apis/appmesh/v1beta2"
+	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/k8s"
+	"github.com/aws/aws-app-mesh-controller-for-k8s/pkg/mesh"
+	"github.com/go-logr/logr"
+	"k8s.io/client-go/util/workqueue"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+)
+
+func NewEnqueueRequestsForMeshEvents(k8sClient client.Client, log logr.Logger) *enqueueRequestsForMeshEvents {
+	return &enqueueRequestsForMeshEvents{
+		k8sClient: k8sClient,
+		log:       log,
+	}
+}
+
+var _ handler.EventHandler = (*enqueueRequestsForMeshEvents)(nil)
+
+type enqueueRequestsForMeshEvents struct {
+	k8sClient client.Client
+	log       logr.Logger
+}
+
+// Create is called in response to an create event
+func (h *enqueueRequestsForMeshEvents) Create(e event.CreateEvent, queue workqueue.RateLimitingInterface) {
+	// no-op
+}
+
+// Update is called in response to an update event
+func (h *enqueueRequestsForMeshEvents) Update(e event.UpdateEvent, queue workqueue.RateLimitingInterface) {
+	// gatewayRoute reconcile depends on mesh is active or not.
+	// so we only need to trigger gatewayRoute reconcile if mesh's active status changed.
+	msOld := e.ObjectOld.(*appmesh.Mesh)
+	msNew := e.ObjectNew.(*appmesh.Mesh)
+
+	if mesh.IsMeshActive(msOld) != mesh.IsMeshActive(msNew) {
+		h.enqueueGatewayRoutesForMesh(context.Background(), queue, msNew)
+	}
+}
+
+// Delete is called in response to a delete event
+func (h *enqueueRequestsForMeshEvents) Delete(e event.DeleteEvent, queue workqueue.RateLimitingInterface) {
+	// no-op
+}
+
+// Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or
+// external trigger request
+func (h *enqueueRequestsForMeshEvents) Generic(e event.GenericEvent, queue workqueue.RateLimitingInterface) {
+	// no-op
+}
+
+func (h *enqueueRequestsForMeshEvents) enqueueGatewayRoutesForMesh(ctx context.Context, queue workqueue.RateLimitingInterface, ms *appmesh.Mesh) {
+	grList := &appmesh.GatewayRouteList{}
+	if err := h.k8sClient.List(ctx, grList); err != nil {
+		h.log.Error(err, "failed to enqueue gatewayRoutes for mesh events",
+			"mesh", k8s.NamespacedName(ms))
+		return
+	}
+	for _, gr := range grList.Items {
+		if gr.Spec.MeshRef == nil || !mesh.IsMeshReferenced(ms, *gr.Spec.MeshRef) {
+			continue
+		}
+		queue.Add(ctrl.Request{NamespacedName: k8s.NamespacedName(&gr)})
+	}
+}