Merge pull request #23366 from eddiezane/ez/add-retry-to-eks-cluster-delete

Add retry to eks cluster delete

Author: anGie44

.changelog/23366.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+resource/aws_eks_cluster: Retry when `ResourceInUseException` errors are returned from the AWS API during resource deletion
+```

.semgrep.yml

@@ -410,7 +410,7 @@ rules:
 
   - id: helper-schema-TimeoutError-check-doesnt-return-output
     languages: [go]
-    message: If the resource.Retry() or resource.RetryContext() function returns a value, ensure the isResourceTimeoutError() check does as well
+    message: If the resource.Retry(), resource.RetryContext(), or tfresource.RetryConfigContext() function returns a value, ensure the isResourceTimeoutError() check does as well
     paths:
       exclude:
         - "*_test.go"
@@ -439,6 +439,14 @@ rules:
               })
               ...
               if isResourceTimeoutError($ERR) { ... }
+          - pattern-not-inside: |
+              $ERR = tfresource.RetryConfigContext(..., func() *resource.RetryError {
+                ...
+                _, $ERR2 = $CONN.$FUNC(...)
+                ...
+              })
+              ...
+              if tfresource.TimedOut($ERR) { ... }
         - patterns:
           - pattern: |
               if tfresource.TimedOut($ERR) {
@@ -460,6 +468,14 @@ rules:
               })
               ...
               if tfresource.TimedOut($ERR) { ... }
+          - pattern-not-inside: |
+              $ERR = tfresource.RetryConfigContext(..., func() *resource.RetryError {
+                ...
+                _, $ERR2 = $CONN.$FUNC(...)
+                ...
+              })
+              ...
+              if tfresource.TimedOut($ERR) { ... }
     severity: WARNING
 
   - id: is-not-found-error

internal/service/eks/cluster.go

@@ -498,10 +498,34 @@ func resourceClusterDelete(d *schema.ResourceData, meta interface{}) error {
 	conn := meta.(*conns.AWSClient).EKSConn
 
 	log.Printf("[DEBUG] Deleting EKS Cluster: %s", d.Id())
-	_, err := conn.DeleteCluster(&eks.DeleteClusterInput{
+
+	input := &eks.DeleteClusterInput{
 		Name: aws.String(d.Id()),
+	}
+
+	// If a cluster is scaling up due to load a delete request will fail
+	// This is a temporary workaround until EKS supports multiple parallel mutating operations
+	err := tfresource.RetryConfigContext(context.Background(), 0*time.Second, 1*time.Minute, 0*time.Second, 30*time.Second, clusterDeleteRetryTimeout, func() *resource.RetryError {
+		var err error
+
+		_, err = conn.DeleteCluster(input)
+
+		if tfawserr.ErrMessageContains(err, eks.ErrCodeResourceInUseException, "in progress") {
+			log.Printf("[DEBUG] eks cluster update in progress: %v", err)
+			return resource.RetryableError(err)
+		}
+
+		if err != nil {
+			return resource.NonRetryableError(err)
+		}
+
+		return nil
 	})
 
+	if tfresource.TimedOut(err) {
+		_, err = conn.DeleteCluster(input)
+	}
+
 	if tfawserr.ErrCodeEquals(err, eks.ErrCodeResourceNotFoundException) {
 		return nil
 	}
@@ -516,9 +540,7 @@ func resourceClusterDelete(d *schema.ResourceData, meta interface{}) error {
 		return fmt.Errorf("error deleting EKS Cluster (%s): %w", d.Id(), err)
 	}
 
-	_, err = waitClusterDeleted(conn, d.Id(), d.Timeout(schema.TimeoutDelete))
-
-	if err != nil {
+	if _, err = waitClusterDeleted(conn, d.Id(), d.Timeout(schema.TimeoutDelete)); err != nil {
 		return fmt.Errorf("error waiting for EKS Cluster (%s) to delete: %w", d.Id(), err)
 	}
 

internal/service/eks/wait.go

@@ -14,6 +14,8 @@ const (
 	addonCreatedTimeout = 20 * time.Minute
 	addonUpdatedTimeout = 20 * time.Minute
 	addonDeletedTimeout = 40 * time.Minute
+
+	clusterDeleteRetryTimeout = 60 * time.Minute
 )
 
 func waitAddonCreated(ctx context.Context, conn *eks.EKS, clusterName, addonName string) (*eks.Addon, error) {