From 449b19734b5a8febe9d5f56f21402f5827a50f23 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 19 Sep 2024 12:46:32 -0700 Subject: [PATCH] AWS System Test: EMR Container Ensure that the EMR cluster stays running for long enough for all the tasks in the DAG to complete their operations against the cluster. --- tests/system/providers/amazon/aws/example_emr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/system/providers/amazon/aws/example_emr.py b/tests/system/providers/amazon/aws/example_emr.py index 507ff8588a863..7d62180d07d9c 100644 --- a/tests/system/providers/amazon/aws/example_emr.py +++ b/tests/system/providers/amazon/aws/example_emr.py @@ -83,7 +83,10 @@ "InstanceCount": 1, }, ], - "KeepJobFlowAliveWhenNoSteps": False, + # If the EMR steps complete too quickly the cluster will be torn down before the other system test + # tasks have a chance to run (such as the modify cluster step, the addition of more EMR steps, etc). + # Set KeepJobFlowAliveWhenNoSteps to False to avoid the cluster from being torn down prematurely. + "KeepJobFlowAliveWhenNoSteps": True, "TerminationProtected": False, }, "Steps": SPARK_STEPS,