Skip to content

Commit f101352

Browse files
ashbkaxil
authored andcommitted
[AIRFLOW-XXX] Speed up DagBagTest cases (#3974)
I noticed that many of the tests of DagBags operate on a specific DAG only, and don't need to load the example or test dags. By not loading the dags we don't need to this shaves about 10-20s of test time.
1 parent a85fe0b commit f101352

File tree

1 file changed

+14
-7
lines changed

1 file changed

+14
-7
lines changed

tests/models.py

+14-7
Original file line numberDiff line numberDiff line change
@@ -1159,12 +1159,19 @@ def with_all_tasks_removed(dag):
11591159

11601160

11611161
class DagBagTest(unittest.TestCase):
1162+
@classmethod
1163+
def setUpClass(cls):
1164+
cls.empty_dir = mkdtemp()
1165+
1166+
@classmethod
1167+
def tearDownClass(cls):
1168+
os.rmdir(cls.empty_dir)
11621169

11631170
def test_get_existing_dag(self):
11641171
"""
11651172
test that were're able to parse some example DAGs and retrieve them
11661173
"""
1167-
dagbag = models.DagBag(include_examples=True)
1174+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=True)
11681175

11691176
some_expected_dag_ids = ["example_bash_operator",
11701177
"example_branch_operator"]
@@ -1181,7 +1188,7 @@ def test_get_non_existing_dag(self):
11811188
"""
11821189
test that retrieving a non existing dag id returns None without crashing
11831190
"""
1184-
dagbag = models.DagBag(include_examples=True)
1191+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
11851192

11861193
non_existing_dag_id = "non_existing_dag_id"
11871194
self.assertIsNone(dagbag.get_dag(non_existing_dag_id))
@@ -1194,7 +1201,7 @@ def test_process_file_that_contains_multi_bytes_char(self):
11941201
f.write('\u3042'.encode('utf8')) # write multi-byte char (hiragana)
11951202
f.flush()
11961203

1197-
dagbag = models.DagBag(include_examples=True)
1204+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
11981205
self.assertEqual([], dagbag.process_file(f.name))
11991206

12001207
def test_zip_skip_log(self):
@@ -1216,7 +1223,7 @@ def test_zip(self):
12161223
"""
12171224
test the loading of a DAG within a zip file that includes dependencies
12181225
"""
1219-
dagbag = models.DagBag()
1226+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
12201227
dagbag.process_file(os.path.join(TEST_DAGS_FOLDER, "test_zip.zip"))
12211228
self.assertTrue(dagbag.get_dag("test_zip_dag"))
12221229

@@ -1226,7 +1233,7 @@ def test_process_file_cron_validity_check(self):
12261233
as schedule interval can be identified
12271234
"""
12281235
invalid_dag_files = ["test_invalid_cron.py", "test_zip_invalid_cron.zip"]
1229-
dagbag = models.DagBag(dag_folder=mkdtemp())
1236+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
12301237

12311238
self.assertEqual(len(dagbag.import_errors), 0)
12321239
for d in invalid_dag_files:
@@ -1290,7 +1297,7 @@ def process_dag(self, create_dag):
12901297
f.write(source.encode('utf8'))
12911298
f.flush()
12921299

1293-
dagbag = models.DagBag(include_examples=False)
1300+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
12941301
found_dags = dagbag.process_file(f.name)
12951302
return dagbag, found_dags, f.name
12961303

@@ -1601,7 +1608,7 @@ def test_process_file_with_none(self):
16011608
"""
16021609
test that process_file can handle Nones
16031610
"""
1604-
dagbag = models.DagBag(include_examples=True)
1611+
dagbag = models.DagBag(dag_folder=self.empty_dir, include_examples=False)
16051612

16061613
self.assertEqual([], dagbag.process_file(None))
16071614

0 commit comments

Comments
 (0)