fix(nwp): Test with concurrency

openclimatefix · Jan 27, 2025 · cc15b5b · cc15b5b
1 parent 3b4d156
commit cc15b5b
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 5 deletions.
diff --git a/src/dagster_dags/assets/nwp/ceda_mo_um_global.py b/src/dagster_dags/assets/nwp/ceda_mo_um_global.py
@@ -64,10 +64,13 @@ def ceda_mo_um_global_asset(
             "NWP_CONSUMER_NOTIFICATION_REPOSITORY": "dagster-pipes",
             "CEDA_FTP_USER": os.environ["CEDA_FTP_USER"],
             "CEDA_FTP_PASS": os.environ["CEDA_FTP_PASS"],
-            "CONCURRENCY": "false",
+            "CONCURRENCY": "true",
         },
+        # See https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run
         container_kwargs={
             "volumes": [f"{ARCHIVE_FOLDER}:/work"],
+            "mem_limit": "8g",
+            "nano_cpus": 4e9,
         },
         context=context,
     ).get_results()

diff --git a/src/dagster_dags/assets/nwp/ecmwf_ens_stat_india.py b/src/dagster_dags/assets/nwp/ecmwf_ens_stat_india.py
@@ -68,10 +68,13 @@ def ecmwf_ens_stat_india_asset(
             "ECMWF_API_KEY": os.environ["ECMWF_API_KEY"],
             "ECMWF_API_EMAIL": os.environ["ECMWF_API_EMAIL"],
             "ECMWF_API_URL": os.environ["ECMWF_API_URL"],
-            "CONCURRENCY": "false",
+            "CONCURRENCY": "true",
         },
+        # See https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run
         container_kwargs={
             "volumes": [f"{ARCHIVE_FOLDER}:/work"],
+            "mem_limit": "8g",
+            "nano_cpus": 4e9,
         },
         context=context,
     ).get_results()

diff --git a/src/dagster_dags/assets/nwp/ecmwf_hres_ifs_india.py b/src/dagster_dags/assets/nwp/ecmwf_hres_ifs_india.py
@@ -67,10 +67,13 @@ def ecmwf_hres_ifs_india_asset(
             "ECMWF_API_KEY": os.environ["ECMWF_API_KEY"],
             "ECMWF_API_EMAIL": os.environ["ECMWF_API_EMAIL"],
             "ECMWF_API_URL": os.environ["ECMWF_API_URL"],
-            "CONCURRENCY": "false",
+            "CONCURRENCY": "true",
         },
+        # See https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run
         container_kwargs={
             "volumes": [f"{ARCHIVE_FOLDER}:/work"],
+            "mem_limit": "8g",
+            "nano_cpus": 4e9,
         },
         context=context,
     ).get_results()

diff --git a/src/dagster_dags/assets/nwp/ecmwf_hres_ifs_west_europe.py b/src/dagster_dags/assets/nwp/ecmwf_hres_ifs_west_europe.py
@@ -67,10 +67,13 @@ def ecmwf_hres_ifs_west_europe_asset(
             "ECMWF_API_KEY": os.environ["ECMWF_API_KEY"],
             "ECMWF_API_EMAIL": os.environ["ECMWF_API_EMAIL"],
             "ECMWF_API_URL": os.environ["ECMWF_API_URL"],
-            "CONCURRENCY": "false",
+            "CONCURRENCY": "true",
         },
+        # See https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run
         container_kwargs={
             "volumes": [f"{ARCHIVE_FOLDER}:/work"],
+            "mem_limit": "8g",
+            "nano_cpus": 4e9,
         },
         context=context,
     ).get_results()

diff --git a/src/dagster_dags/assets/nwp/noaa-gfs-global.py b/src/dagster_dags/assets/nwp/noaa-gfs-global.py
@@ -63,10 +63,13 @@ def ncep_gfs_global_asset(
         env={
             "MODEL_REPOSITORY": "gfs",
             "NOTIFICATION_REPOSITORY": "dagster-pipes",
-            "CONCURRENCY": "false",
+            "CONCURRENCY": "true",
         },
+        # See https://docker-py.readthedocs.io/en/stable/containers.html#docker.models.containers.ContainerCollection.run
         container_kwargs={
             "volumes": [f"{ARCHIVE_FOLDER}:/work"],
+            "mem_limit": "8g",
+            "nano_cpus": 4e9,
         },
         context=context,
     ).get_results()