Merge pull request #67 from OCHA-DAP/humanitarian_needs

HAPI-142 HAPI-143 Humanitarian Needs theme: Add 3 countries to pipeline

pyproject.toml

@@ -34,10 +34,10 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>= 0.4.0",
+    "hapi-schema>= 0.5.0",
     "hdx-python-country>= 3.6.3",
     "hdx-python-database[postgresql]>= 1.2.9",
-    "hdx-python-scraper>= 2.3.0",
+    "hdx-python-scraper>= 2.3.2",
     "libhxl",
     "sqlalchemy"
 ]

requirements.txt

@@ -6,7 +6,7 @@
 #
 annotated-types==0.6.0
     # via pydantic
-attrs==23.1.0
+attrs==23.2.0
     # via
     #   frictionless
     #   jsonlines
@@ -29,7 +29,7 @@ click==8.1.7
     # via typer
 colorama==0.4.6
     # via typer
-coverage[toml]==7.3.3
+coverage[toml]==7.4.0
     # via pytest-cov
 cryptography==41.0.7
     # via pyopenssl
@@ -53,17 +53,17 @@ filelock==3.13.1
     # via virtualenv
 frictionless==5.16.0
     # via hdx-python-utilities
-google-auth==2.25.2
+google-auth==2.26.0
     # via
     #   google-auth-oauthlib
     #   gspread
 google-auth-oauthlib==1.2.0
     # via gspread
-greenlet==3.0.2
+greenlet==3.0.3
     # via sqlalchemy
-gspread==5.12.2
+gspread==5.12.4
     # via hdx-python-scraper
-hapi-schema==0.4.0
+hapi-schema==0.5.0
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.2.0
     # via hdx-python-scraper
@@ -74,7 +74,7 @@ hdx-python-country==3.6.3
     #   hdx-python-scraper
 hdx-python-database[postgresql]==1.2.9
     # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.3.0
+hdx-python-scraper==2.3.2
     # via hapi-pipelines (pyproject.toml)
 hdx-python-utilities==3.6.3
     # via
@@ -149,9 +149,9 @@ pockets==0.9.1
     # via sphinxcontrib-napoleon
 pre-commit==3.6.0
     # via hapi-pipelines (pyproject.toml)
-psycopg[binary]==3.1.15
+psycopg[binary]==3.1.16
     # via hdx-python-database
-psycopg-binary==3.1.15
+psycopg-binary==3.1.16
     # via psycopg
 pyasn1==0.5.1
     # via
@@ -163,11 +163,11 @@ pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pydantic==2.5.2
+pydantic==2.5.3
     # via
     #   frictionless
     #   inflect
-pydantic-core==2.14.5
+pydantic-core==2.14.6
     # via pydantic
 pygments==2.17.2
     # via rich
@@ -179,7 +179,7 @@ pyphonetics==0.5.3
     # via hdx-python-country
 pyrsistent==0.20.0
     # via jsonschema
-pytest==7.4.3
+pytest==7.4.4
     # via
     #   hapi-pipelines (pyproject.toml)
     #   pytest-cov
@@ -205,7 +205,7 @@ quantulum3==0.9.0
     # via hdx-python-api
 ratelimit==2.2.1
     # via hdx-python-utilities
-regex==2023.10.3
+regex==2023.12.25
     # via hdx-python-scraper
 requests==2.31.0
     # via
@@ -243,14 +243,14 @@ six==1.16.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-sqlalchemy==2.0.23
+sqlalchemy==2.0.25
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema
     #   hdx-python-database
 stringcase==1.2.0
     # via frictionless
-structlog==23.2.0
+structlog==23.3.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities

src/hapi/pipelines/app/__main__.py

@@ -2,7 +2,8 @@
 """
 import argparse
 import logging
-from os import getenv
+from os import getenv, remove
+from os.path import exists
 from typing import List, Optional
 
 from hdx.api.configuration import Configuration
@@ -48,6 +49,7 @@ def parse_args():
         default=None,
         help="Database connection parameters. Overrides --db-uri.",
     )
+    parser.add_argument("-th", "--themes", default=None, help="Themes to run")
     parser.add_argument(
         "-sc", "--scrapers", default=None, help="Scrapers to run"
     )
@@ -71,6 +73,7 @@ def parse_args():
 def main(
     db_uri: Optional[str] = None,
     db_params: Optional[str] = None,
+    themes_to_run: Optional[List[str]] = None,
     scrapers_to_run: Optional[List[str]] = None,
     save: bool = False,
     use_saved: bool = False,
@@ -83,7 +86,10 @@ def main(
     Args:
         db_uri (Optional[str]): Database connection URI. Defaults to None.
         db_params (Optional[str]): Database connection parameters. Defaults to None.
+        themes_to_run (Optional[ListTuple[str]]): Themes to run. Defaults to None (all themes).
+        scrapers_to_run (Optional[ListTuple[str]]): Scrapers to run. Defaults to None (all scrapers).
         save (bool): Whether to save state for testing. Defaults to False.
+        use_saved (bool): Whether to use saved state for testing. Defaults to False.
 
     Returns:
         None
@@ -94,7 +100,10 @@ def main(
     elif db_uri:
         params = get_params_from_connection_uri(db_uri)
     else:
-        params = {"dialect": "sqlite", "database": "hapi.db"}
+        filename = "hapi.db"
+        if exists(filename):
+            remove(filename)
+        params = {"dialect": "sqlite", "database": filename}
     logger.info(f"> Database parameters: {params}")
     configuration = Configuration.read()
     with ErrorsOnExit() as errors_on_exit:
@@ -121,6 +130,7 @@ def main(
                     configuration,
                     session,
                     today,
+                    themes_to_run,
                     scrapers_to_run,
                     errors_on_exit,
                 )
@@ -152,13 +162,18 @@ def main(
         db_uri = getenv("DB_URI")
     if db_uri and "://" not in db_uri:
         db_uri = f"postgresql://{db_uri}"
+    if args.themes:
+        themes_to_run = args.themes.split(",")
+    else:
+        themes_to_run = None
     if args.scrapers:
         scrapers_to_run = args.scrapers.split(",")
     else:
         scrapers_to_run = None
     project_configs = [
         "core.yaml",
         "food_security.yaml",
+        "humanitarian_needs.yaml",
         "operational_presence.yaml",
         "population.yaml",
     ]
@@ -173,6 +188,7 @@ def main(
         project_config_dict=project_config_dict,
         db_uri=db_uri,
         db_params=args.db_params,
+        themes_to_run=themes_to_run,
         scrapers_to_run=scrapers_to_run,
         save=args.save,
         use_saved=args.use_saved,