Skip to content

Commit

Permalink
Fix flake8
Browse files Browse the repository at this point in the history
  • Loading branch information
webb-ben committed Jan 30, 2025
1 parent b85798b commit d5518df
Show file tree
Hide file tree
Showing 6 changed files with 44 additions and 8 deletions.
1 change: 1 addition & 0 deletions src/wqp/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def process_county_stations():
"""Job to process stations for a single county partition."""
fetch_and_process_stations()


if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: python jobs.py <county1> <county2> ...")
Expand Down
10 changes: 10 additions & 0 deletions src/wqp/mapping.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
# =================================================================
#
# Authors: Ben Webb <bwebb@lincolninst.edu>
#
# Copyright (c) 2025 Lincoln Institute of Land Policy
#
# Licensed under the MIT License.
#
# =================================================================

# flake8: noqa
MAPPING = {
"Perfluoro(4-isopropyltoluene)": {
Expand Down
11 changes: 9 additions & 2 deletions src/wqp/ops/datastreams.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
# =================================================================
#
# Authors: Ben Webb <bwebb@lincolninst.edu>
#
# Copyright (c) 2025 Lincoln Institute of Land Policy
#
# Licensed under the MIT License.
#
# =================================================================

from csv import DictReader
from io import StringIO
import re
from requests import Session
from typing import Iterable
from dagster import get_dagster_logger

from wqp.env import MONITORING_LOCATIONS_URL
from wqp.util import deterministic_hash, url_join
from wqp.mapping import MAPPING
from wqp.models import Datastream, UnitOfMeasurement, ObservedProperty

LOGGER = get_dagster_logger()

Expand Down
7 changes: 5 additions & 2 deletions src/wqp/ops/transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@
# Logger setup
LOGGER = get_dagster_logger()


def upsert_collection_item(collection: str, item: Dict[str, Any]) -> bool:
api_url = url_join(API_BACKEND_URL, collection)
response = requests.post(api_url, json=item)
return response.status_code == 201
return response.status_code == 201


def station_exists(station_id: str) -> bool:
"""
Expand Down Expand Up @@ -106,7 +108,8 @@ def publish_station_collection(stations_data: StationsData) -> bool:
LOGGER.info(f'No datastreams for {station_identifier}')
continue
except Exception:
LOGGER.warning(f'Failed to load datastreams for {station_identifier}')
msg = f'Failed to load datastreams for {station_identifier}'
LOGGER.warning(msg)
continue

try:
Expand Down
6 changes: 4 additions & 2 deletions src/wqp/partitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ def load_us_counties() -> List[str]:
Load all US county codes from GeoConnex API.
"""
url = "https://reference.geoconnex.us/collections/counties/items"
params = {"limit": 5000, "properties": "statefp,countyfp", "skipGeometry": "true"}
params = {"limit": 5000,
"properties": "statefp,countyfp",
"skipGeometry": "true"}
counties = []
LOGGER.debug(url)
while True:
Expand Down Expand Up @@ -51,4 +53,4 @@ def load_us_counties() -> List[str]:
dumps({
partition: "" for partition in load_us_counties()
})
)
)
17 changes: 15 additions & 2 deletions src/wqp/util.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
# =================================================================
#
# Authors: Ben Webb <bwebb@lincolninst.edu>
#
# Copyright (c) 2025 Lincoln Institute of Land Policy
#
# Licensed under the MIT License.
#
# =================================================================

import re
import os
import hashlib
from typing import Union, Any, Dict
from typing import Union, Any
from uuid import UUID


Expand Down Expand Up @@ -73,15 +83,18 @@ def get_env(key: str, fallback: Any = None) -> str:

return val


def deterministic_hash(name: str, desiredLength: int) -> int:
"""Python's built-in hash function is not deterministic, so this is a workaround"""
"""Python's built-in hash function is not deterministic,
so this is a workaround"""
data = name.encode("utf-8")
hash_hex = hashlib.md5(data).hexdigest()
hash_int = int(hash_hex, 16)
trimmed_hash = hash_int % (10**desiredLength)
# handle case where it hashes to 0
return trimmed_hash if trimmed_hash != 0 else trimmed_hash + 1


def make_uuid(input: str, raw: bool = False) -> UUID:
"""
helper function to make uuid
Expand Down

0 comments on commit d5518df

Please sign in to comment.