Merge branch 'main' into estimate_memory_usage

Author: ungarj

.github/workflows/python-package.yml

@@ -37,8 +37,8 @@ jobs:
           sudo apt-get -y update
           sudo apt-get install -y gdal-bin python-tk libgdal-dev libproj-dev libgeos-dev
           python -m pip install --upgrade pip wheel
-          pip install -r test/requirements.txt -r requirements.txt
           pip install -e .[complete]
+          pip install -r test/requirements.txt
           pip freeze
 
       - name: Lint with flake8

.pre-commit-config.yaml

@@ -16,4 +16,10 @@ repos:
 -   repo: https://github.com/PyCQA/autoflake
     rev: v2.1.1
     hooks:
-    -   id: autoflake
+    -   id: autoflake
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: ["--profile", "black"]

CHANGELOG.rst

@@ -3,6 +3,66 @@ Changelog
 #########
 
 
+---------------------
+2023.9.0 - 2023-09-05
+---------------------
+
+* packaging
+
+  * limit dependent versions to `"aiobotocore>=1.1.2,<=2.5.4"` and `"s3fs<2023.9.0"`
+
+* core
+
+  * make sure opened/written files are removed upon exception (#576)
+  * CLI: apply tiled-assets hack also to create-item CLI (#577)
+  * provide path schema to configure how tile paths are created (#581)
+  * `IndexedFeatures`: allow reprojection of object bounds to a target CRS (#585)
+
+
+---------------------
+2023.8.1 - 2023-08-09
+---------------------
+
+* packaging
+
+  * require `Shapely>=2.0.0` (#572)
+  * remmove `cached_property` package requirement (#573)
+  * add `isort` to pre-commit (#573)
+
+* core
+
+  * fix `ReferencedRaster.to_file()` on 2D arrays (#574)
+
+
+---------------------
+2023.8.0 - 2023-08-09
+---------------------
+
+* packaging
+
+  * add `pydantic<2.0.0` as dependency
+
+
+* CI
+
+  * also test on Python 3.11 (#562)
+
+* core
+
+  * enable adding default read parameters to TileDirectory input (#565)
+  * configuration schema (#564)
+
+    * add `pydantic<2.0.0` as dependency
+    * `mapchete.config.ProcessConfig` now defines the mapchete process configuration schema
+    * process function parameters should now go into the `process_parameters` section of the configuration
+    * add `mapchete.config.ProcessFunc` abstraction class to load and handle user process functions
+
+  * CLI: fix passing on storage options; add storage options to convert command (#568)
+  * update STACTA file schema to STAC 1.0.0 (#569)
+  * added `ReferencedRaster.to_file()` (#570)
+  * added `read_raster(tile=...)` kwarg to resample incoming raster (#570)
+
+
 ---------------------
 2023.7.1 - 2023-07-18
 ---------------------

doc/source/process_output.rst

@@ -51,6 +51,18 @@ inserting the values or pointing to environment variables for increased security
             AWS_SECRET_ACCESS_KEY: ${SOME_KEY_SECRET}
 
 
+Also, the tile path schema can be edited if desired:
+
+.. code-block:: yaml
+
+    output:
+        type: geodetic
+        format: GTiff
+        metatiling: 4  # optional
+        pixelbuffer: 10  # optional
+        tile_path_schema: "{zoom}/{col}/{row}.{extension}"
+
+
 ----------------------
 Default output formats
 ----------------------

mapchete/__init__.py

@@ -19,7 +19,7 @@
     "SkippedFuture",
     "Job",
 ]
-__version__ = "2023.7.1"
+__version__ = "2023.9.0"
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())

mapchete/_core.py

@@ -24,7 +24,7 @@
 from mapchete.errors import MapcheteNodataTile, ReprojectionFailed
 from mapchete.formats import read_output_metadata
 from mapchete.io import MPath, fs_from_path, tiles_exist
-from mapchete.stac import update_tile_directory_stac_item
+from mapchete.stac import tile_direcotry_item_to_dict, update_tile_directory_stac_item
 from mapchete.tile import count_tiles
 from mapchete.validate import validate_tile, validate_zooms
 
@@ -705,11 +705,12 @@ def write_stac(self, indent=4):
                 item_metadata=self.config.output.stac_item_metadata,
                 tile_pyramid=self.config.output_pyramid,
                 bands_type=self.config.output.stac_asset_type,
+                band_asset_template=self.config.output.tile_path_schema,
             )
             logger.debug("write STAC item JSON to %s", self.config.output.stac_path)
             self.config.output.stac_path.parent.makedirs()
             with self.config.output.stac_path.open("w") as dst:
-                dst.write(json.dumps(item.to_dict(), indent=indent))
+                dst.write(json.dumps(tile_direcotry_item_to_dict(item), indent=indent))
         except ReprojectionFailed:
             logger.warning(
                 "cannot create STAC item because footprint cannot be reprojected into EPSG:4326"

mapchete/_executor.py

@@ -7,9 +7,7 @@
 import sys
 import warnings
 from concurrent.futures._base import CancelledError
-from functools import partial
-
-from cached_property import cached_property
+from functools import cached_property, partial
 
 from mapchete._timer import Timer
 from mapchete.errors import JobCancelledError, MapcheteTaskFailed
@@ -55,8 +53,6 @@ class _ExecutorBase:
     cancelled = False
     running_futures = None
     finished_futures = None
-    _as_completed = None
-    _executor = None
     _executor_cls = None
     _executor_args = ()
     _executor_kwargs = {}
@@ -687,8 +683,8 @@ def future_raise_exception(future, raise_errors=True):
     # Let's directly re-raise these to be more transparent.
     try:
         # when using dask, also directly raise specific dask errors
-        from distributed import CancelledError
         from dask.distributed import TimeoutError
+        from distributed import CancelledError
         from distributed.comm.core import CommClosedError
 
         keep_exceptions = (CancelledError, TimeoutError, CommClosedError)

mapchete/_processing.py

@@ -19,6 +19,7 @@
 from mapchete._tasks import TaskBatch, TileTask, TileTaskBatch, to_dask_collection
 from mapchete._timer import Timer
 from mapchete.errors import MapcheteNodataTile
+from mapchete.path import batch_sort_property
 from mapchete.types import Bounds, ZoomLevels
 
 FUTURE_TIMEOUT = float(os.environ.get("MP_FUTURE_TIMEOUT", 10))
@@ -179,7 +180,12 @@ def task_batches(
 
                 for tile, skip, _ in _filter_skipable(
                     process=process,
-                    tiles_batches=process.get_process_tiles(zoom, batch_by="row"),
+                    tiles_batches=process.get_process_tiles(
+                        zoom,
+                        batch_by=batch_sort_property(
+                            process.config.output_reader.tile_path_schema
+                        ),
+                    ),
                     target_set=(
                         overview_parents if process.config.baselevels and i else None
                     ),
@@ -726,7 +732,12 @@ def _run_multi_overviews(
                 )
                 for tile, skip, process_msg in _filter_skipable(
                     process=process,
-                    tiles_batches=process.get_process_tiles(zoom, batch_by="row"),
+                    tiles_batches=process.get_process_tiles(
+                        zoom,
+                        batch_by=batch_sort_property(
+                            process.config.output_reader.tile_path_schema
+                        ),
+                    ),
                     target_set=(
                         overview_parents if process.config.baselevels and i else None
                     ),
@@ -810,7 +821,12 @@ def _run_multi_no_overviews(
                 tiles_batches=(
                     batch
                     for zoom in zoom_levels.descending()
-                    for batch in process.get_process_tiles(zoom, batch_by="row")
+                    for batch in process.get_process_tiles(
+                        zoom,
+                        batch_by=batch_sort_property(
+                            process.config.output_reader.tile_path_schema
+                        ),
+                    )
                 ),
                 target_set=None,
                 skip_output_check=skip_output_check,

mapchete/cli/default/convert.py

@@ -102,6 +102,8 @@ def _validate_bidx(ctx, param, bidx):
 @options.opt_logfile
 @options.opt_vrt
 @options.opt_idx_out_dir
+@options.opt_src_fs_opts
+@options.opt_dst_fs_opts
 def convert(
     tiledir,
     output,

mapchete/cli/default/create.py

@@ -5,9 +5,9 @@
 from string import Template
 
 import click
+import tilematrix
 from importlib_resources import files
 from oyaml import dump
-import tilematrix
 
 from mapchete.cli import options
 from mapchete.formats import available_output_formats

mapchete/cli/default/index.py

@@ -1,8 +1,6 @@
 """Create index for process output."""
 
 import logging
-import os
-import sys
 
 import click
 import click_spinner

mapchete/cli/default/serve.py

@@ -168,7 +168,7 @@ def _tile_response(mp, web_tile, debug):
 
 
 def _valid_tile_response(mp, data):
-    from flask import make_response, jsonify
+    from flask import jsonify, make_response
     from flask_rangerequest import RangeRequest
 
     out_data, mime_type = mp.config.output.for_web(data)

mapchete/cli/default/stac.py

@@ -10,7 +10,11 @@
 from mapchete.config import raw_conf, raw_conf_output_pyramid
 from mapchete.formats import read_output_metadata
 from mapchete.io import MPath
-from mapchete.stac import create_prototype_files, tile_directory_stac_item
+from mapchete.stac import (
+    create_prototype_files,
+    tile_direcotry_item_to_dict,
+    tile_directory_stac_item,
+)
 from mapchete.validate import validate_zooms
 
 logger = logging.getLogger(__name__)
@@ -66,6 +70,7 @@ def create_item(
         default_bounds_crs,
         default_zoom,
         default_item_metadata,
+        band_asset_template,
     ) = output_info(input_)
 
     if default_zoom:
@@ -94,11 +99,12 @@ def create_item(
         item_path=item_path,
         asset_basepath=asset_basepath,
         relative_paths=relative_paths,
+        band_asset_template=band_asset_template,
         bands_type=None,
         crs_unit_to_meter=1,
     )
     logger.debug("item_path: %s", item_path)
-    item_json = json.dumps(item.to_dict(), indent=indent)
+    item_json = json.dumps(tile_direcotry_item_to_dict(item), indent=indent)
     click.echo(item_json)
     if force or click.confirm(f"Write output to {item_path}?", abort=True):
         with fsspec.open(item_path, "w") as dst:
@@ -118,16 +124,19 @@ def output_info(inp):
             conf.get("bounds_crs"),
             conf.get("zoom_levels"),
             conf["output"].get("stac"),
+            conf["output"].get("tile_path_schema", "{zoom}/{row}/{col}.{extension}"),
         )
 
+    output_metadata = read_output_metadata(path / "metadata.json")
     return (
-        read_output_metadata(path / "metadata.json")["pyramid"],
+        output_metadata["pyramid"],
         path,
         path.name,
         None,
         None,
         None,
         None,
+        output_metadata.get("tile_path_schema", "{zoom}/{row}/{col}.{extension}"),
     )
 
 

mapchete/commands/_convert.py

@@ -65,6 +65,8 @@ def convert(
     cog: bool = False,
     msg_callback: Callable = None,
     as_iterator: bool = False,
+    src_fs_opts: Union[dict, None] = None,
+    dst_fs_opts: Union[dict, None] = None,
 ) -> mapchete.Job:
     """
     Convert mapchete outputs or other geodata.
@@ -176,8 +178,8 @@ def _empty_callback(*args, **kwargs):
     workers = workers or multi or cpu_count()
     creation_options = creation_options or {}
     bidx = [bidx] if isinstance(bidx, int) else bidx
-    tiledir = MPath.from_inp(tiledir) if isinstance(tiledir, str) else tiledir
-    output = MPath.from_inp(output)
+    tiledir = MPath.from_inp(tiledir, storage_options=src_fs_opts)
+    output = MPath.from_inp(output, storage_options=dst_fs_opts)
     try:
         input_info = _get_input_info(tiledir)
         logger.debug("input params: %s", input_info)

mapchete/commands/_cp.py

@@ -27,14 +27,14 @@ def cp(
     point: Tuple[float, float] = None,
     point_crs: Tuple[float, float] = None,
     overwrite: bool = False,
-    workers: int = None,
-    multi: int = None,
-    concurrency: str = None,
-    dask_scheduler: str = None,
+    workers: Union[int, None] = None,
+    multi: Union[int, None] = None,
+    concurrency: Union[str, None] = None,
+    dask_scheduler: Union[str, None] = None,
     dask_client=None,
-    src_fs_opts: dict = None,
-    dst_fs_opts: dict = None,
-    msg_callback: Callable = None,
+    src_fs_opts: Union[dict, None] = None,
+    dst_fs_opts: Union[dict, None] = None,
+    msg_callback: Union[Callable, None] = None,
     as_iterator: bool = False,
 ) -> mapchete.Job:
     """
@@ -112,8 +112,8 @@ def _empty_callback(*_):
     if zoom is None:  # pragma: no cover
         raise ValueError("zoom level(s) required")
 
-    src_tiledir = MPath.from_inp(src_tiledir, **src_fs_opts)
-    dst_tiledir = MPath.from_inp(dst_tiledir, **dst_fs_opts)
+    src_tiledir = MPath.from_inp(src_tiledir, storage_options=src_fs_opts)
+    dst_tiledir = MPath.from_inp(dst_tiledir, storage_options=dst_fs_opts)
     src_fs = src_tiledir.fs
     dst_fs = dst_tiledir.fs
 

mapchete/commands/_index.py

@@ -8,6 +8,7 @@
 
 import mapchete
 from mapchete.index import zoom_index_gen
+from mapchete.path import MPath
 
 logger = logging.getLogger(__name__)
 
@@ -115,7 +116,6 @@ def _empty_callback(*_):
         pass
 
     msg_callback = msg_callback or _empty_callback
-    fs_opts = fs_opts or {}
 
     msg_callback(f"create index(es) for {tiledir}")
     # process single tile