[1pt] PR: Replace GRASS with Whitebox

Dockerfile

@@ -52,7 +52,7 @@ ARG dataDir=/data
 ARG projectDir=/foss_fim
 ARG depDir=/dependencies
 ENV inputDataDir=$dataDir/inputs
-ENV outputDataDir=$dataDir/outputs
+ENV outputDataDir=/outputs
 ENV srcDir=$projectDir/src
 ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
@@ -70,14 +70,12 @@ COPY --from=builder $depDir $depDir
 RUN apt update --fix-missing 
 RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 tmux rsync
 
-RUN DEBIAN_FRONTEND=noninteractive apt install -y grass=7.8.2-1build3 grass-doc=7.8.2-1build3
-
 RUN apt auto-remove
 
 ## adding AWS CLI (for bash) ##
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
     unzip awscliv2.zip && \
-	./aws/install
+    ./aws/install
 
 ## adding environment variables for numba and python ##
 ENV LC_ALL=C.UTF-8
@@ -92,6 +90,14 @@ ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/tests:$projectDir/tools
 COPY Pipfile .
 COPY Pipfile.lock .
 RUN pip3 install pipenv==2022.4.8 && PIP_NO_CACHE_DIR=off PIP_NO_BINARY=shapely,pygeos pipenv install --system --deploy --ignore-pipfile
+#RUN pip3 install pipenv==2022.4.8 && pipenv install --system --deploy --ignore-pipfile (too slow to
+#     leave out shapely,pygeos at this time. Likely better after upgrading)
+
+# TEMP FIX as neither shapely or Shapely is staying in the pip list. If we manually add
+# it with pip (not pipenv), it works. Notice case for Shapely versus shapely. 
+# This temp fix works for now until we can reconsile the shapely package,
+# pygeos, geopanda's and possibly others (coming soon)
+RUN pip install shapely==1.7.0
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##
 ADD ./src/entrypoint.sh /

Pipfile

@@ -17,8 +17,6 @@ rasterio = "==1.1.5"
 rasterstats = "==0.15.0"
 richdem = "==0.3.4"
 tqdm = "==4.48.0"
-Shapely = "==1.7.0"
-grass-session = "==0.5"
 seaborn = "==0.11.0"
 python-dotenv = "*"
 natsort = "*"
@@ -34,6 +32,8 @@ jupyter = "*"
 jupyterlab = "*"
 ipympl = "*"
 pytest = "*"
+whitebox = "*"
+shapely = "==1.7.0"
 
 [requires]
 python_version = "3.8"

docs/CHANGELOG.md

@@ -2,6 +2,28 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.3.0.0 - 2023-02-15 - [PR#814](https://github.com/NOAA-OWP/inundation-mapping/pull/814)
+
+Replaces GRASS with Whitebox. This addresses several issues, including Windows permissions and GRASS projection issues. Whitebox also has a slight performance benefit over GRASS.
+
+### Removals
+
+- `src/r_grow_distance.py`: Deletes file
+
+### Changes
+
+- `Dockerfile`: Removes GRASS, update `$outputDataDir` from `/data/outputs` to `/outputs`
+- `Pipfile` and `Pipfile.lock`: Adds Whitebox and removes GRASS
+- `src/`
+    - `agreedem.py`: Removes `r_grow_distance`; refactors to use with context and removes redundant raster reads.
+    - `adjust_lateral_thalweg.py` and `agreedem.py`: Refactors to use `with` context and removes redundant raster reads
+    - `unique_pixel_and_allocation.py`: Replaces GRASS with Whitebox and remove `r_grow_distance`
+    - `gms/`
+        - `delineate_hydros_and_produce_HAND.sh` and `run_by_unit.sh`: Removes GRASS parameter
+        - `mask_dem.py`: Removes unnecessary line
+
+<br/><br/>
+
 ## v4.2.1.0 - 2023-02-21 - [PR#829](https://github.com/NOAA-OWP/inundation-mapping/pull/829)
 
 During the merge from remove-fim3 PR into dev, merge conflicts were discovered in the unit_tests folders and files. Attempts to fix them at that time failed, so some files were removed, other renamed, other edited to get the merge to work.  Here are the fixes to put the unit tests system back to par.
@@ -300,7 +322,6 @@ BUT.... you have to be careful not to overload your system.  **You need to multi
 
 <br/><br/>
 
-
 ## v4.0.19.5 - 2023-01-24 - [PR#801](https://github.com/NOAA-OWP/inundation-mapping/pull/801)
 
 When running tools/test_case_by_hydroid.py, it throws an error of local variable 'stats' referenced before assignment.

src/adjust_thalweg_lateral.py

@@ -32,32 +32,6 @@ def make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window
 
         return(zone_min_dict)
 
-    # Open files.
-    elevation_raster_object = rasterio.open(elevation_raster)
-    allocation_zone_raster_object = rasterio.open(allocation_raster)
-    cost_distance_raster_object = rasterio.open(cost_distance_raster)
-
-    meta = elevation_raster_object.meta.copy()
-    meta['tiled'], meta['compress'] = True, 'lzw'
-
-    # -- Create zone_min_dict -- #
-    zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums
-    # Update catchment_min_dict with pixel sheds minimum.
-    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template
-        elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window
-        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window
-        cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window
-
-        # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
-        zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), meta['nodata'])
-
-    # ------------------------------------------------------------------------------------------------------------------------ #
-
-    elevation_raster_object.close()
-    allocation_zone_raster_object.close()
-    cost_distance_raster_object.close()
-
-
     # ------------------------------------------- Assign zonal min to thalweg ------------------------------------------------ #
     @njit
     def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window):
@@ -82,30 +56,46 @@ def minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_w
 
         return(dem_window_to_return)
 
-    # Specify raster object metadata.
-    elevation_raster_object = rasterio.open(elevation_raster)
-    allocation_zone_raster_object = rasterio.open(allocation_raster)
-    thalweg_object = rasterio.open(stream_raster)
+    # Open files.
+    with rasterio.open(elevation_raster) as elevation_raster_object, rasterio.open(allocation_raster) as allocation_zone_raster_object:
+
+        with rasterio.open(cost_distance_raster) as cost_distance_raster_object:
+
+            meta = elevation_raster_object.meta.copy()
+            meta['tiled'], meta['compress'] = True, 'lzw'
+            ndv = meta['nodata']
+
+            # -- Create zone_min_dict -- #
+            zone_min_dict = typed.Dict.empty(types.int32,types.float32)  # Initialize an empty dictionary to store the catchment minimums
+            # Update catchment_min_dict with pixel sheds minimum.
+            for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using elevation_raster_object as template
+                elevation_window = elevation_raster_object.read(1,window=window).ravel()  # Define elevation_window
+                zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define zone_window
+                cost_window = cost_distance_raster_object.read(1, window=window).ravel()  # Define cost_window
+
+                # Call numba-optimized function to update catchment_min_dict with pixel sheds minimum.
+                zone_min_dict = make_zone_min_dict(elevation_window, zone_min_dict, zone_window, cost_window, int(cost_distance_tolerance), ndv)
+
+            # ------------------------------------------------------------------------------------------------------------------------ #
+
 
-    dem_lateral_thalweg_adj_object = rasterio.open(dem_lateral_thalweg_adj, 'w', **meta)
+        # Specify raster object metadata.
+        with rasterio.open(stream_raster) as thalweg_object, rasterio.open(dem_lateral_thalweg_adj, 'w', **meta) as dem_lateral_thalweg_adj_object:
 
-    for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
-        dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window
-        window_shape = dem_window.shape
-        dem_window = dem_window.ravel()
+            for ji, window in elevation_raster_object.block_windows(1):  # Iterate over windows, using dem_rasterio_object as template
+                dem_window = elevation_raster_object.read(1,window=window)  # Define dem_window
+                window_shape = dem_window.shape
+                dem_window = dem_window.ravel()
 
-        zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window
-        thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window
+                zone_window = allocation_zone_raster_object.read(1,window=window).ravel()  # Define catchments_window
+                thalweg_window = thalweg_object.read(1,window=window).ravel()  # Define thalweg_window
 
-        # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
-        minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
-        minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
+                # Call numba-optimized function to reassign thalweg cell values to catchment minimum value.
+                minimized_dem_window = minimize_thalweg_elevation(dem_window, zone_min_dict, zone_window, thalweg_window)
+                minimized_dem_window = minimized_dem_window.reshape(window_shape).astype(np.float32)
 
-        dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
+                dem_lateral_thalweg_adj_object.write(minimized_dem_window, window=window, indexes=1)
 
-    elevation_raster_object.close()
-    allocation_zone_raster_object.close()
-    cost_distance_raster_object.close()
 
 
 if __name__ == '__main__':