[1pt]: Remove FIM3 and most references to GMS

config/deny_gms_branch_unittests.lst → config/deny_branch_unittests.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif

config/deny_gms_branch_zero.lst → config/deny_branch_zero.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branch zero to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif

config/deny_gms_branches.lst → config/deny_branches.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for branches to delete
 # Use comment to allow list the file
 # Use curly braces to denote branch_id
 agree_bufgrid.tif

config/deny_gms_unit.lst → config/deny_unit.lst

@@ -1,4 +1,4 @@
-# List of files for gms branches to delete
+# List of files for units to delete
 # Use comment to allow list the file
 #branch_id.lst
 #branch_polygons.gpkg

config/params_template.env

@@ -20,7 +20,7 @@ export slope_min=0.001
 export min_catchment_area=0.25
 export min_stream_length=0.5
 
-#### gms parameters ####
+#### branch parameters ####
 export branch_id_attribute=levpa_id
 export branch_buffer_distance_meters=7000
 export branch_timeout=4000 # pass int or float. To make a percentage of median, pass a '%' at the end.

data/nws/preprocess_ahps_nws.py

@@ -14,6 +14,15 @@
 from tools_shared_functions import mainstem_nwm_segs, get_metadata, aggregate_wbd_hucs, get_thresholds, get_datum, ngvd_to_navd_ft, get_rating_curve, select_grids, get_nwm_segs, flow_data, process_extent, process_grid, raster_to_feature
 
 
+########################################################
+'''
+Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+   a significant review and/or upgrade.
+'''
+
+########################################################
+
+
 def get_env_paths():
     load_dotenv()
     #import variables from .env file
@@ -354,6 +363,15 @@ def preprocess_nws(source_dir, destination, reference_raster):
 
 
 if __name__ == '__main__':
+
+    ########################################################
+    '''
+    Feb 15, 2023 - This file may be deprecated. At a minimum, it needs
+    a significant review and/or upgrade.
+    '''
+
+    ########################################################
+
     #Parse arguments
     parser = argparse.ArgumentParser(description = 'Create preprocessed USGS benchmark datasets at AHPS locations.')
     parser.add_argument('-s', '--source_dir', help = 'Workspace where all source data is located.', required = True)

data/usgs/acquire_and_preprocess_3dep_dems.py

@@ -174,7 +174,7 @@ def __download_usgs_dems(extent_files, output_folder_path, number_of_jobs, retry
 
                 sys.exit(1)
 
-        # Send the executor to the progress bar and wait for all FR tasks to finish
+        # Send the executor to the progress bar and wait for all tasks to finish
         sf.progress_bar_handler(executor_dict, f"Downloading USGG 3Dep Dems")
 
     print(f"-- Downloading USGS DEMs Completed")

docs/CHANGELOG.md

@@ -2,6 +2,116 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
 
+## v4.1.(pending) - 2023-02-16 - [PR#816](https://github.com/NOAA-OWP/inundation-mapping/pull/816)
+
+
+This update removes the remaining elements of FIM3 code.  It further removes the phrases "GMS" as basically the entire FIM4 model. FIM4 is GMS. With removing FIM3, it also means remove concepts of "MS" and "FR" which were no longer relevant in FIM4.  There are only a few remaining places that will continue with the phrase "GMS" which is in some inundation files which are being re-evaluated.  Some deprecated files have been removed and some subfolders removed.
+
+There are a lot of duplicate explanations for some of the changes, so here is a shortcut system.
+
+- desc 1:  Remove or rename values based on phrase "GMS, MS and/or FR"
+- desc 2:  Moved file from the /src/gms folder to /src  or /tools/gms_tools to /tools
+- desc 3:  No longer needed as we now use the `fim_pipeline.sh` processing model.
+
+### Removals
+
+- `data`
+    - `acquire_and_preprocess_inputs.py`:  No longer needed
+- `gms_pipeline.sh` : see desc 3
+- `gms_run_branch.sh` : see desc 3
+- `gms_run_post_processing.sh` : see desc 3
+- `gms_run_unit.sh` : see desc 3
+- `src`
+    - `gms`
+        - `init.py` : folder removed, no longer needed.
+        - `aggregate_branch_lists.py`: no longer needed.  Newer version already exists in src directory.
+        - `remove_error_branches.py` :  see desc 3
+        - `run_by_unit.sh` : see desc 3
+        - `test_new_crosswalk.sh` : no longer needed
+        - `time_and_tee_run_by_branch.sh` : see desc 3
+        - `time_and_tee_run_by_unit.sh` : see desc 3
+    - `output_cleanup.py` : see desc 3
+ - `tools/gms_tools`
+     - `init.py` : folder removed, no longer needed.
+
+### Changes
+
+- `config`
+   - `deny_branch_unittests.lst` :  renamed from `deny_gms_branch_unittests.lst`
+   - `deny_branch_zero.lst` : renamed from `deny_gms_branch_zero.lst`
+   - `deny_branches.lst` :  renamed from `deny_gms_branches.lst`
+   - `deny_unit.lst`  : renamed from `deny_gms_unit.lst`
+   - `params_template.env` : see desc 1
+
+- `data`
+    - `nws`
+        - `preprocess_ahps_nws.py`:   Added deprecation note: If reused, it needs review and/or upgrades.
+    - `acquire_and_preprocess_3dep_dems.py` : see desc 1
+ - `fim_post_processing.sh` : see desc 1, plus a small pathing change.
+ - `fim_pre_processing.sh` : see desc 1
+ - ` src`
+     - `add_crosswalk.py` : see desc 1. Also cleaned up some formatting and commented out a code block in favor of a better way to pass args from "__main__"
+     - `bash_variables.env` : see desc 1
+     - `buffer_stream_branches.py` : see desc 2
+     - `clip_rasters_to_branches.py` : see desc 2
+     - `crosswalk_nwm_demDerived.py` :  see desc 1 and desc 2
+     - `delineate_hydros_and_produce_HAND.sh` : see desc 1 and desc 2
+     - `derive_level_paths.py`  :  see desc 1 and desc 2
+     - `edit_points.py` : see desc  2
+     - `filter_inputs_by_huc.py`: see desc 1 and desc 2
+     - `finalize_srcs.py`:  see desc 2
+     - `generate_branch_list.py` : see desc 1
+     - `make_rem.py` : see desc 2
+     - `make_dem.py` : see desc  2
+     - `outputs_cleanup.py`:  see desc 1
+     - `process_branch.sh`:  see desc 1
+     - `query_vectors_by_branch_polygons.py`: see desc 2
+     - `reset_mannings.py` : see desc 2
+     - `run_by_branch.sh`:  see desc 1
+     - `run_unit_wb.sh`: see desc 1
+     - `stream_branches.py`:  see desc 2
+     - `subset_catch_list_by_branch_id.py`: see desc 2
+     - `toDo.md`: see desc 2
+     - `usgs_gage_aggregate.py`:  see desc 1
+     - `usgs_gage_unit_setup.py` : see desc 1
+     - `utils`
+         - `fim_enums.py` : see desc 1
+
+- `tools`
+    - `combine_crosswalk_tables.py` : see desc 2
+    - `compare_ms_and_non_ms_metrics.py` : see desc 2
+    - `compile_comp_stats.py`: see desc 2  and added note about possible deprecation.
+    - `compile_computation_stats.py` : see desc 2  and added note about possible deprecation.
+    - `composite_inundation.py` : see desc 1 : note.. references a file called inundate_gms which retains it's name for now.
+    - `consolidate_metrics.py`: added note about possible deprecation.
+    - `copy_test_case_folders.py`: see desc 1
+    - `eval_plots.py` : see desc 1
+    - `evaluate_continuity.py`: see desc 2
+    - `find_max_catchment_breadth.py` : see desc 2
+    - `generate_categorical_fim_mapping.py` : see desc 1
+    - `inundate_gms.py`: see desc 1 and desc 2. Note: This file has retained its name with the phrase "gms" in it as it might be upgraded later and there are some similar files with similar names.
+    - `inundate_nation.py` : see desc 1
+    - `inundation.py`:  text styling change
+    - `make_boxes_from_bounds.py`: text styling change
+    - `mosaic_inundation.py`:  see desc 1 and desc 2
+    - `overlapping_inundation.py`: see desc 2
+    - `plots.py` : see desc 2
+    - `run_test_case.py`:  see desc 1
+    - `synthesize_test_cases.py`: see desc 1
+
+- `unit_tests`
+    - `README.md`: see desc 1
+    - `__template_unittests.py`: see desc 1
+    - `check_unit_errors_params.json`  and `check_unit_errors_unittests.py` : see desc 1
+    - `derive_level_paths_params.json` and `derive_level_paths_unittests.py` : see desc 1 and desc 2
+    - `filter_catchments_and_add_attributes_unittests.py`: see desc 1
+    - `outputs_cleanup_params.json` and `outputs_cleanup_unittests.py`: see desc 1 and desc 2
+    - `split_flows_unittests.py` : see desc 1
+    - `tools`
+        - `inundate_gms_params.json` and `inundate_gms_unittests.py`: see desc 1 and desc 2
+
+<br/><br/>
+
 ## v4.1.0.0 - 2023-01-30 - [PR#806](https://github.com/NOAA-OWP/inundation-mapping/pull/806)
 
 As we move to Amazon Web Service, AWS, we need to change our processing system. Currently, it is `gms_pipeline.sh` using bash "parallel" as an iterator which then first processes all HUCs, but not their branches. One of `gms_pipeline.sh`'s next steps is to do branch processing which is again iterated via "parallel". AKA. Units processed as one step, branches processed as second independent step.

fim_post_processing.sh

@@ -69,7 +69,7 @@ rm -f $outputRunDataDir/logs/subdiv_src_.log
 
 # load up enviromental information
 args_file=$outputRunDataDir/runtime_args.env
-gms_inputs=$outputRunDataDir/gms_inputs.csv
+fim_inputs=$outputRunDataDir/fim_inputs.csv
 
 source $args_file
 source $outputRunDataDir/params.env
@@ -87,7 +87,7 @@ post_proc_start_time=`date +%s`
 
 ## AGGREGATE BRANCH LISTS INTO ONE ##
 echo -e $startDiv"Start branch aggregation"
-python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $gms_inputs
+python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.csv" -o $fim_inputs
 
 ## GET NON ZERO EXIT CODES FOR UNITS ##
 ## No longer applicable
@@ -96,17 +96,9 @@ python3 $srcDir/aggregate_branch_lists.py -d $outputRunDataDir -f "branch_ids.cs
 echo -e $startDiv"Start non-zero exit code checking"
 find $outputRunDataDir/logs/branch -name "*_branch_*.log" -type f | xargs grep -E "Exit status: ([1-9][0-9]{0,2})" > "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" &
 
-
-## REMOVE FAILED BRANCHES ##
-# Needed in case aggregation fails, we will need the logs
-#echo
-#echo -e $startDiv"Removing branches that failed with Exit status: 61"
-#Tstart
-#python3 $srcDir/gms/remove_error_branches.py -f "$outputRunDataDir/branch_errors/non_zero_exit_codes.log" -g #$gms_inputs
-
 ## RUN AGGREGATE BRANCH ELEV TABLES ##
 echo "Processing usgs gage aggregation"
-python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -gms $gms_inputs
+python3 $srcDir/usgs_gage_aggregate.py -fim $outputRunDataDir -i $fim_inputs
 
 ## RUN SYNTHETIC RATING CURVE BANKFULL ESTIMATION ROUTINE ##
 if [ "$src_bankfull_toggle" = "True" ]; then
@@ -191,7 +183,7 @@ echo
 echo -e $startDiv"Combining crosswalk tables"
 # aggregate outputs
 Tstart
-python3 /foss_fim/tools/gms_tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
+python3 /foss_fim/tools/combine_crosswalk_tables.py -d $outputRunDataDir -o $outputRunDataDir/crosswalk_table.csv
 Tcount
 date -u
 

fim_pre_processing.sh

@@ -26,22 +26,22 @@ usage ()
     echo '                    Default (if arg not added) : /foss_fim/config/params_template.env'    
     echo '  -ud/--unitDenylist'
     echo '                    A file with a line delimited list of files in UNIT (HUC) directories to be removed'
-    echo '                    upon completion (see config/deny_gms_unit.lst for a starting point)'
-    echo '                    Default (if arg not added) : /foss_fim/config/deny_gms_unit.lst'
+    echo '                    upon completion (see config/deny_unit.lst for a starting point)'
+    echo '                    Default (if arg not added) : /foss_fim/config/deny_unit.lst'
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -bd/--branchDenylist'
     echo '                    A file with a line delimited list of files in BRANCHES directories to be removed' 
     echo '                    upon completion of branch processing.'
-    echo '                    (see config/deny_gms_branches.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branches.lst'   
+    echo '                    (see config/deny_branches.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_branches.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'
     echo '  -zd/--branchZeroDenylist'
     echo '                    A file with a line delimited list of files in BRANCH ZERO directories to' 
     echo '                    be removed upon completion of branch zero processing.'
-    echo '                    (see config/deny_gms_branch_zero.lst for a starting point)'
-    echo '                    Default: /foss_fim/config/deny_gms_branch_zero.lst'   
+    echo '                    (see config/deny_branch_zero.lst for a starting point)'
+    echo '                    Default: /foss_fim/config/deny_branch_zero.lst'   
     echo '                    -- Note: if you want to keep all output files (aka.. no files removed),'
     echo '                    use the word NONE as this value for this parameter.'    
     echo '  -jh/--jobLimit   : Max number of concurrent HUC jobs to run. Default 1 job at time.'
@@ -141,7 +141,7 @@ if [ -z "$isAWS" ]; then isAWS=0; fi
 # validate and set defaults for the deny lists
 if [ "$deny_unit_list" = "" ]
 then
-    deny_unit_list=$projectDir/config/deny_gms_unit.lst
+    deny_unit_list=$projectDir/config/deny_unit.lst
 elif [ "${deny_unit_list^^}" != "NONE" ] && [ ! -f "$deny_unit_list" ]
 then
     # NONE is not case sensitive
@@ -152,7 +152,7 @@ fi
 # validate and set defaults for the deny lists
 if [ "$deny_branches_list" = "" ]
 then
-    deny_branches_list=$projectDir/config/deny_gms_branches.lst
+    deny_branches_list=$projectDir/config/deny_branches.lst
 elif [ "${deny_branches_list^^}" != "NONE" ] && [ ! -f "$deny_branches_list" ]
 then
     # NONE is not case sensitive
@@ -167,7 +167,7 @@ fi
 has_deny_branch_zero_override=0
 if [ "$deny_branch_zero_list" = "" ]
 then
-    deny_branch_zero_list=$projectDir/config/deny_gms_branch_zero.lst
+    deny_branch_zero_list=$projectDir/config/deny_branch_zero.lst
 elif [ "${deny_branch_zero_list^^}" != "NONE" ]   # NONE is not case sensitive
 then
     if [ ! -f "$deny_branch_zero_list" ]
@@ -211,7 +211,7 @@ else
     rm -rdf $outputRunDataDir/unit_errors
     rm -rdf $outputRunDataDir/eval
     rm -f $outputRunDataDir/crosswalk_table.csv
-    rm -f $outputRunDataDir/gms_inputs*
+    rm -f $outputRunDataDir/fim_inputs*
     rm -f $outputRunDataDir/*.env
 fi