NOAA-OWP · BradfordBates-NOAA · Mar 9, 2021 · Mar 3, 2021 · Mar 3, 2021 · Mar 3, 2021
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 <br/><br/>
 
+## v3.0.8.0 - 2021-03-09 - [PR #279](https://github.com/NOAA-OWP/cahaba/pull/279)
+
+Refactored NWS Flood Categorical HAND FIM (CatFIM) pipeline to open source.
+
+### Changes
+ - Added `VIZ_PROJECTION` to `shared_variables.py`.
+ - Added missing library referenced in `inundation.py`.
+ - Cleaned up and converted evaluation scripts in `generate_categorical_fim.py` to open source.
+ - Removed `util` folders under `tools` directory.
+
+<br/><br/>
 ## v3.0.7.1 - 2021-03-02 - [PR #290](https://github.com/NOAA-OWP/cahaba/pull/290)
 
 Renamed benchmark layers in `test_cases` and updated variable names in evaluation scripts.

diff --git a/src/utils/shared_variables.py b/src/utils/shared_variables.py
@@ -3,7 +3,7 @@
 # Projections.
 #PREP_PROJECTION = "+proj=aea +datum=NAD83 +x_0=0.0 +y_0=0.0 +lon_0=96dW +lat_0=23dN +lat_1=29d30'N +lat_2=45d30'N +towgs84=-0.9956000824677655,1.901299877314078,0.5215002840524426,0.02591500053005733,0.009425998542707753,0.01159900118427752,-0.00062000005129903 +no_defs +units=m"
 PREP_PROJECTION = 'PROJCS["USA_Contiguous_Albers_Equal_Area_Conic_USGS_version",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.2572221010042,AUTHORITY["EPSG","7019"]],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4269"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",29.5],PARAMETER["standard_parallel_2",45.5],PARAMETER["latitude_of_center",23],PARAMETER["longitude_of_center",-96],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]'
-
+VIZ_PROJECTION ='PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]'
 # -- Data URLs-- #
 NHD_URL_PARENT = r'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/'
 NWM_HYDROFABRIC_URL = r'http://www.nohrsc.noaa.gov/pub/staff/keicher/NWM_live/web/data_tools/NWM_channel_hydrofabric.tar.gz'  # Temporary

diff --git a/tools/__init__.py b/tools/__init__.py
diff --git a/tools/aggregate_mannings_calibration.py b/tools/aggregate_mannings_calibration.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-
 import os
 import pandas as pd
 import csv

diff --git a/tools/aggregate_metrics.py b/tools/aggregate_metrics.py
@@ -3,17 +3,17 @@
 import json
 import os
 import csv
-    
+
 import argparse
 
 TEST_CASES_DIR = r'/data/test_cases_new/'
 # TEMP = r'/data/temp'
 
 # Search through all previous_versions in test_cases
-from utils.shared_functions import compute_stats_from_contingency_table
+from tools_shared_functions import compute_stats_from_contingency_table
 
 def create_master_metrics_csv():
-            
+
     # Construct header
     metrics_to_write = ['true_negatives_count',
                         'false_negatives_count',
@@ -57,33 +57,33 @@ def create_master_metrics_csv():
                         'masked_perc',
                         'masked_area_km2'
                         ]
-    
+
     additional_header_info_prefix = ['version', 'nws_lid', 'magnitude', 'huc']
     list_to_write = [additional_header_info_prefix + metrics_to_write + ['full_json_path'] + ['flow'] + ['benchmark_source']]
-    
+
     for benchmark_type in ['ble', 'ahps']:
-        
+
         if benchmark_type == 'ble':
-        
+
             test_cases = r'/data/test_cases'
             test_cases_list = os.listdir(test_cases)
             # AHPS test_ids
             versions_to_aggregate = ['fim_1_0_0', 'fim_2_3_3', 'fim_3_0_0_3_fr_c']
-                            
+
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
-                    
+
                     huc = test_case.split('_')[0]
                     previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
-                    
+
                     for magnitude in ['100yr', '500yr']:
                         for version in versions_to_aggregate:
                             version_dir = os.path.join(previous_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
 
                             if os.path.exists(magnitude_dir):
-                                
+
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
                                     if '.json' in f:
@@ -99,40 +99,40 @@ def create_master_metrics_csv():
                                             sub_list_to_append.append(full_json_path)
                                             sub_list_to_append.append(flow)
                                             sub_list_to_append.append(benchmark_source)
-                                            
+
                                             list_to_write.append(sub_list_to_append)
-                                                                                    
+
                 except ValueError:
                     pass
-                
+
         if benchmark_type == 'ahps':
-    
+
             test_cases = r'/data/test_cases_ahps_testing'
             test_cases_list = os.listdir(test_cases)
             # AHPS test_ids
-            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021', 
+            versions_to_aggregate = ['fim_1_0_0_nws_1_21_2021', 'fim_1_0_0_usgs_1_21_2021',
                                      'fim_2_x_ms_nws_1_21_2021', 'fim_2_x_ms_usgs_1_21_2021',
                                      'fim_3_0_0_3_ms_c_nws_1_21_2021', 'fim_3_0_0_3_ms_c_usgs_1_21_2021',
                                      'ms_xwalk_fill_missing_cal_nws', 'ms_xwalk_fill_missing_cal_usgs']
-            
+
             for test_case in test_cases_list:
                 try:
                     int(test_case.split('_')[0])
-                    
+
                     huc = test_case.split('_')[0]
                     previous_versions = os.path.join(test_cases, test_case, 'performance_archive', 'previous_versions')
-                    
+
                     for magnitude in ['action', 'minor', 'moderate', 'major']:
                         for version in versions_to_aggregate:
-                            
+
                             if 'nws' in version:
                                 benchmark_source = 'ahps_nws'
                             if 'usgs' in version:
                                 benchmark_source = 'ahps_usgs'
-                            
+
                             version_dir = os.path.join(previous_versions, version)
                             magnitude_dir = os.path.join(version_dir, magnitude)
-                            
+
                             if os.path.exists(magnitude_dir):
                                 magnitude_dir_list = os.listdir(magnitude_dir)
                                 for f in magnitude_dir_list:
@@ -147,7 +147,7 @@ def create_master_metrics_csv():
                                                 parent_dir = 'usgs_1_21_2021'
                                             if 'nws' in version:
                                                 parent_dir = 'nws_1_21_2021'
-                                                
+
                                             flow_file = os.path.join(test_cases, parent_dir, huc, nws_lid, magnitude, 'ahps_' + nws_lid + '_huc_' + huc + '_flows_' + magnitude + '.csv')
                                             if os.path.exists(flow_file):
                                                 with open(flow_file, newline='') as csv_file:
@@ -157,18 +157,18 @@ def create_master_metrics_csv():
                                                         flow = row[1]
                                                     if nws_lid == 'mcc01':
                                                         print(flow)
-                                            
+
                                             stats_dict = json.load(open(full_json_path))
                                             for metric in metrics_to_write:
                                                 sub_list_to_append.append(stats_dict[metric])
                                             sub_list_to_append.append(full_json_path)
                                             sub_list_to_append.append(flow)
                                             sub_list_to_append.append(benchmark_source)
                                             list_to_write.append(sub_list_to_append)
-                                        
+
                 except ValueError:
                     pass
-        
+
     with open(output_csv, 'w', newline='') as csvfile:
         csv_writer = csv.writer(csvfile)
         csv_writer.writerows(list_to_write)
@@ -201,7 +201,7 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
     for magnitude in ['100yr', '500yr', 'action', 'minor', 'moderate', 'major']:
         huc_path_list = [['huc', 'path']]
         true_positives, true_negatives, false_positives, false_negatives, cell_area, masked_count = 0, 0, 0, 0, 0, 0
-        
+
         for test_case in test_cases_dir_list:
 
             if test_case not in ['other', 'validation_data_ble', 'validation_data_legacy', 'validation_data_ahps']:
@@ -227,19 +227,19 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
                     cell_area = json_dict['cell_area_m2']
 
                     huc_path_list.append([huc, stats_json_path])
-                
-                    
+
+
             if cell_area == 0:
                 continue
-            
+
             # Pass all sums to shared function to calculate metrics.
             stats_dict = compute_stats_from_contingency_table(true_negatives, false_negatives, false_positives, true_positives, cell_area=cell_area, masked_count=masked_count)
 
             list_to_write = [['metric', 'value']]  # Initialize header.
 
             for stat in stats_dict:
                 list_to_write.append([stat, stats_dict[stat]])
-                
+
             # Map path to output directory for aggregate metrics.
             output_file = os.path.join(aggregate_output_dir, branch + '_aggregate_metrics_' + magnitude + special_string + '.csv')
 
@@ -249,7 +249,7 @@ def aggregate_metrics(config="DEV", branch="", hucs="", special_string="", outfo
                 csv_writer.writerows(list_to_write)
                 csv_writer.writerow([])
                 csv_writer.writerows(huc_path_list)
-    
+
             print()
             print("Finished aggregating for the '" + magnitude + "' magnitude. Aggregated metrics over " + str(len(huc_path_list)-1) + " test cases.")
             print()

diff --git a/tools/cache_metrics.py b/tools/cache_metrics.py
diff --git a/tools/comparing_src.py b/tools/comparing_src.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import matplotlib.pyplot as plt
 import numpy as np
 import json