expose checks for valid queries through the RecManager.

Author: davemfish

src/natcap/invest/recreation/recmodel_client.py

@@ -646,6 +646,26 @@ def _retrieve_user_days(
         'flickr': 'PUD',
         'twitter': 'TUD'
     }
+    for dataset in dataset_list:
+        # validate available year range
+        min_year, max_year = recmodel_manager.get_valid_year_range(dataset)
+        LOGGER.info(
+            f"{dataset} server supports year queries between {min_year} and {max_year}")
+        if not min_year <= int(start_year) <= max_year:
+            raise ValueError(
+                f"Start year must be between {min_year} and {max_year}.\n"
+                f" User input: ({start_year})")
+        if not min_year <= int(end_year) <= max_year:
+            raise ValueError(
+                f"End year must be between {min_year} and {max_year}.\n"
+                f" User input: ({end_year})")
+        n_points, max_allowable = recmodel_manager.get_aoi_query_size(dataset)
+        if n_points > max_allowable:
+            raise ValueError(
+                f'The AOI extent is too large. Its bounding box contains '
+                f'{n_points} {dataset} points. Please reduce the extent of '
+                f'the AOI until it contains fewer than '
+                f'{max_allowable} points.')
     results = recmodel_manager.calculate_userdays(
         zip_file_binary, start_year, end_year, dataset_list)
     for dataset in dataset_list:

src/natcap/invest/recreation/recmodel_server.py

@@ -44,6 +44,8 @@
 CSV_ROWS_PER_PARSE = 2 ** 10
 LOGGER_TIME_DELAY = 5.0
 INITIAL_BOUNDING_BOX = [-180, -90, 180, 90]
+# Max points within an AOI bounding box before rejecting the AOI.
+MAX_ALLOWABLE_QUERY = 30_000_000
 
 Pyro5.config.SERIALIZER = 'marshal'  # lets us pass null bytes in strings
 
@@ -85,7 +87,8 @@ def __init__(
             raw_csv_filename=None,
             quadtree_pickle_filename=None,
             max_points_per_node=GLOBAL_MAX_POINTS_PER_NODE,
-            max_depth=GLOBAL_DEPTH, dataset_name='flickr'):
+            max_depth=GLOBAL_DEPTH, dataset_name='flickr',
+            max_allowable_query=MAX_ALLOWABLE_QUERY):
         """Initialize RecModel object.
 
         The object can be initialized either with a path to a CSV file
@@ -156,6 +159,7 @@ def __init__(
         # self.global_cache_dir = global_cache
         self.min_year = min_year
         self.max_year = max_year
+        self.max_allowable_query = max_allowable_query
         self.acronym = 'PUD' if dataset_name == 'flickr' else 'TUD'
 
     def get_valid_year_range(self):
@@ -198,6 +202,9 @@ def fetch_workspace_aoi(self, workspace_id):  # pylint: disable=no-self-use
         with open(out_zip_file_path, 'rb') as out_zipfile:
             return out_zipfile.read()
 
+    def get_aoi_query_size(self):
+        return (50_000_000, self.max_allowable_query)
+
     # @_try_except_wrapper("exception in calc_user_days_in_aoi")
     def calc_user_days_in_aoi(
             self, zip_file_binary, date_range, out_vector_filename):
@@ -982,25 +989,33 @@ class RecManager(object):
     def __init__(self, servers_dict):
         self.servers = servers_dict
 
+    def get_valid_year_range(self, dataset):
+        server = self.servers[dataset]
+        return server.get_valid_year_range()
+
+    def get_aoi_query_size(self, dataset):
+        server = self.servers[dataset]
+        return server.get_aoi_query_size()
+
     @_try_except_wrapper("calculate_userdays exited while multiprocessing.")
     def calculate_userdays(self, zip_file_binary, start_year, end_year, dataset_list):
         results = {}
         with concurrent.futures.ProcessPoolExecutor(max_workers=2) as executor:
             future_to_label = {}
             for dataset in dataset_list:
                 server = self.servers[dataset]
-                # validate available year range
-                min_year, max_year = server.get_valid_year_range()
-                LOGGER.info(
-                    f"Server supports year queries between {min_year} and {max_year}")
-                if not min_year <= int(start_year) <= max_year:
-                    raise ValueError(
-                        f"Start year must be between {min_year} and {max_year}.\n"
-                        f" User input: ({start_year})")
-                if not min_year <= int(end_year) <= max_year:
-                    raise ValueError(
-                        f"End year must be between {min_year} and {max_year}.\n"
-                        f" User input: ({end_year})")
+            #     # validate available year range
+            #     min_year, max_year = server.get_valid_year_range()
+            #     LOGGER.info(
+            #         f"Server supports year queries between {min_year} and {max_year}")
+            #     if not min_year <= int(start_year) <= max_year:
+            #         raise ValueError(
+            #             f"Start year must be between {min_year} and {max_year}.\n"
+            #             f" User input: ({start_year})")
+            #     if not min_year <= int(end_year) <= max_year:
+            #         raise ValueError(
+            #             f"End year must be between {min_year} and {max_year}.\n"
+            #             f" User input: ({end_year})")
 
                 # append jan 1 to start and dec 31 to end
                 date_range = (str(start_year)+'-01-01',

tests/test_recreation.py

@@ -246,11 +246,11 @@ def test_local_aggregate_points(self):
         # transfer zipped file to server
         date_range = (('2005-01-01'), ('2014-12-31'))
         out_vector_filename = 'test_aoi_for_subset_pud.shp'
-        print('calc photo userdays')
+
         zip_result, workspace_id, version_str = (
             recreation_server.calc_user_days_in_aoi(
                 zip_file_binary, date_range, out_vector_filename))
-        print('calc photo userdays done')
+
         # unpack result
         result_zip_path = os.path.join(self.workspace_dir, 'pud_result.zip')
         with open(result_zip_path, 'wb') as file:
@@ -427,7 +427,7 @@ def test_construct_query_twitter_qt(self):
         # user,date,lat,lon
         # 1117195232,2023-01-01,-22.908,-43.1975
         # 54900515,2023-01-01,44.62804,10.60603
-        
+
         def make_twitter_csv(target_filename):
             dates = numpy.arange(
                 numpy.datetime64('2017-01-01'), numpy.datetime64('2017-12-31'))
@@ -780,6 +780,34 @@ def test_end_year_out_of_range(self):
         with self.assertRaises(ValueError):
             recmodel_client.execute(args)
 
+    def test_aoi_too_large(self):
+        """Test server checks aoi size; client raises exception."""
+        from natcap.invest.recreation import recmodel_client
+
+        args = {
+            'aoi_path': os.path.join(SAMPLE_DATA, 'andros_aoi.shp'),
+            'cell_size': 7000.0,
+            'compute_regression': True,
+            'start_year': MIN_YEAR,
+            'end_year': MAX_YEAR,
+            'grid_aoi': True,
+            'grid_type': 'hexagon',
+            'predictor_table_path': os.path.join(
+                SAMPLE_DATA, 'predictors.csv'),
+            'results_suffix': '',
+            'scenario_predictor_table_path': os.path.join(
+                SAMPLE_DATA, 'predictors_scenario.csv'),
+            'workspace_dir': self.workspace_dir,
+            'hostname': self.hostname,
+            'port': self.port
+        }
+
+        with self.assertRaises(ValueError) as cm:
+            recmodel_client.execute(args)
+        actual_message = str(cm.exception)
+        expected_message = 'The AOI extent is too large'
+        self.assertIn(expected_message, actual_message)
+
 
 class RecreationClientRegressionTests(unittest.TestCase):
     """Regression & Unit tests for recmodel_client."""