Merge pull request #419 from mkavulich/feature/track_variables_throug…

…h_suite

Add initial version of ccpp_track_variables.py

scripts/ccpp_track_variables.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+# Standard modules
+import os
+import argparse
+import logging
+import glob
+
+# CCPP framework imports
+from metadata_table import find_scheme_names, parse_metadata_file
+from ccpp_prebuild import import_config, gather_variable_definitions
+from mkstatic import Suite
+from parse_checkers import registered_fortran_ddt_names
+from parse_tools import init_log, set_log_level
+from framework_env import CCPPFrameworkEnv
+
+###############################################################################
+# Set up the command line argument parser and other global variables          #
+###############################################################################
+
+###############################################################################
+# Functions and subroutines                                                   #
+###############################################################################
+
+def parse_arguments():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-s', '--sdf', help='suite definition file to parse', required=True)
+    parser.add_argument('-m', '--metadata_path',
+                        help='path to CCPP scheme metadata files', required=True)
+    parser.add_argument('-c', '--config', 
+                        help='path to CCPP prebuild configuration file', required=True)
+    parser.add_argument('-v', '--variable', help='variable to track through CCPP suite', 
+                        required=True)
+    parser.add_argument('--debug', action='store_true', help='enable debugging output', 
+                        default=False)
+
+    args = parser.parse_args()
+
+    return(args)
+
+def setup_logging(debug):
+    """Sets up the logging module and logging level."""
+
+    #Use capgen logging tools
+    logger = init_log('ccpp_track_variables')
+
+    if debug:
+        set_log_level(logger, logging.DEBUG)
+        logger.info('Logging level set to DEBUG')
+    else:
+        set_log_level(logger, logging.WARNING)
+    return logger
+
+def parse_suite(sdf, run_env):
+    """Reads the provided sdf, parses into a Suite data structure, including the "call tree":
+       the ordered list of schemes for the suite specified by the provided sdf"""
+    run_env.logger.info(f'Reading sdf {sdf} and populating Suite object')
+    suite = Suite(sdf_name=sdf)
+    run_env.logger.info(f'Reading sdf {sdf} and populating Suite object')
+    success = suite.parse(make_call_tree=True)
+    if not success:
+        raise Exception(f'Parsing suite definition file {sdf} failed.')
+    run_env.logger.info(f'Successfully read sdf {suite.sdf_name}')
+    return suite
+
+def create_metadata_filename_dict(metapath):
+    """Given a path, read all .meta files in that directory and add them to a dictionary: the keys
+       are the name of the scheme, and the values are the filename of the .meta file associated
+       with that scheme"""
+
+    metadata_dict = {}
+    scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"))
+    if not scheme_filenames:
+        raise Exception(f'No files found in {metapath} with ".meta" extension')
+
+    for scheme_fn in scheme_filenames:
+        schemes = find_scheme_names(scheme_fn)
+        # The above returns a list of schemes in each filename, but
+        # we want a dictionary of schemes associated with filenames:
+        for scheme in schemes:
+            metadata_dict[scheme] = scheme_fn
+
+    return metadata_dict
+
+
+def create_var_graph(suite, var, config, metapath, run_env):
+    """Given a suite, variable name, a 'config' dictionary, and a path to .meta files:
+         1. Creates a dictionary associating schemes with their .meta files
+         2. Loops through the call tree of the provided suite
+         3. For each scheme, reads .meta file for said scheme, checks for variable within that
+            scheme, and if it exists, adds an entry to a list of tuples, where each tuple includes
+            the name of the scheme and the intent of the variable within that scheme"""
+
+    # Create a list of tuples that will hold the in/out information for each scheme
+    var_graph = []
+
+    run_env.logger.debug(f"reading .meta files in path:\n {metapath}")
+    metadata_dict=create_metadata_filename_dict(metapath)
+
+    run_env.logger.debug(f"reading metadata files for schemes defined in config file: "
+                  f"{config['scheme_files']}")
+
+    # Loop through call tree, find matching filename for scheme via dictionary schemes_in_files,
+    # then parse that metadata file to find variable info
+    partial_matches = {}
+    for scheme in suite.call_tree:
+        run_env.logger.debug(f"reading meta file for scheme {scheme} ")
+
+        if scheme in metadata_dict:
+            scheme_filename = metadata_dict[scheme]
+        else:
+            raise Exception(f"Error, scheme '{scheme}' from suite '{suite.sdf_name}' "
+                            f"not found in metadata files in {metapath}")
+
+        run_env.logger.debug(f"reading metadata file {scheme_filename} for scheme {scheme}")
+
+        new_metadata_headers = parse_metadata_file(scheme_filename,
+                                                   known_ddts=registered_fortran_ddt_names(), run_env=run_env)
+        for scheme_metadata in new_metadata_headers:
+            for section in scheme_metadata.sections():
+                found_var = []
+                intent = ''
+                for scheme_var in section.variable_list():
+                    exact_match = False
+                    if var == scheme_var.get_prop_value('standard_name'):
+                        run_env.logger.debug(f"Found variable {var} in scheme {section.title}")
+                        found_var = var
+                        exact_match = True
+                        intent = scheme_var.get_prop_value('intent')
+                        break
+                    scheme_var_standard_name = scheme_var.get_prop_value('standard_name')
+                    if scheme_var_standard_name.find(var) != -1:
+                        run_env.logger.debug(f"{var} matches {scheme_var_standard_name}")
+                        found_var.append(scheme_var_standard_name)
+                if not found_var:
+                    run_env.logger.debug(f"Did not find variable {var} in scheme {section.title}")
+                elif exact_match:
+                    run_env.logger.debug(f"Exact match found for variable {var} in scheme {section.title},"
+                                  f" intent {intent}")
+                    var_graph.append((section.title,intent))
+                else:
+                    run_env.logger.debug(f"Found inexact matches for variable(s) {var} "
+                                  f"in scheme {section.title}:\n{found_var}")
+                    partial_matches[section.title] = found_var
+    if var_graph:
+        success = True
+        run_env.logger.debug(f"Successfully generated variable graph for sdf {suite.sdf_name}\n")
+    else:
+        success = False
+        run_env.logger.error(f"Variable {var} not found in any suites for sdf {suite.sdf_name}\n")
+        if partial_matches:
+            print("Did find partial matches that may be of interest:\n")
+            for key in partial_matches:
+                print(f"In {key} found variable(s) {partial_matches[key]}")
+
+    return (success,var_graph)
+
+def main():
+    """Main routine that traverses a CCPP suite and outputs the list of schemes that use given variable"""
+
+    args = parse_arguments()
+
+    logger = setup_logging(args.debug)
+
+    #Use new capgen class CCPPFrameworkEnv 
+    run_env = CCPPFrameworkEnv(logger, host_files="", scheme_files="", suites="")
+
+    suite = parse_suite(args.sdf,run_env)
+
+    (success, config) = import_config(args.config, None)
+    if not success:
+        raise Exception('Call to import_config failed.')
+
+    # Variables defined by the host model; this call is necessary because it converts some old
+    # metadata formats so they can be used later in the script
+    (success, _, _) = gather_variable_definitions(config['variable_definition_files'], 
+                                                   config['typedefs_new_metadata'])
+    if not success:
+        raise Exception('Call to gather_variable_definitions failed.')
+
+    (success, var_graph) = create_var_graph(suite, args.variable, config, args.metadata_path, run_env)
+    if success:
+        print(f"For suite {suite.sdf_name}, the following schemes (in order) "
+              f"use the variable {args.variable}:")
+        for entry in var_graph:
+            print(f"{entry[0]} (intent {entry[1]})")
+
+
+if __name__ == '__main__':
+    main()

scripts/metadata_parser.py

@@ -26,7 +26,7 @@
 # Output: This routine converts the argument tables for all subroutines / typedefs / kind / module variables
 # into dictionaries suitable to be used with ccpp_prebuild.py (which generates the fortran code for the caps)
 
-# Items in this dictionary are used for checking valid entries in metadata tables. For columsn with no keys/keys
+# Items in this dictionary are used for checking valid entries in metadata tables. For columns with no keys/keys
 # commented out, no check is performed. This is the case for 'type' and 'kind' right now, since models use their
 # own derived data types and kind types.
 VALID_ITEMS = {

scripts/mkstatic.py

@@ -504,6 +504,7 @@ def __init__(self, **kwargs):
         self._sdf_name = None
         self._all_schemes_called = None
         self._all_subroutines_called = None
+        self._call_tree = []
         self._caps = None
         self._module = None
         self._subroutines = None
@@ -545,7 +546,7 @@ def update_cap(self):
     def update_cap(self, value):
         self._update_cap = value
 
-    def parse(self):
+    def parse(self, make_call_tree=False):
         '''Parse the suite definition file.'''
         success = True
 
@@ -568,6 +569,10 @@ def parse(self):
         self._all_schemes_called = []
         self._all_subroutines_called = []
 
+        if make_call_tree:
+            # Call tree of all schemes in SDF (with duplicates and subcycles)
+            self._call_tree = []
+
         # Build hierarchical structure as in SDF
         self._groups = []
         for group_xml in suite_xml:
@@ -594,14 +599,22 @@ def parse(self):
                     loop=int(subcycle_xml.get('loop'))
                     for ccpp_stage in CCPP_STAGES:
                         self._all_subroutines_called.append(scheme_xml.text + '_' + CCPP_STAGES[ccpp_stage])
+
                 subcycles.append(Subcycle(loop=loop, schemes=schemes))
 
+                if make_call_tree:
+                    # Populate call tree from SDF's heirarchical structure, including multiple calls in subcycle loops
+                    for loop in range(0,int(subcycle_xml.get('loop'))):
+                        for scheme_xml in subcycle_xml:
+                            self._call_tree.append(scheme_xml.text)
+
             self._groups.append(Group(name=group_xml.get('name'), subcycles=subcycles, suite=self._name))
 
         # Remove duplicates from list of all subroutines an schemes
         self._all_schemes_called = list(set(self._all_schemes_called))
         self._all_subroutines_called = list(set(self._all_subroutines_called))
 
+
         return success
 
     def print_debug(self):
@@ -618,6 +631,11 @@ def all_schemes_called(self):
         '''Get the list of all schemes.'''
         return self._all_schemes_called
 
+    @property
+    def call_tree(self):
+        '''Get the call tree of the suite (all schemes, in order, with duplicates and loops).'''
+        return self._call_tree
+
     @property
     def all_subroutines_called(self):
         '''Get the list of all subroutines.'''