add option --filter and --helpfilter in buildtest report (#449)

* add option --filter and --helpfilter in buildtest report
We can now filter by buildspec,name,state,returncode,executor,tags.
The values must be passed as key=value and multiple argument can
be separated by comma. If multiple filter argument are specified it
will perform a logical AND.
The --filter and --format can be used together to filter and format
report.
Add test coverage for filter options

* add comments

buildtest/menu/__init__.py

@@ -23,6 +23,41 @@
 from buildtest.menu.schema import func_schema
 
 
+def handle_kv_string(val):
+    """This method is used as type field in --filter argument in ``buildtest buildspec find``.
+       This method returns a dict of key,value pair where input is in format
+       key1=val1,key2=val2,key3=val3
+
+       :param val: input value
+       :type val: str
+       :return: dictionary of key/value pairs
+       :rtype: dict
+    """
+
+    kv_dict = {}
+
+    if "," in val:
+        args = val.split(",")
+        for kv in args:
+            if "=" in kv:
+                key, value = kv.split("=")[0], kv.split("=")[1]
+                kv_dict[key] = value
+            else:
+                raise argparse.ArgumentTypeError("Must specify k=v")
+
+    else:
+        if "=" in val:
+            key, value = val.split("=")[0], val.split("=")[1]
+            kv_dict[key] = value
+
+    return kv_dict
+    # if '=' in split_args
+    # if '=' in val:
+    #    return val.split('=')
+    # else:
+    #    raise argparse.ArgumentTypeError('Must specify k=v')
+
+
 class BuildTestParser:
     def __init__(self):
         epilog_str = (
@@ -270,7 +305,16 @@ def report_menu(self):
             "--format",
             help="format field for printing purposes. For more details see --helpformat for list of available fields. Fields must be separated by comma (--format <field1>,<field2>,...)",
         )
-
+        parser_report.add_argument(
+            "--filter",
+            type=handle_kv_string,
+            help="Filter report by filter fields. The filter fields must be set in format: --filter key1=val1,key2=val2,...",
+        )
+        parser_report.add_argument(
+            "--helpfilter",
+            action="store_true",
+            help="Report a list of filter fields to be used with --filter option",
+        )
         ##################### buildtest report   ###########################
 
         parser_report.set_defaults(func=func_report)

buildtest/menu/report.py

@@ -3,7 +3,7 @@
 import sys
 from tabulate import tabulate
 from buildtest.defaults import BUILD_REPORT
-from buildtest.utils.file import is_file, create_dir
+from buildtest.utils.file import is_file, create_dir, resolve_path
 
 
 def func_report(args=None):
@@ -70,6 +70,43 @@ def func_report(args=None):
         )
         return
 
+    filter_field_table = [
+        ["buildspec", "Filter by buildspec file", "FILE"],
+        ["name", "Filter by test name", "STRING"],
+        ["executor", "Filter by executor name", "STRING"],
+        ["state", "Filter by test state ", "PASS/FAIL"],
+        ["tags", "Filter tests by tag name ", "STRING"],
+        ["returncode", "Filter tests by returncode ", "INT"],
+    ]
+    filter_fields = ["buildspec", "name", "executor", "state", "tags", "returncode"]
+    # filter_args contains a dict of filter field argument
+    filter_args = {}
+
+    if args.helpfilter:
+        print(
+            tabulate(
+                filter_field_table,
+                headers=["Filter Fields", "Description", "Expected Value"],
+                tablefmt="simple",
+            )
+        )
+        return
+
+    if args.filter:
+
+        filter_args = args.filter
+
+        raiseError = False
+        # check if filter keys are accepted filter fields, if not we raise error
+        for key in filter_args.keys():
+            if key not in filter_fields:
+                print(f"Invalid filter key: {key}")
+                raiseError = True
+
+        # raise error if any filter field is invalid
+        if raiseError:
+            sys.exit(1)
+
     # default table format fields
     display_table = {
         "id": [],
@@ -85,7 +122,7 @@ def func_report(args=None):
     fields = display_table.keys()
 
     # if buildtest report --format specified split field by "," and validate each
-    # format field and generate display_table
+    # format field and reassign display_table
     if args.format:
         fields = args.format.split(",")
 
@@ -94,15 +131,80 @@ def func_report(args=None):
             if field not in format_fields:
                 sys.exit(f"Invalid format field: {field}")
 
+        # reassign display_table to format fields
         display_table = {}
 
         for field in fields:
             display_table[field] = []
 
-    for buildspec in report.keys():
+    filter_buildspecs = report.keys()
+
+    # This section filters the buildspec, if its invalid file or not found in cache
+    # we raise error, otherwise we set filter_buildspecs to the filter argument 'buildspec'
+    if filter_args.get("buildspec"):
+        # resolve path for buildspec filter key, its possible if file doesn't exist method returns None
+        resolved_buildspecs = resolve_path(filter_args["buildspec"])
+
+        # if file doesn't exist we terminate with message
+        if not resolved_buildspecs:
+            print(
+                f"Invalid File Path for filter field 'buildspec': {filter_args['buildspec']}"
+            )
+            sys.exit(0)
+
+        # if file not found in cache we exit
+        if not resolved_buildspecs in report.keys():
+            print(f"buildspec file: {resolved_buildspecs} not found in cache")
+            sys.exit(0)
+
+        # need to set as a list since we will loop over all tests
+        filter_buildspecs = [resolved_buildspecs]
+
+    # ensure 'state' field in filter is either 'PASS' or 'FAIL', if not raise error
+    if filter_args.get("state"):
+        if filter_args["state"] not in ["PASS", "FAIL"]:
+            print(
+                f"filter argument 'state' must be 'PASS' or 'FAIL' got value {filter_args['state']}"
+            )
+            sys.exit(0)
+
+    # process all filtered buildspecs and add rows to display_table.
+    # filter_buildspec is either all buildspec or a single buildspec if
+    # 'buildspec' filter field was set
+    for buildspec in filter_buildspecs:
+
+        # process each test in buildspec file
         for name in report[buildspec].keys():
+
+            if filter_args.get("name"):
+                # skip tests that don't equal filter 'name' field
+                if name != filter_args["name"]:
+                    continue
+
+            # process all tests for an associated script. There can be multiple
+            # test runs for a single test depending on how many tests were run
             for test in report[buildspec][name]:
 
+                # filter by tags, if filter tag not found in test tag list we skip test
+                if filter_args.get("tags"):
+                    if filter_args["tags"] not in test.get("tags"):
+                        continue
+
+                # if 'executor' filter defined, skip test that don't match executor key
+                if filter_args.get("executor"):
+                    if filter_args.get("executor") != test.get("executor"):
+                        continue
+
+                # if state filter defined, skip any tests that don't match test state
+                if filter_args.get("state"):
+                    if filter_args["state"] != test.get("state"):
+                        continue
+
+                # if state filter defined, skip any tests that don't match test state
+                if filter_args.get("returncode"):
+                    if int(filter_args["returncode"]) != test.get("returncode"):
+                        continue
+
                 if "buildspec" in display_table.keys():
                     display_table["buildspec"].append(buildspec)
 

tests/menu/test_report.py

@@ -1,7 +1,9 @@
 import os
 import pytest
+import random
 import shutil
-from buildtest.defaults import BUILD_REPORT
+import string
+from buildtest.defaults import BUILD_REPORT, BUILDTEST_ROOT
 from buildtest.menu.report import func_report
 
 
@@ -11,32 +13,152 @@ def test_report_format():
 
     class args:
         helpformat = False
+        helpfilter = False
         format = None
+        filter = None
 
+    # run 'buildtest report'
     func_report(args)
 
     class args:
         helpformat = False
+        helpfilter = False
         format = "name,state,returncode,buildspec"
+        filter = None
 
+    # run 'buildtest report --format name,state,returncode,buildspec'
     func_report(args)
 
     class args:
         helpformat = False
+        helpfilter = False
         format = "badfield,state,returncode"
+        filter = None
 
+    # specify invalid format field 'badfield'
     with pytest.raises(SystemExit):
         func_report(args)
 
 
 def test_report_helpformat():
     class args:
         helpformat = True
+        helpfilter = False
         format = None
+        filter = None
 
     func_report(args)
 
 
+def test_report_filter():
+    class args:
+        helpformat = False
+        helpfilter = True
+        format = None
+        filter = None
+
+    # run 'buildtest report --helpfilter'
+    func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {"state": "PASS"}
+        format = None
+
+    # run 'buildtest report --filter state=PASS'
+    func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {"state": "PASS"}
+        format = "name,state"
+
+    # run 'buildtest report --filter state=PASS --format name,state'
+    func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {"state": "UNKNOWN"}
+        format = "name,state"
+
+    # run 'buildtest report --filter state=UNKNOWN --format name,state',
+    # this raises error because UNKNOWN is not valid value for state field
+    with pytest.raises(SystemExit):
+        func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {"returncode": "0", "executor": "local.bash"}
+        format = "name,returncode,executor"
+
+    # run 'buildtest report --filter returncode=0,executor=local.bash --format name,returncode,executor
+    func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {
+            "buildspec": os.path.join(
+                BUILDTEST_ROOT, "tutorials", "pass_returncode.yml"
+            )
+        }
+        format = "name,returncode,buildspec"
+
+    # run 'buildtest report --filter buildspec=tutorials/pass_returncode.yml --format name,returncode,buildspec
+    func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {"name": "exit1_pass"}
+        format = "name,returncode,state"
+
+    # run 'buildtest report --filter name=exit1_pass --format name,returncode,state
+    func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {
+            "buildspec": "".join(random.choice(string.ascii_letters) for i in range(10))
+        }
+        format = "name,returncode,state"
+
+    # the filter argument buildspec is a random string which will be invalid file
+    # and we expect an exception to be raised
+    with pytest.raises(SystemExit):
+        func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {"buildspec": "$HOME/.bashrc"}
+        format = "name,returncode,state"
+
+    # run 'buildtest report --filter buildspec=$HOME/.bashrc --format name,returncode,state
+    # this will raise error even though file is valid it won't be found in cache
+    with pytest.raises(SystemExit):
+        func_report(args)
+
+    class args:
+        helpformat = False
+        helpfilter = False
+        filter = {
+            "tags": "tutorials",
+            "executor": "local.bash",
+            "state": "PASS",
+            "returncode": 0,
+        }
+        format = "name,returncode,state,executor,tags"
+
+    # run 'buildtest report --filter tags=tutorials,executor=local.bash,state=PASS,returncode=0 --format name,returncode,state,executor,tags
+    func_report(args)
+
+
 def test_func_report_when_BUILD_REPORT_missing():
 
     backupfile = f"{BUILD_REPORT}.bak"