Add support for ignore rules (#184)

Author: ildyria

components/resc-vcs-scanner/README.md

@@ -163,7 +163,7 @@ To create vcs_instances_config.json file please refer to: [Structure of vcs_inst
   cd components/resc-vcs-scanner
   pip install virtualenv
   virtualenv venv
-  source venv/Scripts/activate
+  source venv/bin/activate
   ```
  #### 2. Install resc_vcs_scanner package:
   ```bash
@@ -174,19 +174,19 @@ The CLI has 3 modes of operation, please make use of the --help argument to see
 - Scanning a non-git directory: 
   ```bash
   secret_scanner dir --help
-  secret_scanner dir --gitleaks-rules-path=<path to gitleaks toml rule> --gitleaks-path=<path to gitleaks binary> --dir=<directory to scan>
+  secret_scanner dir --gitleaks-rules-path=<path to gitleaks toml rule> --gitleaks-path=<path to gitleaks binary> --ignored-blocker-path=<path to resc-ignore.dsv file> --dir=<directory to scan>
   ```
 
 - Scanning an already cloned git repository: 
   ```bash
   secret_scanner repo local --help
-  secret_scanner repo local --gitleaks-rules-path=<path to gitleaks toml rule> --gitleaks-path=<path to gitleaks binary> --dir=<directory of repository to scan>
+  secret_scanner repo local --gitleaks-rules-path=<path to gitleaks toml rule> --gitleaks-path=<path to gitleaks binary> --ignored-blocker-path=<path to resc-ignore.dsv file> --dir=<directory of repository to scan>
   ```
 
 - Scanning a remote git repository: 
   ```bash
   secret_scanner repo remote --help
-  secret_scanner repo remote --gitleaks-rules-path=<path to gitleaks toml rule> --gitleaks-path=<path to gitleaks binary> --repo-url=<url of repository to scan>
+  secret_scanner repo remote --gitleaks-rules-path=<path to gitleaks toml rule> --gitleaks-path=<path to gitleaks binary> --ignored-blocker-path=<path to resc-ignore.dsv file> --repo-url=<url of repository to scan>
   ```
 Most CLI arguments can also be provided by setting the corresponding environment variable. 
 Please see the --help options on the arguments that can be provided using environment variables, and the expected environment variable names.
@@ -195,6 +195,37 @@ These will always be prefixed with RESC_
 Example: the argument **--gitleaks-path** can be provided using the environment variable **RESC_GITLEAKS_PATH**
 </details>
 
+### Ignoring findings
+
+<details>
+  <summary>Preview</summary>
+
+It is possible to ignore some blocker findings (e.g. false positive) by providing
+a `resc-ignore.dsv` file. The bockers will be downgraded to a warning level and marked as **ignored**. Such file has the following structure:
+
+```sh
+# This is a comment
+finding_path|finding_rule|finding_line_number|expiration_date
+finding_path_2|finding_rule_2|finding_line_number_2
+```
+
+- `finding_path` contains the path to the file with the blocking finding.
+- `finding_rule` contains the name of the blocking rule.
+- `finding_line_number` contains the line number of the finding.
+- `expiration_date` is optional, contains the date in ISO 8601 format until which this ignore rule should be considered valid.
+
+For example, if we want to ignore the finding in file `/etc/passwd` for rule `root_value_found` on line `1` until April 1st 2024 at 23:59 the following line should be used.
+```sh
+/etc/passwd|root_value_found|1|2024-04-01T23:59:00
+```
+To ignore this finding _ad vitam aeternam_:
+```sh
+/etc/passwd|root_value_found|1
+```
+
+
+</details>
+
 ## Testing 
 Run below commands to make sure that the unit tests are running and that the code matches quality standards:
 

components/resc-vcs-scanner/src/vcs_scanner/helpers/finding_action.py

@@ -5,4 +5,5 @@
 class FindingAction(str, Enum):
     INFO = "Info"
     WARN = "Warn"
+    IGNORED = "Ignored"
     BLOCK = "Block"

components/resc-vcs-scanner/src/vcs_scanner/secret_scanners/cli.py

@@ -47,6 +47,11 @@ def create_cli_argparser() -> ArgumentParser:
                                envvar="RESC_GITLEAKS_RULES_PATH", help="Path to the gitleaks rules file. "
                                                                        "Can also be set via the "
                                                                        "RESC_GITLEAKS_RULES_PATH environment variable")
+    parser_common.add_argument("--ignored-blocker-path", type=pathlib.Path, action=EnvDefault, required=False,
+                               envvar="RESC_IGNORED_BLOCKER_PATH", help="Path to the resc-ignore.dsv file. "
+                                                                        "Can also be set via the "
+                                                                        "RESC_IGNORED_BLOCKER_PATH "
+                                                                        "environment variable")
     parser_common.add_argument("-w", "--exit-code-warn", required=False, action=EnvDefault, default=2, type=int,
                                envvar="RESC_EXIT_CODE_WARN",
                                help="Exit code given if CLI encounters findings tagged with Warn, default 2. "
@@ -199,8 +204,10 @@ def scan_directory(args: Namespace):
     )
 
     output_plugin = STDOUTWriter(toml_rule_file_path=args.gitleaks_rules_path,
-                                 exit_code_warn=args.exit_code_warn, exit_code_block=args.exit_code_block,
-                                 filter_tag=args.filter_tag)
+                                 exit_code_warn=args.exit_code_warn,
+                                 exit_code_block=args.exit_code_block,
+                                 filter_tag=args.filter_tag,
+                                 ignore_findings_path=args.ignored_blocker_path)
     with open(args.gitleaks_rules_path, encoding="utf-8") as rule_pack:
         rule_pack_version = get_rule_pack_version_from_file(rule_pack.read())
     if not rule_pack_version:
@@ -244,8 +251,10 @@ def scan_repository(args: Namespace):
 
     else:
         output_plugin = STDOUTWriter(toml_rule_file_path=args.gitleaks_rules_path,
-                                     exit_code_warn=args.exit_code_warn, exit_code_block=args.exit_code_block,
-                                     filter_tag=args.filter_tag)
+                                     exit_code_warn=args.exit_code_warn,
+                                     exit_code_block=args.exit_code_block,
+                                     filter_tag=args.filter_tag,
+                                     ignore_findings_path=args.ignored_blocker_path)
         with open(args.gitleaks_rules_path, encoding="utf-8") as rule_pack:
             rule_pack_version = get_rule_pack_version_from_file(rule_pack.read())
     if not rule_pack_version:

components/resc-vcs-scanner/src/vcs_scanner/secret_scanners/ignore_list_provider.py

@@ -0,0 +1,59 @@
+# pylint: disable=E1101
+# Standard Library
+import csv
+import logging
+from datetime import datetime
+
+# Third Party
+import dateutil.parser
+
+logger = logging.getLogger(__name__)
+
+
+class IgnoredListProvider():  # pylint: disable=R0902
+
+    def __init__(self, ignore_findings_path: str):
+        self.ignore_findings_path: str = ignore_findings_path
+        self.today: datetime = datetime.now()
+
+    def get_ignore_list(self) -> dict:
+        """
+            Get the dictionary of ignored findings according to the file
+            The output will contain a dictionary with the findings id as the key and the tags as a list in the value
+        """
+        ignored = {}
+
+        try:
+            # read dsv: `path|rule_name|line_number|expiry_date`
+            with open(self.ignore_findings_path, encoding="utf-8") as ignore_findings_file:
+                csv_ignore_list = csv.reader(ignore_findings_file, delimiter='|')
+                for row in csv_ignore_list:
+                    expire: datetime = datetime.now()
+
+                    # rows starting with # are comments
+                    if row[0][:1] == '#':
+                        continue
+
+                    if len(row) < 3:
+                        string_row: str = "".join(row)
+                        logger.warning(f"Skipping: incomplete entry for {string_row}")
+                        continue
+
+                    if len(row) > 3:
+                        date = row[3]
+                        try:
+                            expire: datetime = dateutil.parser.isoparse(date)
+                        except ValueError:
+                            logger.warning(f"Skipping: invalid date entry for {date}")
+                            continue
+
+                    if expire < self.today:
+                        continue
+
+                    # we use the path, rule_name, line_number as a dictionary key
+                    ignored[row[0] + '|' + row[1] + '|' + row[2]] = True
+        except FileNotFoundError:  # <- File does not exists: we just fail silently
+            logger.warning(f"could not find {self.ignore_findings_path}")
+            return {}
+
+        return ignored

components/resc-vcs-scanner/src/vcs_scanner/secret_scanners/stdout_writer.py

@@ -18,20 +18,28 @@
 from vcs_scanner.helpers.finding_action import FindingAction
 from vcs_scanner.model import VCSInstanceRuntime
 from vcs_scanner.output_module import OutputModule
+from vcs_scanner.secret_scanners.ignore_list_provider import IgnoredListProvider
 
 logger = logging.getLogger(__name__)
 
 
 class STDOUTWriter(OutputModule):
 
-    def __init__(self, toml_rule_file_path: str, exit_code_warn: int, exit_code_block: int, filter_tag: str = None):
+    def __init__(self,
+                 toml_rule_file_path: str,
+                 exit_code_warn: int,
+                 exit_code_block: int,
+                 filter_tag: str = None,
+                 ignore_findings_path: str = ""):
         self.toml_rule_file_path: str = toml_rule_file_path
         self.exit_code_warn: int = exit_code_warn
         self.exit_code_block: int = exit_code_block
         self.filter_tag: str = filter_tag
         self.exit_code_success = 0
+        self.ignore_findings_providers: IgnoredListProvider = IgnoredListProvider(ignore_findings_path)
 
-    def write_vcs_instance(self, vcs_instance_runtime: VCSInstanceRuntime) -> Optional[VCSInstanceRead]:
+    def write_vcs_instance(self,
+                           vcs_instance_runtime: VCSInstanceRuntime) -> Optional[VCSInstanceRead]:
         vcs_instance = VCSInstanceRead(id_=1,
                                        name=vcs_instance_runtime.name,
                                        provider_type=vcs_instance_runtime.provider_type,
@@ -66,13 +74,15 @@ def _get_rule_tags(self) -> dict:
         return rule_tags
 
     @staticmethod
-    def _determine_finding_action(finding: FindingCreate, rule_tags: dict) -> FindingAction:
+    def _determine_finding_action(finding: FindingCreate, rule_tags: dict, ignore_dictionary: dict) -> FindingAction:
         """
             Determine the action to take for the finding, based on the rule tags
         :param finding:
             FindingCreate instance of the finding
         :param rule_tags:
-            Dictionary continuing all the rules and there respective tags
+            Dictionary containing all the rules and there respective tags
+        :param ignore_dictionary:
+            Dictionary containing all the list of ignored blockers
         :return: FindingAction.
             FindingAction to take for this finding
         """
@@ -81,6 +91,12 @@ def _determine_finding_action(finding: FindingCreate, rule_tags: dict) -> Findin
             rule_action = FindingAction.WARN
         if FindingAction.BLOCK in rule_tags.get(finding.rule_name, []):
             rule_action = FindingAction.BLOCK
+
+        if rule_action == FindingAction.BLOCK:
+            key: str = finding.file_path + "|" + finding.rule_name + "|" + str(finding.line_number)
+            if key in ignore_dictionary:
+                rule_action = FindingAction.IGNORED
+
         return rule_action
 
     def _finding_tag_filter(self, finding: FindingCreate, rule_tags: dict, filter_tag: str) -> bool:
@@ -89,7 +105,7 @@ def _finding_tag_filter(self, finding: FindingCreate, rule_tags: dict, filter_ta
         :param finding:
             FindingCreate instance of the finding
         :param rule_tags:
-            Dictionary continuing all the rules and there respective tags
+            Dictionary containing all the rules and there respective tags
         :Param: filter_tag.
             filter_tag will check for the tag
         :return bool:
@@ -122,14 +138,15 @@ def write_findings(self, scan_id: int, repository_id: int, scan_findings: List[F
         exit_code = self.exit_code_success
 
         rule_tags = self._get_rule_tags()
+        ignore_dictionary = self.ignore_findings_providers.get_ignore_list()
         for finding in scan_findings:
             should_process_finding = self._finding_tag_filter(finding, rule_tags, self.filter_tag)
             if should_process_finding:
-                finding_action = self._determine_finding_action(finding, rule_tags)
+                finding_action = self._determine_finding_action(finding, rule_tags, ignore_dictionary)
                 if finding_action == FindingAction.BLOCK:
                     finding_action_value = colored(finding_action.value, "red", attrs=["bold"])
                     block_count += 1
-                elif finding_action == FindingAction.WARN:
+                elif finding_action in [FindingAction.WARN, FindingAction.IGNORED]:
                     finding_action_value = colored(finding_action.value, "light_red", attrs=["bold"])
                     warn_count += 1
                 elif finding_action == FindingAction.INFO:
@@ -140,7 +157,8 @@ def write_findings(self, scan_id: int, repository_id: int, scan_findings: List[F
                     info_count += 1
 
                 if exit_code != self.exit_code_block:
-                    if exit_code == self.exit_code_success and finding_action == FindingAction.WARN:
+                    if exit_code == self.exit_code_success and \
+                       finding_action in [FindingAction.WARN, FindingAction.IGNORED]:
                         exit_code = self.exit_code_warn
                     elif finding_action == FindingAction.BLOCK:
                         exit_code = self.exit_code_block

components/resc-vcs-scanner/tests/fixtures/ignore-findings-list-for-writer.dsv

@@ -0,0 +1 @@
+file_path_1|rule_1|1

components/resc-vcs-scanner/tests/fixtures/ignore-findings-list.dsv

@@ -0,0 +1,6 @@
+#commented_path|commented_rule|commented_line_number|expiry_date
+empty_line
+expired_path|expired_rule|38|2022-09-09T15:30:30
+wrong_date_path|wrong_date_rule|38|2022X09-09T15:30:30
+active_path|active_rule|57|2122-09-09T15:30:30
+active_path_2|active_rule_2|58

components/resc-vcs-scanner/tests/fixtures/rules.toml

@@ -0,0 +1,21 @@
+title = "gitleaks config"
+
+version = "2.0.13"
+
+[[rules]]
+	id = "rule_1"
+	description = "Fake rule"
+	regex = '''something'''
+	tags = ["Block"]
+
+[[rules]]
+	id = "rule_2"
+	description = "Another Fake rule"
+	regex = '''something'''
+	tags = ["Block"]
+
+[[rules]]
+	id = "rule_3"
+	description = "Another Fake rule again"
+	regex = '''something'''
+	tags = ["Warn"]

components/resc-vcs-scanner/tests/vcs_scanner/secret_scanners/test_ignore_list_provider.py

@@ -0,0 +1,15 @@
+# Standard Library
+from pathlib import Path
+
+# First Party
+from vcs_scanner.secret_scanners.ignore_list_provider import IgnoredListProvider
+
+THIS_DIR = Path(__file__).parent.parent
+
+
+# We check that given a file, we get only 1 line:
+def test_ignore_list_provider():
+    ignore_list_path = THIS_DIR.parent / "fixtures/ignore-findings-list.dsv"
+    listProvider = IgnoredListProvider(str(ignore_list_path))
+    assert {'active_path|active_rule|57': True,
+            'active_path_2|active_rule_2|58': True} == listProvider.get_ignore_list()