2353459 Helm Wizard Multiple bug fix and improvements

deployment/resc-helm-wizard/setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
 name = resc_helm_wizard
 description = Repository Scanner - Helm Wizard
-version = 1.0.4
+version = 1.0.5
 author = ABN AMRO
 author_email = resc@nl.abnamro.com
 url = https://github.com/ABNAMRO/repository-scanner

deployment/resc-helm-wizard/src/resc_helm_wizard/common.py

@@ -1,7 +1,6 @@
 # Standard Library
 import logging
 import os
-import subprocess
 import sys
 from typing import List
 from urllib.parse import urlparse
@@ -15,8 +14,8 @@
 from resc_helm_wizard import constants, questions
 from resc_helm_wizard.helm_utilities import (
     add_helm_repository,
+    check_helm_release_exists,
     install_or_upgrade_helm_release,
-    is_chart_already_installed,
     update_helm_repository,
     validate_helm_deployment_status
 )
@@ -294,7 +293,8 @@ def download_rule_toml_file(url: str, file: str) -> bool:
         Returns true if rule downloaded successfully else returns false
     """
     downloaded = False
-    response = requests.get(url, timeout=100, verify=True)
+    verify_ssl = questions.ask_ssl_verification(msg="Do you want to verify SSL certificates for HTTPS requests?")
+    response = requests.get(url, timeout=100, verify=verify_ssl)
     with open(file, "wb") as output:
         output.write(response.content)
     if os.path.exists(file) and os.path.getsize(file) > 0:
@@ -334,11 +334,9 @@ def run_deployment():
         namespace_created = create_namespace_if_not_exists(namespace_name=constants.NAMESPACE)
 
     if namespace_created:
-        # Check if release is already installed
-        output = subprocess.run(["helm", "list", "-n", constants.NAMESPACE], capture_output=True, text=True, check=True)
-        # Check if deployment is already running
-        chart_installed = is_chart_already_installed()
-        if constants.RELEASE_NAME in output.stdout and chart_installed:
+        # Check if release already exists
+        helm_release_exists = check_helm_release_exists()
+        if helm_release_exists:
             run_upgrade_confirm_msg = f"Release {constants.RELEASE_NAME} is already installed in " \
                                       f"{constants.NAMESPACE} namespace. Do you want to upgrade the release?"
             run_upgrade_confirm = questions.ask_user_confirmation(msg=run_upgrade_confirm_msg)

deployment/resc-helm-wizard/src/resc_helm_wizard/constants.py

@@ -4,5 +4,8 @@
 RESC_HELM_REPO_URL = "https://abnamro.github.io/repository-scanner"
 HELM_REPO_NAME = "resc-helm-repo"
 RELEASE_NAME = "resc"
-CHART_NAME = "resc"
+CHART_NAME = "resc-helm-repo/resc"
 VALUES_FILE = "custom-values.yaml"
+DEFAULT_GITHUB_URL = "https://github.com"
+DEFAULT_AZURE_DEVOPS_URL = "https://dev.azure.com"
+HELM_DEPLOY_TIMEOUT = "20m0s"

deployment/resc-helm-wizard/src/resc_helm_wizard/helm_utilities.py

@@ -19,9 +19,9 @@ def install_or_upgrade_helm_release(action: str) -> bool:
         Returns true if install or upgrade succeeded else returns false
     """
     logging.info(f"Running {action}. Please wait for a moment...")
-    helm_command = ["helm", action, "-n", constants.NAMESPACE, constants.RELEASE_NAME, constants.CHART_NAME, "-f",
-                    constants.VALUES_FILE, "--set-file", "global.secretScanRulePackConfig=" + constants.RULE_FILE,
-                    "--repo", constants.RESC_HELM_REPO_URL]
+    helm_command = ["helm", action, "--timeout", constants.HELM_DEPLOY_TIMEOUT, "-n",
+                    constants.NAMESPACE, constants.RELEASE_NAME, constants.CHART_NAME, "-f",
+                    constants.VALUES_FILE, "--set-file", "global.secretScanRulePackConfig=" + constants.RULE_FILE]
     try:
         output = subprocess.check_output(helm_command)
         logging.info(output.decode("utf-8"))
@@ -31,28 +31,15 @@ def install_or_upgrade_helm_release(action: str) -> bool:
         return False
 
 
-def get_deployment_status_from_installed_chart() -> str:
+def check_helm_release_exists() -> bool:
     """
-        Get status of the installed chart
-    :return: str
-        Returns status of the installed chart
-    """
-    cmd = f"helm list -f {constants.CHART_NAME} -n {constants.NAMESPACE} -o json"
-    output = subprocess.check_output(cmd, shell=True)
-    chart_info = json.loads(output.decode("utf-8"))
-    if chart_info and "status" in chart_info[0]:
-        return chart_info[0]["status"]
-    return None
-
-
-def is_chart_already_installed() -> bool:
-    """
-        Checks if chart installed or not
+        Checks if helm release exists or not
     :return: bool
-        Returns true if chart already installed else returns false
+        Returns true if helm release exists else returns false
     """
-    status = get_deployment_status_from_installed_chart()
-    return bool(status == "deployed")
+    output = subprocess.run(["helm", "list", "-f", constants.RELEASE_NAME, "-n", constants.NAMESPACE],
+                            capture_output=True, text=True, check=True)
+    return bool(constants.RELEASE_NAME in output.stdout.strip())
 
 
 def get_version_from_downloaded_chart() -> str:
@@ -104,6 +91,10 @@ def validate_helm_deployment_status():
         output = result.stdout.strip()
         if "STATUS: deployed" in output:
             logging.info("The deployment was successful. Visit http://127.0.0.1:30000 to get started with RESC!")
+            logging.info("Refer this link for more information on how to trigger the scan: "
+                         "https://github.com/abnamro/repository-scanner/tree/main/"
+                         "deployment/kubernetes#trigger-scanning")
     except subprocess.CalledProcessError:
-        logging.error("An error occurred during deployment.")
+        logging.error("An error occurred during deployment. Please run this command to debug any issue: "
+                      "kubectl get pods -n resc")
         sys.exit(1)

deployment/resc-helm-wizard/src/resc_helm_wizard/questions.py

@@ -5,12 +5,15 @@
 import questionary
 
 # First Party
+from resc_helm_wizard import constants
 from resc_helm_wizard.validator import (
     azure_devops_token_validator,
     bitbucket_token_validator,
     github_account_name_validator,
     github_token_validator,
-    password_validator
+    github_username_validator,
+    password_validator,
+    vcs_url_validator
 )
 
 
@@ -86,19 +89,27 @@ def ask_vcs_instance_details(vcs_type: str) -> dict:
     """
     username = "NA"
     organization = ""
-    url = questionary.text(f"Please enter {vcs_type} url").unsafe_ask()
 
     if vcs_type == "GitHub":
-        username = questionary.text(f"What's your {vcs_type} username").unsafe_ask()
+        url = questionary.text(f"Please enter {vcs_type} url",
+                               default=constants.DEFAULT_GITHUB_URL,
+                               validate=vcs_url_validator).unsafe_ask()
+        username = questionary.text(f"What's your {vcs_type} username",
+                                    validate=github_username_validator).unsafe_ask()
         token = questionary.password(f"Please enter your {vcs_type} personal access token",
                                      validate=github_token_validator).unsafe_ask()
 
     if vcs_type == "Bitbucket":
+        url = questionary.text(f"Please enter {vcs_type} url",
+                               validate=vcs_url_validator).unsafe_ask()
         username = questionary.text(f"What's your {vcs_type} username").unsafe_ask()
         token = questionary.password(f"Please enter your {vcs_type} personal access token",
                                      validate=bitbucket_token_validator).unsafe_ask()
 
     if vcs_type == "Azure Devops":
+        url = questionary.text(f"Please enter {vcs_type} url",
+                               default=constants.DEFAULT_AZURE_DEVOPS_URL,
+                               validate=vcs_url_validator).unsafe_ask()
         organization = questionary.text(f"What's your organization name in {vcs_type}").unsafe_ask()
         token = questionary.password(f"Please enter your {vcs_type} personal access token",
                                      validate=azure_devops_token_validator).unsafe_ask()
@@ -116,3 +127,15 @@ def ask_which_github_accounts_to_scan(default_github_accounts: str) -> [str]:
                                        default=default_github_accounts,
                                        validate=github_account_name_validator).unsafe_ask()
     return github_accounts
+
+
+def ask_ssl_verification(msg: str) -> bool:
+    """
+        Asks for ssl verification
+    :param msg:
+        confirmation message
+    :return: bool
+        Returns True or False based on user's confirmation
+    """
+    answer = questionary.confirm(msg, default=True).unsafe_ask()
+    return answer

deployment/resc-helm-wizard/src/resc_helm_wizard/validator.py

@@ -82,8 +82,51 @@ def github_account_name_validator(github_accounts):
     for account in input_list:
         if not re.fullmatch(regex, account):
             if account:
-                msg = f"{account} is not a valid GitHub account"
+                msg = f"{account} is not a valid GitHub account. " \
+                      f"GitHub account must contain alphanumeric characters or single hyphens, " \
+                      f"can't begin or end with a hyphen and maximum 39 characters allowed."
             else:
                 msg = "Please enter a valid comma separated list of GitHub accounts you want to scan"
             return msg
     return True
+
+
+def github_username_validator(username):
+    """
+        GitHub username validator
+    :param username:
+        username of GitHub account
+    :return: str or bool.
+        If validation fails, the output will contain a validation error message.
+        Otherwise, the output will return true if validation was successful
+    """
+    regex = re.compile(r"^[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$")
+
+    if not re.fullmatch(regex, username):
+        msg = f"{username} is not a valid GitHub username. " \
+              f"GitHub username must contain alphanumeric characters or single hyphens, " \
+              f"can't begin or end with a hyphen and maximum 39 characters allowed."
+        return msg
+    return True
+
+
+def vcs_url_validator(url):
+    """
+        VCS provider url validator
+    :param url:
+        url which needs to be validated
+    :return: str or bool.
+        If validation fails, the output will contain a validation error message.
+        Otherwise, the output will return true if validation was successful
+    """
+    regex = re.compile(
+        r'^(?:http)s?://'  # Scheme
+        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # Domain
+        r'localhost|'  # Localhost
+        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # IP address
+        r'(?::\d+)?'  # Port (optional)
+        r'(?:/?|[/?]\S+)$', re.IGNORECASE)  # Path and query (optional)
+
+    if not re.fullmatch(regex, url):
+        return "Please provide a valid URL"
+    return True

deployment/resc-helm-wizard/tests/test_common.py

@@ -318,13 +318,15 @@ def test_create_helm_values_sys_exit_when_key_not_exists(mock_error_log, mock_re
     assert excinfo.value.code == 1
 
 
+@patch("resc_helm_wizard.questions.ask_ssl_verification")
 @patch("requests.get")
 @patch("logging.Logger.debug")
-def test_download_rule_toml_file_success(mock_debug_log, mock_get):
+def test_download_rule_toml_file_success(mock_debug_log, mock_get, mock_ask_ssl_verification_confirm):
     url = "https://example.com/rule_file.toml"
     file = "temp_file.toml"
     content = b'file content'
     expected_debug_log = f"{file} successfully downloaded"
+    mock_ask_ssl_verification_confirm.return_value = True
     mock_get.return_value.status_code = 200
     mock_get.return_value.content = content
     downloaded = download_rule_toml_file(url=url, file=file)
@@ -336,15 +338,18 @@ def test_download_rule_toml_file_success(mock_debug_log, mock_get):
         os.remove(file)
 
 
+@patch("resc_helm_wizard.questions.ask_ssl_verification")
 @patch("requests.get")
 @patch("os.path.exists")
 @patch("os.path.getsize")
 @patch("logging.Logger.error")
-def test_download_rule_toml_file_failure(mock_error_log, mock_os_path_getsize, mock_os_path_exists, mock_get):
+def test_download_rule_toml_file_failure(mock_error_log, mock_os_path_getsize, mock_os_path_exists, mock_get,
+                                         mock_ask_ssl_verification_confirm):
     url = "https://example.com/rule_file.toml"
     file = "temp_file.toml"
     content = b'file content'
     expected_error_log = "Unable to download the rule file"
+    mock_ask_ssl_verification_confirm.return_value = True
     mock_os_path_exists.return_value = False
     mock_os_path_getsize.return_value = -1
     mock_get.return_value.status_code = 500

deployment/resc-helm-wizard/tests/test_helm_utilities.py

@@ -8,10 +8,9 @@
 from resc_helm_wizard import constants
 from resc_helm_wizard.helm_utilities import (
     add_helm_repository,
-    get_deployment_status_from_installed_chart,
+    check_helm_release_exists,
     get_version_from_downloaded_chart,
     install_or_upgrade_helm_release,
-    is_chart_already_installed,
     update_helm_repository,
     validate_helm_deployment_status
 )
@@ -41,40 +40,6 @@ def test_install_or_upgrade_helm_release_failure(mock_error_log):
         mock_error_log.assert_called_with(expected_error_log)
 
 
-@patch("resc_helm_wizard.helm_utilities.get_deployment_status_from_installed_chart")
-def test_is_chart_already_installed_is_true(get_deployment_status_from_installed_chart):
-    get_deployment_status_from_installed_chart.return_value = "deployed"
-    already_installed = is_chart_already_installed()
-    get_deployment_status_from_installed_chart.assert_called_once_with()
-    assert already_installed is True
-
-
-@patch("resc_helm_wizard.helm_utilities.get_deployment_status_from_installed_chart")
-def test_is_chart_already_installed_is_false(get_deployment_status_from_installed_chart):
-    get_deployment_status_from_installed_chart.return_value = None
-    already_installed = is_chart_already_installed()
-    get_deployment_status_from_installed_chart.assert_called_once_with()
-    assert already_installed is False
-
-
-@patch("subprocess.check_output")
-def test_get_deployment_status_from_installed_chart_success(mock_check_output):
-    expected_output = b'[{"name":"resc","namespace":"resc","revision":"1",' \
-                      b'"updated":"2023-03-30 10:16:56.211749 +0200 CEST","status":"deployed",' \
-                      b'"chart":"resc-1.1.0","app_version":"1.1.0"}]\n'
-    mock_check_output.return_value = expected_output
-    actual_output = get_deployment_status_from_installed_chart()
-    assert actual_output == "deployed"
-
-
-@patch("subprocess.check_output")
-def test_get_deployment_status_from_installed_chart_failure(mock_check_output):
-    expected_output = b'{}'
-    mock_check_output.return_value = expected_output
-    actual_output = get_deployment_status_from_installed_chart()
-    assert actual_output is None
-
-
 @patch("subprocess.check_output")
 def test_get_version_from_downloaded_chart_success(mock_check_output):
     expected_output = b'[{"name":"resc-helm-repo/resc","version":"1.1.0","app_version":"1.1.0",' \
@@ -141,7 +106,8 @@ def test_update_helm_repository_failure(mock_error_log):
 def test_validate_helm_deployment_status_success(mock_info_log, mock_check_output):
     cmd = ['helm', 'status', constants.RELEASE_NAME, "-n", constants.NAMESPACE]
     expected_output = "RELEASE STATUS: deployed"
-    expected_info_log = "The deployment was successful. Visit http://127.0.0.1:30000 to get started with RESC!"
+    expected_info_log = "Refer this link for more information on how to trigger the scan: " \
+                        "https://github.com/abnamro/repository-scanner/tree/main/deployment/kubernetes#trigger-scanning"
     mock_check_output.return_value.stdout = expected_output
     validate_helm_deployment_status()
     assert mock_check_output.called
@@ -152,7 +118,8 @@ def test_validate_helm_deployment_status_success(mock_info_log, mock_check_outpu
 @patch("logging.Logger.error")
 def test_validate_helm_deployment_status_failure(mock_error_log):
     cmd = ['helm', 'status', constants.RELEASE_NAME, "-n", constants.NAMESPACE]
-    expected_error_log = "An error occurred during deployment."
+    expected_error_log = "An error occurred during deployment. Please run this command to debug any issue: " \
+                         "kubectl get pods -n resc"
     with mock.patch("subprocess.run") as mock_check_output, \
             mock.patch("sys.exit") as mock_sys_exit:
         mock_check_output.side_effect = subprocess.CalledProcessError(returncode=1, cmd=cmd)
@@ -161,3 +128,25 @@ def test_validate_helm_deployment_status_failure(mock_error_log):
         mock_error_log.assert_called_with(expected_error_log)
         mock_check_output.assert_called_once_with(cmd, capture_output=True, check=True, text=True)
         mock_sys_exit.assert_called_once_with(1)
+
+
+@patch("subprocess.run")
+def test_check_helm_release_exists_true(mock_check_output):
+    cmd = ["helm", "list", "-f", constants.RELEASE_NAME, "-n", constants.NAMESPACE]
+    expected_output = f"NAME: {constants.RELEASE_NAME}"
+    mock_check_output.return_value.stdout = expected_output
+    release_exists = check_helm_release_exists()
+    assert mock_check_output.called
+    mock_check_output.assert_called_once_with(cmd, capture_output=True, text=True, check=True)
+    assert release_exists is True
+
+
+@patch("subprocess.run")
+def test_check_helm_release_exists_false(mock_check_output):
+    cmd = ["helm", "list", "-f", constants.RELEASE_NAME, "-n", constants.NAMESPACE]
+    expected_output = "NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION"
+    mock_check_output.return_value.stdout = expected_output
+    release_exists = check_helm_release_exists()
+    assert mock_check_output.called
+    mock_check_output.assert_called_once_with(cmd, capture_output=True, text=True, check=True)
+    assert release_exists is False