Merge pull request #177 from abnamro/2672644-improve-pipeline-execution-time

[#2672644] Improve resc-backend pipeline execution time, updated test dependencies for resc-backend, resc-vcs-scraper, resc-vcs-scanner and resc-helm-wizard

Author: amrityamrout

.github/workflows/backend-ci.yaml

@@ -48,7 +48,6 @@ jobs:
       run: |
         cd ${{ env.RESC_BACKEND_DIR }}
         pip install -r test-requirements.txt
-        tox
 
     - name: Test with pytest
       run: |

.github/workflows/vcs-scanner-ci.yaml

@@ -49,7 +49,6 @@ jobs:
       run: |
         cd ${{ env.RESC_VCS_SCANNER_DIR }}
         pip install -r test-requirements.txt
-        tox
 
     - name: Test with pytest
       run: |

.github/workflows/vcs-scraper-ci.yaml

@@ -48,7 +48,6 @@ jobs:
       run: |
         cd ${{ env.RESC_VCS_SCRAPER_DIR }}
         pip install -r test-requirements.txt
-        tox
 
     - name: Test with pytest
       run: |

components/resc-backend/requirements.txt

@@ -15,6 +15,6 @@ tomlkit==0.12.1
 python-multipart==0.0.6
 aiofiles==0.8.0
 GitPython==3.1.32
-packaging==21.3
+packaging==23.1
 fastapi-cache2==0.2.1
 redis==4.6.0

components/resc-backend/src/resc_backend/bin/rabbitmq_bootup.py

@@ -55,5 +55,5 @@ def bootstrap_rabbitmq_users():
 
     server_up = wait_for_rabbitmq_server_to_up(rabbitmq_api_base_url=args.rabbitmq_url)
     if not server_up:
-        raise Exception("Wait for rabbitmq server to up has been failed.")
+        raise TimeoutError("Wait for rabbitmq server to up has been failed.")
     create_queue_user_and_set_permission(rabbitmq_api_base_url=args.rabbitmq_url)

components/resc-backend/src/resc_backend/helpers/rabbitmq/rabbitmq_initialization.py

@@ -70,7 +70,7 @@ def create_user(rabbitmq_api_base_url: str, username: str, password: str, role:
     uri = rabbitmq_api_base_url + "/api/users/" + username
     user_data = {"password": password, "tags": role}
     response = requests.put(uri, json=user_data,
-                            auth=HTTPBasicAuth(rabbitmq_admin_user, rabbitmq_admin_password))
+                            auth=HTTPBasicAuth(rabbitmq_admin_user, rabbitmq_admin_password), timeout=10)
 
     if hasattr(response, "status_code") and int(response.status_code) == 201:
         logger.info(f"User: {username} with role: {role} created successfully.")
@@ -110,7 +110,7 @@ def set_resource_permissions(rabbitmq_api_base_url: str, v_host: str, username:
                        "read": read_resources_regex}
 
     response = requests.put(uri, json=permission_data,
-                            auth=HTTPBasicAuth(rabbitmq_admin_user, rabbitmq_admin_password))
+                            auth=HTTPBasicAuth(rabbitmq_admin_user, rabbitmq_admin_password), timeout=10)
     if hasattr(response, "status_code") and int(response.status_code) == 201:
         logger.debug(f"vHost permission successfully assigned to user: {username} for vHost: {v_host}.")
         return True
@@ -155,7 +155,7 @@ def set_topic_permissions(rabbitmq_api_base_url: str, v_host: str, username: str
     topic_permission_data = {"exchange": topic_name, "write": write_permission, "read": read_permission}
 
     response = requests.put(uri, json=topic_permission_data,
-                            auth=HTTPBasicAuth(rabbitmq_admin_user, rabbitmq_admin_password))
+                            auth=HTTPBasicAuth(rabbitmq_admin_user, rabbitmq_admin_password), timeout=10)
 
     if hasattr(response, "status_code") and (int(response.status_code) == 201 or int(response.status_code) == 204):
         logger.debug(

components/resc-backend/src/resc_backend/resc_web_service/dependencies.py

@@ -172,8 +172,8 @@ def check_db_initialized():
                 not_found_tables.append(table_name)
 
         if len(not_found_tables) > 0:
-            raise Exception(f"Unable to determine existence of required table(s) "
-                            f"{', '.join(not_found_tables)}")
+            raise RuntimeError(f"Unable to determine existence of required table(s) "
+                               f"{', '.join(not_found_tables)}")
     except Exception as ex:
         logger.error(f"Database is NOT connected or initialized | {ex} | Retrying...")
         raise

components/resc-backend/src/resc_backend/resc_web_service/schema/detailed_finding.py

@@ -98,7 +98,7 @@ def build_commit_url(cls, values) -> Dict:
                                                                file_path=values["file_path"],
                                                                commit_id=values["commit_id"])
         else:
-            raise Exception(f"Unsupported VCSProvider: {values['vcs_provider']}")
+            raise NotImplementedError(f"Unsupported VCSProvider: {values['vcs_provider']}")
         return values
 
     class Config:

components/resc-backend/src/resc_backend/resc_web_service_interface/findings.py

@@ -20,7 +20,7 @@ def create_findings(url: str, findings: List[FindingCreate]) -> requests.Respons
     for finding in findings:
         findings_json.append(json.loads(finding.json()))
 
-    response = requests.post(api_url, json=findings_json, proxies={"http": "", "https": ""})
+    response = requests.post(api_url, json=findings_json, proxies={"http": "", "https": ""}, timeout=10)
     return response
 
 
@@ -31,5 +31,5 @@ def create_findings_with_scan_id(url: str, findings: List[FindingCreate], scan_i
     for finding in findings:
         findings_json.append(json.loads(finding.json()))
 
-    response = requests.post(api_url, json=findings_json, proxies={"http": "", "https": ""})
+    response = requests.post(api_url, json=findings_json, proxies={"http": "", "https": ""}, timeout=10)
     return response

components/resc-backend/src/resc_backend/resc_web_service_interface/repositories.py

@@ -13,11 +13,11 @@
 
 def create_repository(url: str, repository: Repository):
     api_url = f"{url}{RWS_VERSION_PREFIX}{RWS_ROUTE_REPOSITORIES}"
-    response = requests.post(api_url, data=repository.json(), proxies={"http": "", "https": ""})
+    response = requests.post(api_url, data=repository.json(), proxies={"http": "", "https": ""}, timeout=10)
     return response
 
 
 def get_last_scan_for_repository(url: str, repository_id: int):
     api_url = f"{url}{RWS_VERSION_PREFIX}{RWS_ROUTE_REPOSITORIES}/{repository_id}{RWS_ROUTE_LAST_SCAN}"
-    response = requests.get(api_url, proxies={"http": "", "https": ""})
+    response = requests.get(api_url, proxies={"http": "", "https": ""}, timeout=10)
     return response

components/resc-backend/src/resc_backend/resc_web_service_interface/rule_packs.py

@@ -15,7 +15,7 @@
 def upload_rule_pack_toml_file(url: str, rule_file_path: str):
     with open(rule_file_path, "rb") as toml_file:
         files = {"rule_file": ("RESC-SECRETS-RULE.toml", toml_file, "application/octet-stream")}
-        response = requests.post(url=url, files=files, proxies={"http": "", "https": ""})
+        response = requests.post(url=url, files=files, proxies={"http": "", "https": ""}, timeout=10)
         toml_file.close()
     return response
 
@@ -25,7 +25,7 @@ def download_rule_pack_toml_file(rws_url: str, rule_pack_version: Optional[str]
     if rule_pack_version:
         params = {"rule_pack_version": rule_pack_version}
     response = requests.get(url=f"{rws_url}{RWS_VERSION_PREFIX}{RWS_ROUTE_RULE_PACKS}",
-                            params=params)
+                            params=params, timeout=10)
 
     if response.status_code == 200:
         logger.debug(
@@ -42,5 +42,5 @@ def get_rule_packs(url: str, version: Optional[str] = None, active: Optional[boo
     params = {"active": active, "skip": skip, "limit": limit}
     if version:
         params["version"] = version
-    response = requests.get(api_url, params=params, proxies={"http": "", "https": ""})
+    response = requests.get(api_url, params=params, proxies={"http": "", "https": ""}, timeout=10)
     return response

components/resc-backend/src/resc_backend/resc_web_service_interface/scans.py

@@ -13,5 +13,5 @@
 
 def create_scan(url: str, scan: ScanCreate):
     api_url = f"{url}{RWS_VERSION_PREFIX}{RWS_ROUTE_SCANS}"
-    response = requests.post(api_url, data=scan.json(), proxies={"http": "", "https": ""})
+    response = requests.post(api_url, data=scan.json(), proxies={"http": "", "https": ""}, timeout=10)
     return response

components/resc-backend/src/resc_backend/resc_web_service_interface/vcs_instances.py

@@ -13,5 +13,5 @@
 
 def create_vcs_instance(url: str, vcs_instance: VCSInstanceCreate):
     api_url = f"{url}{RWS_VERSION_PREFIX}{RWS_ROUTE_VCS}"
-    response = requests.post(api_url, data=vcs_instance.json(), proxies={"http": "", "https": ""})
+    response = requests.post(api_url, data=vcs_instance.json(), proxies={"http": "", "https": ""}, timeout=10)
     return response

components/resc-backend/test-requirements.txt

@@ -1,14 +1,14 @@
 flake8==4.0.1
 flake8-blind-except==0.2.1
-flake8-builtins==1.5.3
+flake8-builtins==2.1.0
 flake8-polyfill==1.0.2
 flake8-tabs==2.2.2
-flake8-variables-names==0.0.5
-isort==5.10.1
-pytest==7.1.2
-mock==4.0.2
-pytest-cov==2.10.0
-tox==3.23.1
-pylint==2.14.0
+flake8-variables-names==0.0.6
+isort==5.12.0
+pytest==7.4.1
+mock==5.1.0
+pytest-cov==4.1.0
+tox==4.11.1
+pylint==2.17.5
 pytest-asyncio==0.21.1
 httpx==0.24.1

components/resc-backend/tests/resc_backend/resc_web_service_interface/test_client_findings.py

@@ -38,4 +38,4 @@ def test_create_findings(post):
 
     _ = create_findings(url, findings)
     post.assert_called_once()
-    post.assert_called_with(expected_url, json=findings_json, proxies={'http': '', 'https': ''})
+    post.assert_called_with(expected_url, json=findings_json, proxies={'http': '', 'https': ''}, timeout=10)

components/resc-backend/tests/resc_backend/resc_web_service_interface/test_client_repositories.py

@@ -20,4 +20,4 @@ def test_create_repository(post):
 
     _ = create_repository(url, repository)
     post.assert_called_once()
-    post.assert_called_with(expected_url, data=expected_json, proxies={'http': '', 'https': ''})
+    post.assert_called_with(expected_url, data=expected_json, proxies={'http': '', 'https': ''}, timeout=10)

components/resc-backend/tests/resc_backend/resc_web_service_interface/test_client_rule_packs.py

@@ -13,7 +13,7 @@ def test_get_rule_packs(get):
 
     _ = get_rule_packs(url)
     get.assert_called_once()
-    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''})
+    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''}, timeout=10)
 
 
 @patch("requests.get")
@@ -24,7 +24,7 @@ def test_get_rule_packs_when_version_provided(get):
 
     _ = get_rule_packs(url=url, version="1.0.0")
     get.assert_called_once()
-    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''})
+    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''}, timeout=10)
 
 
 @patch("requests.get")
@@ -35,7 +35,7 @@ def test_get_rule_packs_when_active_provided(get):
 
     _ = get_rule_packs(url=url, active=True)
     get.assert_called_once()
-    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''})
+    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''}, timeout=10)
 
 
 @patch("requests.get")
@@ -46,4 +46,4 @@ def test_get_rule_packs_when_version_and_active_provided(get):
 
     _ = get_rule_packs(url=url, version="1.0.0", active=False)
     get.assert_called_once()
-    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''})
+    get.assert_called_with(expected_url, params=expected_params, proxies={'http': '', 'https': ''}, timeout=10)

components/resc-backend/tests/resc_backend/resc_web_service_interface/test_client_scans.py

@@ -18,4 +18,4 @@ def test_create_scan(post):
 
     _ = create_scan(url, scan)
     post.assert_called_once()
-    post.assert_called_with(expected_url, data=expected_json, proxies={'http': '', 'https': ''})
+    post.assert_called_with(expected_url, data=expected_json, proxies={'http': '', 'https': ''}, timeout=10)

components/resc-backend/tests/resc_backend/resc_web_service_interface/test_client_vcs_instances.py

@@ -19,4 +19,4 @@ def test_create_vcs_instance(post):
 
     _ = create_vcs_instance(url, vcs_instance)
     post.assert_called_once()
-    post.assert_called_with(expected_url, data=expected_json, proxies={'http': '', 'https': ''})
+    post.assert_called_with(expected_url, data=expected_json, proxies={'http': '', 'https': ''}, timeout=10)

components/resc-vcs-scanner/test-requirements.txt

@@ -1,12 +1,12 @@
 flake8==4.0.1
 flake8-blind-except==0.2.1
-flake8-builtins==1.5.3
+flake8-builtins==2.1.0
 flake8-polyfill==1.0.2
 flake8-tabs==2.2.2
-flake8-variables-names==0.0.5
-isort==5.10.1
-pytest==7.1.2
-mock==4.0.2
-pytest-cov==2.10.0
-tox==3.23.1
-pylint==2.14.0
+flake8-variables-names==0.0.6
+isort==5.12.0
+pytest==7.4.1
+mock==5.1.0
+pytest-cov==4.1.0
+tox==4.11.1
+pylint==2.17.5

components/resc-vcs-scraper/src/vcs_scraper/project_collector/common.py

@@ -2,6 +2,7 @@
 import logging
 
 # Third Party
+from celery.exceptions import CeleryError
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_exponential
 
 # First Party
@@ -67,7 +68,7 @@ def collect_projects_from_vcs_instance(vcs_instance: VCSInstance) -> None:
                         celery_client.send_task(task_name, kwargs={"project_key": project_name,
                                                                    "vcs_instance_name": vcs_instance.name},
                                                 queue=destination_message_queue)
-                    except Exception as exc:
+                    except CeleryError as exc:
                         logger.error(f"{exc} sending '{project_name}' to 'projects'. Retrying ...")
                         raise
             projects_sent += 1

components/resc-vcs-scraper/test-requirements.txt

@@ -1,12 +1,12 @@
-flake8==3.9.2
-flake8-blind-except==0.1.1
-flake8-builtins==1.5.3
+flake8==4.0.1
+flake8-blind-except==0.2.1
+flake8-builtins==2.1.0
 flake8-polyfill==1.0.2
 flake8-tabs==2.2.2
-flake8-variables-names==0.0.3
-isort==4.3.21
-pytest==7.1.2
-mock==4.0.2
-pytest-cov==2.10.0
-tox==3.23.1
-pylint==2.14.5
+flake8-variables-names==0.0.6
+isort==5.12.0
+pytest==7.4.1
+mock==5.1.0
+pytest-cov==4.1.0
+tox==4.11.1
+pylint==2.17.5

deployment/resc-helm-wizard/README.md

@@ -85,10 +85,10 @@ See below commands for running various (unit/linting) tests locally. To run thes
 
 Run below commands to make sure that the unit tests are running and that the code matches quality standards:
 ```bash
-pip install tox      # install tox locally
+pip install tox         # install tox locally
 
-tox -v -e sort       # Run this command to validate the import sorting
-tox -v -e lint       # Run this command to lint the code according to this repository's standard
-tox -v -e pytest     # Run this command to run the unit tests
-tox -v               # Run this command to run all of the above tests
+tox run -e py -v        # Run this command to run the unit tests
+tox run -e isort -v     # Run this command to validate the import sorting
+tox run -e pylint -v    # Run this command for Python static code analysis
+tox run -e flake8 -v    # Run this command for Python linting
 ```

deployment/resc-helm-wizard/test-requirements.txt

@@ -1,6 +1,6 @@
-flake8==6.0.0
+flake8==6.1.0
 isort==5.12.0
-pytest==7.2.1
-pytest-cov==4.0.0
-tox==3.23.1
-pylint==2.16.2
+pytest==7.4.1
+pytest-cov==4.1.0
+tox==4.11.1
+pylint==2.17.5