[#2634148] Update SQLAlchemy and FastAPI

Author: Peter-v-d-Spek

components/resc-backend/requirements.txt

@@ -1,9 +1,9 @@
 celery==5.3.1
 amqp==5.1.1
 requests==2.31.0
-sqlalchemy==1.4.37
+sqlalchemy==2.0.20
 alembic==1.11.2
-fastapi==0.66.1
+fastapi==0.103.0
 typing==3.7.4.3
 pydantic==1.8.2
 uvicorn==0.17.6

components/resc-backend/src/resc_backend/db/model/__init__.py

@@ -4,7 +4,7 @@
 import os
 
 # Third Party
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import declarative_base
 
 Base = declarative_base()
 basedir = os.path.abspath(os.path.dirname(__file__))

components/resc-backend/src/resc_backend/resc_web_service/crud/audit.py

@@ -1,4 +1,4 @@
-# pylint: disable=R0916,R0912,C0121
+# pylint: disable=R0916,R0912,C0121,not-callable
 # Standard Library
 import logging
 from datetime import datetime, timedelta

components/resc-backend/src/resc_backend/resc_web_service/crud/detailed_finding.py

@@ -1,4 +1,4 @@
-# pylint: disable=R0912,C0121,R0915
+# pylint: disable=R0912,C0121,R0915,not-callable
 # Standard Library
 from typing import List
 

components/resc-backend/src/resc_backend/resc_web_service/crud/finding.py

@@ -1,4 +1,4 @@
-# pylint: disable=R0916,R0912,C0121
+# pylint: disable=R0916,R0912,C0121,not-callable
 # Standard Library
 import logging
 from datetime import datetime, timedelta

components/resc-backend/src/resc_backend/resc_web_service/crud/repository.py

@@ -1,3 +1,4 @@
+# pylint:disable=not-callable
 # Third Party
 from sqlalchemy import and_, func
 from sqlalchemy.orm import Session

components/resc-backend/src/resc_backend/resc_web_service/crud/rule.py

@@ -1,3 +1,4 @@
+# pylint:disable=not-callable
 # Standard Library
 import logging
 from typing import List

components/resc-backend/src/resc_backend/resc_web_service/crud/rule_pack.py

@@ -1,3 +1,4 @@
+# pylint:disable=not-callable
 # Standard Library
 import logging
 from typing import List, Optional

components/resc-backend/src/resc_backend/resc_web_service/crud/rule_tag.py

@@ -1,4 +1,4 @@
-# pylint: disable=E1101
+# pylint: disable=E1101,not-callable
 # Standard Library
 from typing import List
 

components/resc-backend/src/resc_backend/resc_web_service/crud/scan.py

@@ -1,4 +1,4 @@
-# pylint: disable=E1101
+# pylint: disable=E1101,not-callable
 # Standard Library
 from datetime import datetime
 from typing import List

components/resc-backend/src/resc_backend/resc_web_service/crud/scan_finding.py

@@ -1,3 +1,4 @@
+# pylint:disable=not-callable
 # Standard Library
 from typing import List
 

components/resc-backend/src/resc_backend/resc_web_service/crud/vcs_instance.py

@@ -1,4 +1,4 @@
-# pylint: disable=E1101
+# pylint: disable=E1101,not-callable
 # Standard Library
 from typing import List
 

components/resc-backend/src/resc_backend/resc_web_service/endpoints/findings.py

@@ -184,7 +184,7 @@ async def patch_finding(
                    500: {"description": ERROR_MESSAGE_500},
                    503: {"description": ERROR_MESSAGE_503}
                })
-async def delete_finding(finding_id: int, db_connection: Session = Depends(get_db_connection)) -> FindingRead:
+async def delete_finding(finding_id: int, db_connection: Session = Depends(get_db_connection)):
     """
         Delete a finding object
 

components/resc-backend/src/resc_backend/resc_web_service/endpoints/metrics.py

@@ -140,10 +140,10 @@ def convert_rows_to_finding_count_over_time(count_over_time: dict, weeks: int) -
 
     # loop over the counts from the database
     for data in count_over_time:
-        week = f"{data['year']} W{data['week']:02d}"
-        finding_count = data["finding_count"]
+        week = f"{getattr(data, 'year')} W{getattr(data, 'week'):02d}"
+        finding_count = getattr(data, 'finding_count')
 
-        week_groups[week][data["provider_type"]] += finding_count
+        week_groups[week][getattr(data, 'provider_type')] += finding_count
         week_groups[week]["total"] += finding_count
 
     # Convert to the output format
@@ -182,7 +182,7 @@ def get_audit_count_by_auditor_over_time(db_connection: Session = Depends(get_db
     # get the unique auditors from the data
     auditors_default = {}
     for audit in audit_counts:
-        auditors_default[audit['auditor']] = 0
+        auditors_default[getattr(audit, 'auditor')] = 0
 
     # default to 0 per auditor for all weeks in range
     weekly_audit_counts = {}
@@ -194,10 +194,11 @@ def get_audit_count_by_auditor_over_time(db_connection: Session = Depends(get_db
 
     # set the counts based on the data from the database
     for audit in audit_counts:
-        audit_week = f"{audit['year']} W{audit['week']:02d}"
+        audit_week = f"{getattr(audit, 'year')} W{getattr(audit, 'week'):02d}"
         if audit_week in weekly_audit_counts:
-            weekly_audit_counts.get(audit_week).audit_by_auditor_count[audit['auditor']] = audit['audit_count']
-            weekly_audit_counts.get(audit_week).total += audit['audit_count']
+            weekly_audit_counts.get(audit_week).audit_by_auditor_count[getattr(audit, 'auditor')] = \
+                getattr(audit, 'audit_count')
+            weekly_audit_counts.get(audit_week).total += getattr(audit, 'audit_count')
 
     sorted_weekly_audit_counts = dict(sorted(weekly_audit_counts.items()))
     output = list(sorted_weekly_audit_counts.values())
@@ -254,7 +255,7 @@ def determine_audit_rank_current_week(auditor: str, db_connection: Session) -> i
 
     auditor_counts = {}
     for audit in audit_counts_db:
-        auditor_counts[audit['auditor']] = audit['audit_count']
+        auditor_counts[getattr(audit, 'auditor')] = getattr(audit, 'audit_count')
 
     sorted_auditor_counts = sorted(auditor_counts.items(), key=lambda x: x[1], reverse=True)
     for auditor_count in dict(sorted_auditor_counts):

components/resc-backend/src/resc_backend/resc_web_service/endpoints/repositories.py

@@ -49,8 +49,8 @@
 def get_all_repositories(skip: int = Query(default=0, ge=0),
                          limit: int = Query(default=DEFAULT_RECORDS_PER_PAGE_LIMIT, ge=1),
                          vcsproviders: List[VCSProviders] = Query(None, alias="vcsprovider", title="VCSProviders"),
-                         projectfilter: Optional[str] = Query('', regex=r"^[A-z0-9 .\-_%]*$"),
-                         repositoryfilter: Optional[str] = Query('', regex=r"^[A-z0-9 .\-_%]*$"),
+                         projectfilter: Optional[str] = Query('', pattern=r"^[A-z0-9 .\-_%]*$"),
+                         repositoryfilter: Optional[str] = Query('', pattern=r"^[A-z0-9 .\-_%]*$"),
                          db_connection: Session = Depends(get_db_connection)) \
         -> PaginationModel[repository_schema.RepositoryRead]:
     """
@@ -208,7 +208,7 @@ async def delete_repository(repository_id: int, db_connection: Session = Depends
             })
 @cache(namespace=CACHE_NAMESPACE_REPOSITORY, expire=REDIS_CACHE_EXPIRE)
 def get_distinct_projects(vcsproviders: List[VCSProviders] = Query(None, alias="vcsprovider", title="VCSProviders"),
-                          repositoryfilter: Optional[str] = Query('', regex=r"^[A-z0-9 .\-_%]*$"),
+                          repositoryfilter: Optional[str] = Query('', pattern=r"^[A-z0-9 .\-_%]*$"),
                           onlyifhasfindings: bool = Query(default=False),
                           db_connection: Session = Depends(get_db_connection)) -> List[str]:
     """
@@ -241,7 +241,7 @@ def get_distinct_projects(vcsproviders: List[VCSProviders] = Query(None, alias="
             })
 @cache(namespace=CACHE_NAMESPACE_REPOSITORY, expire=REDIS_CACHE_EXPIRE)
 def get_distinct_repositories(vcsproviders: List[VCSProviders] = Query(None, alias="vcsprovider", title="VCSProviders"),
-                              projectname: Optional[str] = Query('', regex=r"^[A-z0-9 .\-_%]*$"),
+                              projectname: Optional[str] = Query('', pattern=r"^[A-z0-9 .\-_%]*$"),
                               onlyifhasfindings: bool = Query(default=False),
                               db_connection: Session = Depends(get_db_connection)) -> List[str]:
     """
@@ -318,9 +318,9 @@ def get_all_repositories_with_findings_metadata(
         vcsproviders: List[VCSProviders] = Query(None, alias="vcsprovider",
                                                  title="VCSProviders"),
         projectfilter: Optional[str] = Query('',
-                                             regex=r"^[A-z0-9 .\-_%]*$"),
+                                             pattern=r"^[A-z0-9 .\-_%]*$"),
         repositoryfilter: Optional[str] = Query('',
-                                                regex=r"^[A-z0-9 .\-_%]*$"),
+                                                pattern=r"^[A-z0-9 .\-_%]*$"),
         onlyifhasfindings: bool = Query(default=False),
         db_connection: Session = Depends(get_db_connection)) \
         -> PaginationModel[repository_enriched_schema.RepositoryEnrichedRead]:
@@ -383,6 +383,7 @@ def get_all_repositories_with_findings_metadata(
             status_code=status.HTTP_200_OK,
             responses={
                 200: {"description": "Retrieve the latest scan related to a repository"},
+                404: {"description": "Scan not found"},
                 500: {"description": ERROR_MESSAGE_500},
                 503: {"description": ERROR_MESSAGE_503}
             })
@@ -398,7 +399,8 @@ def get_last_scan_for_repository(repository_id: int, db_connection: Session = De
         or empty if no scan was found
     """
     last_scan = scan_crud.get_latest_scan_for_repository(db_connection, repository_id=repository_id)
-
+    if last_scan is None:
+        raise HTTPException(status_code=404, detail="Scan not found")
     return last_scan
 
 

components/resc-backend/src/resc_backend/resc_web_service/endpoints/rule_packs.py

@@ -57,7 +57,7 @@
                 503: {"description": ERROR_MESSAGE_503}
             })
 @cache(namespace=CACHE_NAMESPACE_RULE_PACK, expire=REDIS_CACHE_EXPIRE)
-def get_rule_packs(version: Optional[str] = Query(None, regex=r"^\d+(?:\.\d+){2}$"),
+def get_rule_packs(version: Optional[str] = Query(None, pattern=r"^\d+(?:\.\d+){2}$"),
                    active: Optional[bool] = Query(None, description="Filter on active rule packs"),
                    skip: int = Query(default=0, ge=0),
                    limit: int = Query(default=DEFAULT_RECORDS_PER_PAGE_LIMIT, ge=1),
@@ -91,7 +91,7 @@ def get_rule_packs(version: Optional[str] = Query(None, regex=r"^\d+(?:\.\d+){2}
                 500: {"description": ERROR_MESSAGE_500},
                 503: {"description": ERROR_MESSAGE_503}
             })
-async def download_rule_pack_toml_file(version: Optional[str] = Query(None, regex=r"^\d+(?:\.\d+){2}$"),
+async def download_rule_pack_toml_file(version: Optional[str] = Query(None, pattern=r"^\d+(?:\.\d+){2}$"),
                                        db_connection: Session = Depends(get_db_connection)) -> FileResponse:
     """
         Download rule pack in TOML format
@@ -129,7 +129,7 @@ async def download_rule_pack_toml_file(version: Optional[str] = Query(None, rege
                  500: {"description": ERROR_MESSAGE_500},
                  503: {"description": ERROR_MESSAGE_503}
              })
-async def upload_rule_pack_toml_file(version: str = Query(default=Required, regex=r"^\d+(?:\.\d+){2}$"),
+async def upload_rule_pack_toml_file(version: str = Query(default=Required, pattern=r"^\d+(?:\.\d+){2}$"),
                                      rule_file: UploadFile = File(...),
                                      db_connection: Session = Depends(get_db_connection)) -> dict:
     """

components/resc-backend/src/resc_backend/resc_web_service/endpoints/rules.py

@@ -45,8 +45,8 @@
 def get_distinct_rules_from_findings(
         finding_statuses: List[FindingStatus] = Query(None, alias="findingstatus", title="FindingStatuses"),
         vcs_providers: List[VCSProviders] = Query(None, alias="vcsprovider", title="VCSProviders"),
-        project_name: Optional[str] = Query('', regex=r"^[A-z0-9 .\-_%]*$"),
-        repository_name: Optional[str] = Query('', regex=r"^[A-z0-9 .\-_%]*$"),
+        project_name: Optional[str] = Query('', pattern=r"^[A-z0-9 .\-_%]*$"),
+        repository_name: Optional[str] = Query('', pattern=r"^[A-z0-9 .\-_%]*$"),
         start_date_time: Optional[datetime] = Query(None),
         end_date_time: Optional[datetime] = Query(None),
         rule_pack_versions: Optional[List[str]] = Query(None, alias="rule_pack_version", title="RulePackVersion"),

components/resc-backend/src/resc_backend/resc_web_service/endpoints/scans.py

@@ -246,7 +246,7 @@ async def create_scan_findings(scan_id: int,
             })
 def get_scan_findings(scan_id: int, skip: int = Query(default=0, ge=0),
                       limit: int = Query(default=DEFAULT_RECORDS_PER_PAGE_LIMIT, ge=1),
-                      rules: List[str] = Query([], regex=r"^[A-z0-9 .\-_%]*$", alias="rule", title="rule"),
+                      rules: List[str] = Query([], pattern=r"^[A-z0-9 .\-_%]*$", alias="rule", title="rule"),
                       statuses: List[FindingStatus] = Query(None, alias="status", title="status"),
                       db_connection: Session = Depends(get_db_connection)) \
         -> PaginationModel[finding_schema.FindingRead]:
@@ -284,7 +284,7 @@ def get_scan_findings(scan_id: int, skip: int = Query(default=0, ge=0),
 def get_scans_findings(scan_ids: List[int] = Query([], alias="scan_id", title="Scan ids"),
                        skip: int = Query(default=0, ge=0),
                        limit: int = Query(default=DEFAULT_RECORDS_PER_PAGE_LIMIT, ge=1),
-                       rules: List[str] = Query([], regex=r"^[A-z0-9 .\-_%]*$", alias="rule", title="rule"),
+                       rules: List[str] = Query([], pattern=r"^[A-z0-9 .\-_%]*$", alias="rule", title="rule"),
                        statuses: List[FindingStatus] = Query(None, alias="status", title="status"),
                        db_connection: Session = Depends(get_db_connection)) \
         -> PaginationModel[finding_schema.FindingRead]:

components/resc-backend/test-requirements.txt

@@ -11,3 +11,4 @@ pytest-cov==2.10.0
 tox==3.23.1
 pylint==2.14.0
 pytest-asyncio==0.21.1
+httpx==0.24.1