Merge pull request #107 from abnamro/2225563-metrics-for-number-of-findings

[#2225563] Created multiline charts for total findings count, true po…

Author: Peter-v-d-Spek

components/resc-backend/src/resc_backend/constants.py

@@ -13,6 +13,7 @@
 RWS_ROUTE_LAST_SCAN = "/last-scan"
 RWS_ROUTE_FINDINGS = "/findings"
 RWS_ROUTE_RULES = "/rules"
+RWS_ROUTE_METRICS = "/metrics"
 RWS_ROUTE_DETAILED_FINDINGS = "/detailed-findings"
 RWS_ROUTE_TOTAL_COUNT_BY_RULE = "/total-count-by-rule"
 RWS_ROUTE_BY_RULE = "/by-rule"
@@ -22,7 +23,10 @@
 RWS_ROUTE_RULE_PACKS = "/rule-packs"
 RWS_ROUTE_VCS = "/vcs-instances"
 
+RWS_ROUTE_AUDITED_COUNT_OVER_TIME = "/audited-count-over-time"
+RWS_ROUTE_UN_TRIAGED_COUNT_OVER_TIME = "/un-triaged-count-over-time"
 RWS_ROUTE_COUNT_BY_TIME = "/count-by-time"
+RWS_ROUTE_COUNT_PER_VCS_PROVIDER_BY_WEEK = "/count-per-vcs-provider-by-week"
 RWS_ROUTE_SUPPORTED_VCS_PROVIDERS = "/supported-vcs-providers"
 RWS_ROUTE_SUPPORTED_STATUSES = "/supported-statuses"
 RWS_ROUTE_DISTINCT_PROJECTS = "/distinct-projects"
@@ -43,6 +47,7 @@
 RULE_PACKS_TAG = "resc-rule-packs"
 HEALTH_TAG = "health"
 VCS_TAG = "resc-vcs-instances"
+METRICS_TAG = "resc-metrics"
 
 DEFAULT_RECORDS_PER_PAGE_LIMIT = 100
 MAX_RECORDS_PER_PAGE_LIMIT = 500

components/resc-backend/src/resc_backend/resc_web_service/api.py

@@ -25,6 +25,7 @@
     detailed_findings,
     findings,
     health,
+    metrics,
     repositories,
     rules,
     rule_packs,
@@ -95,6 +96,7 @@ def generate_logger_config(log_file_path, debug=True):
     {"name": "resc-scans", "description": "Manage scan information"},
     {"name": "resc-findings", "description": "Manage findings information"},
     {"name": "resc-vcs-instances", "description": "Manage vcs instance information"},
+    {"name": "resc-metrics", "description": "Retrieve metrics"},
 ]
 
 # Check if authentication is required for api endpoints
@@ -126,6 +128,7 @@ def generate_logger_config(log_file_path, debug=True):
 app.include_router(repositories.router, prefix=RWS_VERSION_PREFIX)
 app.include_router(scans.router, prefix=RWS_VERSION_PREFIX)
 app.include_router(vcs_instances.router, prefix=RWS_VERSION_PREFIX)
+app.include_router(metrics.router, prefix=RWS_VERSION_PREFIX)
 
 # Add exception handlers
 add_exception_handlers(app=app)

components/resc-backend/src/resc_backend/resc_web_service/crud/finding.py

@@ -1,11 +1,12 @@
 # pylint: disable=R0916,R0912,C0121
 # Standard Library
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import List
 
 # Third Party
-from sqlalchemy import and_, extract, func, or_
+from sqlalchemy import and_, extract, func, or_, union
+from sqlalchemy.engine import Row
 from sqlalchemy.orm import Session
 
 # First Party
@@ -450,6 +451,7 @@ def get_rule_findings_count_by_status(db_connection: Session, rule_pack_versions
             "clarification_required": 0,
             "total_findings_count": 0
         }
+
     for status_count in status_counts:
         rule_count_dict[status_count[0]]["total_findings_count"] += status_count[2]
         if status_count[1] == FindingStatus.NOT_ANALYZED or status_count[1] is None:
@@ -640,3 +642,160 @@ def delete_findings_by_vcs_instance_id(db_connection: Session, vcs_instance_id:
                 model.vcs_instance.DBVcsInstance.id_ == vcs_instance_id) \
         .delete(synchronize_session=False)
     db_connection.commit()
+
+
+def get_finding_audit_status_count_over_time(db_connection: Session, status: FindingStatus, weeks: int = 13) -> dict:
+    """
+        Retrieve count of true positive findings over time for given weeks
+    :param db_connection:
+        Session of the database connection
+    :param status:
+        mandatory, status for which to get the audit counts over time
+    :param weeks:
+        optional, filter on last n weeks, default 13
+    :return: true_positive_count_over_time
+        list of rows containing finding statuses count over time per week
+    """
+    all_tables = []
+    for week in range(0, weeks):
+        last_nth_week_date_time = datetime.utcnow() - timedelta(weeks=week)
+        query = db_connection.query(extract('year', last_nth_week_date_time).label("year"),
+                                    extract('week', last_nth_week_date_time).label("week"),
+                                    model.DBVcsInstance.provider_type.label("provider_type"),
+                                    func.count(model.DBaudit.id_).label("finding_count")
+                                    )
+        max_audit_subquery = db_connection.query(func.max(model.DBaudit.id_).label("audit_id")) \
+            .filter(extract('year', model.DBaudit.timestamp) == extract('year', last_nth_week_date_time)) \
+            .filter(extract('week', model.DBaudit.timestamp) <= extract('week', last_nth_week_date_time)) \
+            .group_by(model.DBaudit.finding_id).subquery()
+        query = query.join(max_audit_subquery, max_audit_subquery.c.audit_id == model.DBaudit.id_)
+        query = query.join(model.DBfinding, model.DBfinding.id_ == model.DBaudit.finding_id)
+        query = query.join(model.DBbranch, model.DBbranch.id_ == model.DBfinding.branch_id)
+        query = query.join(model.DBrepository, model.DBrepository.id_ == model.DBbranch.repository_id)
+        query = query.join(model.DBVcsInstance, model.DBVcsInstance.id_ == model.DBrepository.vcs_instance)
+        query = query.filter(model.DBaudit.status == status)
+        query = query.group_by(model.DBVcsInstance.provider_type)
+
+        all_tables.append(query)
+
+    # union
+    unioned_query = union(*all_tables)
+    status_count_over_time = db_connection.execute(unioned_query).all()
+    return status_count_over_time
+
+
+def get_finding_count_by_vcs_provider_over_time(db_connection: Session, weeks: int = 13) -> list[Row]:
+    """
+        Retrieve count findings over time for given weeks
+    :param db_connection:
+        Session of the database connection
+    :param weeks:
+        optional, filter on last n weeks, default 13
+    :return: count_over_time
+        list of rows containing finding count over time per week
+    """
+    all_tables = []
+    for week in range(0, weeks):
+        last_nth_week_date_time = datetime.utcnow() - timedelta(weeks=week)
+        query = db_connection.query(extract('year', last_nth_week_date_time).label("year"),
+                                    extract('week', last_nth_week_date_time).label("week"),
+                                    model.DBVcsInstance.provider_type.label("provider_type"),
+                                    func.count(model.DBfinding.id_).label("finding_count")
+                                    )
+        max_base_scan = db_connection.query(func.max(model.DBscan.id_).label("scan_id"),
+                                            model.DBscan.branch_id) \
+            .filter(extract('year', model.DBscan.timestamp) == extract('year', last_nth_week_date_time)) \
+            .filter(extract('week', model.DBscan.timestamp) <= extract('week', last_nth_week_date_time)) \
+            .filter(model.DBscan.scan_type == ScanType.BASE) \
+            .group_by(model.DBscan.branch_id).subquery()
+
+        query = query.join(model.DBscanFinding, model.DBfinding.id_ == model.DBscanFinding.finding_id)
+        query = query.join(model.DBscan, model.DBscan.id_ == model.DBscanFinding.scan_id)
+        query = query.join(max_base_scan, and_(max_base_scan.c.branch_id == model.DBscan.branch_id,
+                                               or_(model.DBscan.id_ == max_base_scan.c.scan_id,
+                                                   (and_(model.DBscan.id_ > max_base_scan.c.scan_id,
+                                                         model.DBscan.scan_type == ScanType.INCREMENTAL,
+                                                         extract('week', model.DBscan.timestamp) <=
+                                                         extract('week', last_nth_week_date_time),
+                                                         extract('year', model.DBscan.timestamp) ==
+                                                         extract('year', last_nth_week_date_time)))
+                                                   )
+                                               )
+                           )
+        query = query.join(model.DBbranch, model.DBbranch.id_ == model.DBscan.branch_id)
+        query = query.join(model.DBrepository, model.DBrepository.id_ == model.DBbranch.repository_id)
+        query = query.join(model.DBVcsInstance, model.DBVcsInstance.id_ == model.DBrepository.vcs_instance)
+        query = query.group_by(model.DBVcsInstance.provider_type)
+
+        all_tables.append(query)
+
+    # union
+    unioned_query = union(*all_tables)
+    count_over_time = db_connection.execute(unioned_query).all()
+    return count_over_time
+
+
+def get_un_triaged_finding_count_by_vcs_provider_over_time(db_connection: Session, weeks: int = 13) -> list[Row]:
+    """
+        Retrieve count of un triaged findings over time for given weeks
+    :param db_connection:
+        Session of the database connection
+    :param weeks:
+        optional, filter on last n weeks, default 13
+    :return: count_over_time
+        list of rows containing un triaged findings count over time per week
+    """
+    all_tables = []
+    for week in range(0, weeks):
+        last_nth_week_date_time = datetime.utcnow() - timedelta(weeks=week)
+        query = db_connection.query(extract('year', last_nth_week_date_time).label("year"),
+                                    extract('week', last_nth_week_date_time).label("week"),
+                                    model.DBVcsInstance.provider_type.label("provider_type"),
+                                    func.count(model.DBfinding.id_).label("finding_count")
+                                    )
+        max_base_scan = db_connection.query(func.max(model.DBscan.id_).label("scan_id"),
+                                            model.DBscan.branch_id) \
+            .filter(extract('year', model.DBscan.timestamp) == extract('year', last_nth_week_date_time)) \
+            .filter(extract('week', model.DBscan.timestamp) <= extract('week', last_nth_week_date_time)) \
+            .filter(model.DBscan.scan_type == ScanType.BASE) \
+            .group_by(model.DBscan.branch_id).subquery()
+
+        max_audit_subquery = db_connection.query(model.DBaudit.finding_id,
+                                                 func.max(model.DBaudit.id_).label("audit_id")) \
+            .filter(extract('year', model.DBaudit.timestamp) == extract('year', last_nth_week_date_time)) \
+            .filter(extract('week', model.DBaudit.timestamp) <= extract('week', last_nth_week_date_time)) \
+            .group_by(model.DBaudit.finding_id).subquery()
+
+        query = query.join(model.DBscanFinding, model.DBfinding.id_ == model.DBscanFinding.finding_id)
+        query = query.join(model.DBscan, model.DBscan.id_ == model.DBscanFinding.scan_id)
+        query = query.join(max_base_scan, and_(max_base_scan.c.branch_id == model.DBscan.branch_id,
+                                               or_(model.DBscan.id_ == max_base_scan.c.scan_id,
+                                                   (and_(model.DBscan.id_ > max_base_scan.c.scan_id,
+                                                         model.DBscan.scan_type == ScanType.INCREMENTAL,
+                                                         extract('week', model.DBscan.timestamp) <=
+                                                         extract('week', last_nth_week_date_time),
+                                                         extract('year', model.DBscan.timestamp) ==
+                                                         extract('year', last_nth_week_date_time)))
+                                                   )
+                                               )
+                           )
+        query = query.join(model.DBbranch, model.DBbranch.id_ == model.DBscan.branch_id)
+        query = query.join(model.DBrepository, model.DBrepository.id_ == model.DBbranch.repository_id)
+        query = query.join(model.DBVcsInstance, model.DBVcsInstance.id_ == model.DBrepository.vcs_instance)
+
+        query = query.join(max_audit_subquery, max_audit_subquery.c.finding_id == model.finding.DBfinding.id_,
+                           isouter=True)
+        query = query.join(model.DBaudit, and_(model.audit.DBaudit.finding_id == model.finding.DBfinding.id_,
+                                               model.audit.DBaudit.id_ == max_audit_subquery.c.audit_id),
+                           isouter=True)
+        query = query.filter(
+            or_(model.DBaudit.id_ == None, model.DBaudit.status == FindingStatus.NOT_ANALYZED))  # noqa: E711
+
+        query = query.group_by(model.DBVcsInstance.provider_type)
+
+        all_tables.append(query)
+
+    # union
+    unioned_query = union(*all_tables)
+    count_over_time = db_connection.execute(unioned_query).all()
+    return count_over_time

components/resc-backend/src/resc_backend/resc_web_service/endpoints/metrics.py

@@ -0,0 +1,144 @@
+# Standard Library
+import logging
+from datetime import datetime, timedelta
+from typing import Optional
+
+# Third Party
+from fastapi import APIRouter, Depends, Query, status
+
+# First Party
+from resc_backend.constants import (
+    ERROR_MESSAGE_500,
+    ERROR_MESSAGE_503,
+    METRICS_TAG,
+    RWS_ROUTE_AUDITED_COUNT_OVER_TIME,
+    RWS_ROUTE_COUNT_PER_VCS_PROVIDER_BY_WEEK,
+    RWS_ROUTE_METRICS,
+    RWS_ROUTE_UN_TRIAGED_COUNT_OVER_TIME
+)
+from resc_backend.db.connection import Session
+from resc_backend.resc_web_service.crud import finding as finding_crud
+from resc_backend.resc_web_service.dependencies import get_db_connection
+from resc_backend.resc_web_service.schema.finding_count_over_time import FindingCountOverTime
+from resc_backend.resc_web_service.schema.finding_status import FindingStatus
+from resc_backend.resc_web_service.schema.vcs_provider import VCSProviders
+
+router = APIRouter(prefix=f"{RWS_ROUTE_METRICS}", tags=[METRICS_TAG])
+logger = logging.getLogger(__name__)
+
+
+@router.get(f"{RWS_ROUTE_AUDITED_COUNT_OVER_TIME}",
+            response_model=list[FindingCountOverTime],
+            summary="Get count of audit status over time for given weeks per vcs provider",
+            status_code=status.HTTP_200_OK,
+            responses={
+                200: {"description": "Retrieve count of audit status over time for given weeks per vcs provider"},
+                500: {"description": ERROR_MESSAGE_500},
+                503: {"description": ERROR_MESSAGE_503}
+            })
+def get_finding_audit_count_over_time(db_connection: Session = Depends(get_db_connection),
+                                      weeks: Optional[int] = Query(default=13, ge=1),
+                                      audit_status: Optional[FindingStatus] = Query(default=FindingStatus.TRUE_POSITIVE)
+                                      ) -> list[FindingCountOverTime]:
+    """
+        Retrieve count of audited findings over time for given weeks per vcs provider
+    - **db_connection**: Session of the database connection
+    - **weeks**: Nr of weeks for which to retrieve the audit status count
+    - **audit_status**: audit status for which to retrieve the counts, defaults to True positive
+    - **return**: [DateCountModel]
+        The output will contain a list of DateCountModel type objects
+    """
+    audit_counts = finding_crud.get_finding_audit_status_count_over_time(db_connection=db_connection,
+                                                                         status=audit_status,
+                                                                         weeks=weeks)
+    output = convert_rows_to_finding_count_over_time(count_over_time=audit_counts, weeks=weeks)
+    return output
+
+
+@router.get(f"{RWS_ROUTE_COUNT_PER_VCS_PROVIDER_BY_WEEK}",
+            response_model=list[FindingCountOverTime],
+            summary="Get count of findings over time for given weeks per vcs provider",
+            status_code=status.HTTP_200_OK,
+            responses={
+                200: {"description": "Retrieve count of findings over time for given weeks per vcs provider"},
+                500: {"description": ERROR_MESSAGE_500},
+                503: {"description": ERROR_MESSAGE_503}
+            })
+def get_finding_total_count_over_time(db_connection: Session = Depends(get_db_connection),
+                                      weeks: Optional[int] = Query(default=13, ge=1)) -> list[FindingCountOverTime]:
+    """
+        Retrieve count of findings over time for given weeks per vcs provider
+    - **db_connection**: Session of the database connection
+    - **weeks**: Nr of weeks for which to retrieve the audit status count
+    - **audit_status**: audit status for which to retrieve the counts, defaults to True positive
+    - **return**: [DateCountModel]
+        The output will contain a list of DateCountModel type objects
+    """
+    audit_counts = finding_crud.get_finding_count_by_vcs_provider_over_time(db_connection=db_connection, weeks=weeks)
+    output = convert_rows_to_finding_count_over_time(count_over_time=audit_counts, weeks=weeks)
+    return output
+
+
+@router.get(f"{RWS_ROUTE_UN_TRIAGED_COUNT_OVER_TIME}",
+            response_model=list[FindingCountOverTime],
+            summary="Get count of UnTriaged findings over time for given weeks per vcs provider",
+            status_code=status.HTTP_200_OK,
+            responses={
+                200: {"description": "Retrieve count of UnTriaged findings over time for given weeks per vcs provider"},
+                500: {"description": ERROR_MESSAGE_500},
+                503: {"description": ERROR_MESSAGE_503}
+            })
+def get_finding_un_triaged_count_over_time(db_connection: Session = Depends(get_db_connection),
+                                           weeks: Optional[int] = Query(default=13, ge=1)) \
+        -> list[FindingCountOverTime]:
+    """
+        Retrieve count of UnTriaged findings over time for given weeks per vcs provider
+    - **db_connection**: Session of the database connection
+    - **weeks**: Nr of weeks for which to retrieve the audit status count
+    - **audit_status**: audit status for which to retrieve the counts, defaults to True positive
+    - **return**: [DateCountModel]
+        The output will contain a list of DateCountModel type objects
+    """
+    audit_counts = finding_crud.get_un_triaged_finding_count_by_vcs_provider_over_time(db_connection=db_connection,
+                                                                                       weeks=weeks)
+    output = convert_rows_to_finding_count_over_time(count_over_time=audit_counts, weeks=weeks)
+    return output
+
+
+def convert_rows_to_finding_count_over_time(count_over_time: dict, weeks: int) -> list[FindingCountOverTime]:
+    """
+        Convert the rows from the database to the format of list[FindingCountOverTime]
+    :param count_over_time:
+        rows from the database
+    :param weeks:
+        number fo weeks that are in the data
+    :return: output
+        list[FindingCountOverTime]
+    """
+    # Define the vcs provider types and finding statuses
+    vcs_provider_types = list(VCSProviders)
+
+    # create defaults with 0 value
+    week_groups = {}
+    for week in range(0, weeks):
+        nth_week = datetime.utcnow() - timedelta(weeks=week)
+        week = f"{nth_week.isocalendar().year} W{nth_week.isocalendar().week:02d}"
+        week_groups[week] = {vcs_provider_type: 0 for vcs_provider_type in vcs_provider_types + ["total"]}
+
+    # loop over the counts from the database
+    for data in count_over_time:
+        week = f"{data['year']} W{data['week']:02d}"
+        finding_count = data["finding_count"]
+
+        week_groups[week][data["provider_type"]] += finding_count
+        week_groups[week]["total"] += finding_count
+
+    # Convert to the output format
+    output = []
+    for week in sorted(week_groups.keys()):
+        week_data = FindingCountOverTime(time_period=week, total=week_groups[week]["total"])
+        for vcs_provider_type in vcs_provider_types:
+            setattr(week_data.vcs_provider_finding_count, vcs_provider_type, week_groups[week][vcs_provider_type])
+
+        output.append(week_data)
+    return output

components/resc-backend/src/resc_backend/resc_web_service/schema/finding_count_over_time.py

@@ -0,0 +1,15 @@
+# pylint: disable=no-name-in-module
+# Third Party
+from pydantic import BaseModel
+
+
+class VcsProviderFindingCount(BaseModel):
+    AZURE_DEVOPS: int = 0
+    BITBUCKET: int = 0
+    GITHUB_PUBLIC: int = 0
+
+
+class FindingCountOverTime(BaseModel):
+    time_period: str
+    vcs_provider_finding_count: VcsProviderFindingCount = VcsProviderFindingCount()
+    total: int = 0