Merge branch 'kiuwan-sca' of github.com:mwager/django-DefectDojo into…

… kiuwan-sca

* 'kiuwan-sca' of github.com:mwager/django-DefectDojo:
  Update versions in application files
  Product Metrics: Performance Enhancements (DefectDojo#10059)
  String Based Filtering: Follow on for DefectDojo#10038 (DefectDojo#10050)
  update semgrep tests (DefectDojo#10058)
  Jira Webhook: Reorg logging and responses (DefectDojo#10049)
  Similar Findings: Create Toggle (DefectDojo#10047)
  Bump social-auth-app-django from 5.4.0 to 5.4.1 (DefectDojo#10026)
  Update versions in application files
  Update versions in application files
  Updated DryRun Security config (DefectDojo#10037)
  Filtering Performance: Add opt-in setting for converting to string ba… (DefectDojo#10038)
  Updates to semgrep parser (DefectDojo#10033)
  Update versions in application files

.dryrunsecurity.yaml

@@ -66,7 +66,7 @@ allowedAuthors:
     - kiblik
     - dsever
     - dogboat
-    - FelixHernandez
+    - hblankenship
 notificationList:
   - '@mtesauro'
   - '@grendel513'

components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "defectdojo",
-  "version": "2.33.5",
+  "version": "2.33.7",
   "license" : "BSD-3-Clause",
   "private": true,
   "dependencies": {

docs/content/en/usage/performance.md

@@ -5,6 +5,16 @@ draft: false
 weight: 4
 ---
 
+## Filter String Matching Optimization
+
+IN the UI, many of the filters for a given object will also query related objects
+for an easy visual match of an item to filter on. For instances with many objects,
+this could lead to a considerable performance hit. To alleviate this constriction,
+enable the "Filter String Matching Optimization" setting in the System Settings to
+change many filters to only search on names, rather than the objects themselves.
+This change will save many large queries, and will improve the performance of UI
+based interactions.
+
 ## Asynchronous Import
 
 DefectDojo offers an experimental feature to aynschronously import security reports. 

dojo/__init__.py

@@ -4,6 +4,6 @@
 # Django starts so that shared_task will use this app.
 from .celery import app as celery_app  # noqa: F401
 
-__version__ = '2.33.5'
+__version__ = '2.33.7'
 __url__ = 'https://github.com/DefectDojo/django-DefectDojo'
 __docs__ = 'https://documentation.defectdojo.com'

dojo/components/views.py

@@ -1,8 +1,8 @@
 from django.shortcuts import render
 from django.db.models import Count, Q
 from django.db.models.expressions import Value
-from dojo.utils import add_breadcrumb, get_page_items
-from dojo.filters import ComponentFilter
+from dojo.utils import add_breadcrumb, get_page_items, get_system_setting
+from dojo.filters import ComponentFilter, ComponentFilterWithoutObjectLookups
 from dojo.components.sql_group_concat import Sql_GroupConcat
 from django.db import connection
 from django.contrib.postgres.aggregates import StringAgg
@@ -52,7 +52,9 @@ def components(request):
         "-total"
     )  # Default sort by total descending
 
-    comp_filter = ComponentFilter(request.GET, queryset=component_query)
+    filter_string_matching = get_system_setting("filter_string_matching", False)
+    filter_class = ComponentFilterWithoutObjectLookups if filter_string_matching else ComponentFilter
+    comp_filter = filter_class(request.GET, queryset=component_query)
     result = get_page_items(request, comp_filter.qs, 25)
 
     # Filter out None values for auto-complete

dojo/db_migrations/0210_system_settings_filter_string_matching.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.13 on 2024-04-25 18:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0209_alter_finding_severity'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='system_settings',
+            name='filter_string_matching',
+            field=models.BooleanField(default=False, help_text='When turned on, all filter operations in the UI will require string matches rather than ID. This is a performance enhancement to avoid fetching objects unnecessarily.', verbose_name='Filter String Matching Optimization'),
+        ),
+    ]

dojo/db_migrations/0211_system_settings_enable_similar_findings.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.13 on 2024-04-26 21:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0210_system_settings_filter_string_matching'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='system_settings',
+            name='enable_similar_findings',
+            field=models.BooleanField(default=True, help_text='Enable the query of similar findings on the view finding page. This feature can involve potentially large queries and negatively impact performance', verbose_name='Enable Similar Findings'),
+        ),
+    ]

dojo/endpoint/views.py

@@ -12,12 +12,12 @@
 from django.db.models import Q, QuerySet, Count
 
 from dojo.endpoint.utils import clean_hosts_run, endpoint_meta_import
-from dojo.filters import EndpointFilter
+from dojo.filters import EndpointFilter, EndpointFilterWithoutObjectLookups
 from dojo.forms import EditEndpointForm, \
     DeleteEndpointForm, AddEndpointForm, DojoMetaDataForm, ImportEndpointMetaForm
 from dojo.models import Product, Endpoint, Finding, DojoMeta, Endpoint_Status
 from dojo.utils import get_page_items, add_breadcrumb, get_period_counts, Product_Tab, calculate_grade, redirect, \
-    add_error_message_to_response, is_scan_file_too_large
+    add_error_message_to_response, is_scan_file_too_large, get_system_setting
 from dojo.notifications.helper import create_notification
 from dojo.authorization.authorization_decorators import user_is_authorized
 from dojo.authorization.roles_permissions import Permissions
@@ -42,12 +42,13 @@ def process_endpoints_view(request, host_view=False, vulnerable=False):
 
     endpoints = endpoints.prefetch_related('product', 'product__tags', 'tags').distinct()
     endpoints = get_authorized_endpoints(Permissions.Endpoint_View, endpoints, request.user)
-
+    filter_string_matching = get_system_setting("filter_string_matching", False)
+    filter_class = EndpointFilterWithoutObjectLookups if filter_string_matching else EndpointFilter
     if host_view:
-        ids = get_endpoint_ids(EndpointFilter(request.GET, queryset=endpoints, user=request.user).qs)
-        endpoints = EndpointFilter(request.GET, queryset=endpoints.filter(id__in=ids), user=request.user)
+        ids = get_endpoint_ids(filter_class(request.GET, queryset=endpoints, user=request.user).qs)
+        endpoints = filter_class(request.GET, queryset=endpoints.filter(id__in=ids), user=request.user)
     else:
-        endpoints = EndpointFilter(request.GET, queryset=endpoints, user=request.user)
+        endpoints = filter_class(request.GET, queryset=endpoints, user=request.user)
 
     paged_endpoints = get_page_items(request, endpoints.qs, 25)
 

dojo/engagement/views.py

@@ -1,6 +1,7 @@
 import logging
 import csv
 import re
+from typing import List
 from django.views import View
 from openpyxl import Workbook
 from openpyxl.styles import Font
@@ -14,7 +15,7 @@
 from django.core.exceptions import ValidationError, PermissionDenied
 from django.urls import reverse, Resolver404
 from django.db.models import Q, Count
-from django.http import HttpResponseRedirect, StreamingHttpResponse, HttpResponse, FileResponse, QueryDict
+from django.http import HttpResponseRedirect, StreamingHttpResponse, HttpResponse, FileResponse, QueryDict, HttpRequest
 from django.shortcuts import render, get_object_or_404
 from django.views.decorators.cache import cache_page
 from django.utils import timezone
@@ -23,7 +24,14 @@
 from django.db import DEFAULT_DB_ALIAS
 
 from dojo.engagement.services import close_engagement, reopen_engagement
-from dojo.filters import EngagementFilter, EngagementDirectFilter, EngagementTestFilter
+from dojo.filters import (
+    EngagementFilter,
+    EngagementFilterWithoutObjectLookups,
+    EngagementDirectFilter,
+    EngagementDirectFilterWithoutObjectLookups,
+    EngagementTestFilter,
+    EngagementTestFilterWithoutObjectLookups
+)
 from dojo.forms import CheckForm, \
     UploadThreatForm, RiskAcceptanceForm, NoteForm, DoneForm, \
     EngForm, TestForm, ReplaceRiskAcceptanceProofForm, AddFindingsRiskAcceptanceForm, DeleteEngagementForm, ImportScanForm, \
@@ -112,7 +120,9 @@ def get_filtered_engagements(request, view):
             'product__jira_project_set__jira_instance'
         )
 
-    engagements = EngagementDirectFilter(request.GET, queryset=engagements)
+    filter_string_matching = get_system_setting("filter_string_matching", False)
+    filter_class = EngagementDirectFilterWithoutObjectLookups if filter_string_matching else EngagementDirectFilter
+    engagements = filter_class(request.GET, queryset=engagements)
 
     return engagements
 
@@ -181,8 +191,9 @@ def engagements_all(request):
             'engagement_set__jira_project__jira_instance',
             'jira_project_set__jira_instance'
         )
-
-    filtered = EngagementFilter(
+    filter_string_matching = get_system_setting("filter_string_matching", False)
+    filter_class = EngagementFilterWithoutObjectLookups if filter_string_matching else EngagementFilter
+    filtered = filter_class(
         request.GET,
         queryset=filter_qs
     )
@@ -384,11 +395,21 @@ def get_risks_accepted(self, eng):
         risks_accepted = eng.risk_acceptance.all().select_related('owner').annotate(accepted_findings_count=Count('accepted_findings__id'))
         return risks_accepted
 
+    def get_filtered_tests(
+        self,
+        request: HttpRequest,
+        queryset: List[Test],
+        engagement: Engagement,
+    ):
+        filter_string_matching = get_system_setting("filter_string_matching", False)
+        filter_class = EngagementTestFilterWithoutObjectLookups if filter_string_matching else EngagementTestFilter
+        return filter_class(request.GET, queryset=queryset, engagement=engagement)
+
     def get(self, request, eid, *args, **kwargs):
         eng = get_object_or_404(Engagement, id=eid)
         tests = eng.test_set.all().order_by('test_type__name', '-updated')
         default_page_num = 10
-        tests_filter = EngagementTestFilter(request.GET, queryset=tests, engagement=eng)
+        tests_filter = self.get_filtered_tests(request, tests, eng)
         paged_tests = get_page_items(request, tests_filter.qs, default_page_num)
         paged_tests.object_list = prefetch_for_view_tests(paged_tests.object_list)
         prod = eng.product
@@ -458,7 +479,7 @@ def post(self, request, eid, *args, **kwargs):
 
         default_page_num = 10
 
-        tests_filter = EngagementTestFilter(request.GET, queryset=tests, engagement=eng)
+        tests_filter = self.get_filtered_tests(request, tests, eng)
         paged_tests = get_page_items(request, tests_filter.qs, default_page_num)
         # prefetch only after creating the filters to avoid https://code.djangoproject.com/ticket/23771 and https://code.djangoproject.com/ticket/25375
         paged_tests.object_list = prefetch_for_view_tests(paged_tests.object_list)