diff --git a/.dryrunsecurity.yaml b/.dryrunsecurity.yaml index 21d26b11375..da92963ddac 100644 --- a/.dryrunsecurity.yaml +++ b/.dryrunsecurity.yaml @@ -66,7 +66,7 @@ allowedAuthors: - kiblik - dsever - dogboat - - FelixHernandez + - hblankenship notificationList: - '@mtesauro' - '@grendel513' diff --git a/components/package.json b/components/package.json index 336f73dfe74..8b076e0bbbc 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.33.5", + "version": "2.33.7", "license" : "BSD-3-Clause", "private": true, "dependencies": { diff --git a/docs/content/en/usage/performance.md b/docs/content/en/usage/performance.md index d7957ddb724..70dfa44b421 100644 --- a/docs/content/en/usage/performance.md +++ b/docs/content/en/usage/performance.md @@ -5,6 +5,16 @@ draft: false weight: 4 --- +## Filter String Matching Optimization + +IN the UI, many of the filters for a given object will also query related objects +for an easy visual match of an item to filter on. For instances with many objects, +this could lead to a considerable performance hit. To alleviate this constriction, +enable the "Filter String Matching Optimization" setting in the System Settings to +change many filters to only search on names, rather than the objects themselves. +This change will save many large queries, and will improve the performance of UI +based interactions. + ## Asynchronous Import DefectDojo offers an experimental feature to aynschronously import security reports. diff --git a/dojo/__init__.py b/dojo/__init__.py index c2f8acb4004..24987ce2acc 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = '2.33.5' +__version__ = '2.33.7' __url__ = 'https://github.com/DefectDojo/django-DefectDojo' __docs__ = 'https://documentation.defectdojo.com' diff --git a/dojo/components/views.py b/dojo/components/views.py index 717f79d86a1..82898f6fa8e 100644 --- a/dojo/components/views.py +++ b/dojo/components/views.py @@ -1,8 +1,8 @@ from django.shortcuts import render from django.db.models import Count, Q from django.db.models.expressions import Value -from dojo.utils import add_breadcrumb, get_page_items -from dojo.filters import ComponentFilter +from dojo.utils import add_breadcrumb, get_page_items, get_system_setting +from dojo.filters import ComponentFilter, ComponentFilterWithoutObjectLookups from dojo.components.sql_group_concat import Sql_GroupConcat from django.db import connection from django.contrib.postgres.aggregates import StringAgg @@ -52,7 +52,9 @@ def components(request): "-total" ) # Default sort by total descending - comp_filter = ComponentFilter(request.GET, queryset=component_query) + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = ComponentFilterWithoutObjectLookups if filter_string_matching else ComponentFilter + comp_filter = filter_class(request.GET, queryset=component_query) result = get_page_items(request, comp_filter.qs, 25) # Filter out None values for auto-complete diff --git a/dojo/db_migrations/0210_system_settings_filter_string_matching.py b/dojo/db_migrations/0210_system_settings_filter_string_matching.py new file mode 100644 index 00000000000..de1f617c4d0 --- /dev/null +++ b/dojo/db_migrations/0210_system_settings_filter_string_matching.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.13 on 2024-04-25 18:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0209_alter_finding_severity'), + ] + + operations = [ + migrations.AddField( + model_name='system_settings', + name='filter_string_matching', + field=models.BooleanField(default=False, help_text='When turned on, all filter operations in the UI will require string matches rather than ID. This is a performance enhancement to avoid fetching objects unnecessarily.', verbose_name='Filter String Matching Optimization'), + ), + ] diff --git a/dojo/db_migrations/0211_system_settings_enable_similar_findings.py b/dojo/db_migrations/0211_system_settings_enable_similar_findings.py new file mode 100644 index 00000000000..014977be7d0 --- /dev/null +++ b/dojo/db_migrations/0211_system_settings_enable_similar_findings.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.13 on 2024-04-26 21:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0210_system_settings_filter_string_matching'), + ] + + operations = [ + migrations.AddField( + model_name='system_settings', + name='enable_similar_findings', + field=models.BooleanField(default=True, help_text='Enable the query of similar findings on the view finding page. This feature can involve potentially large queries and negatively impact performance', verbose_name='Enable Similar Findings'), + ), + ] diff --git a/dojo/endpoint/views.py b/dojo/endpoint/views.py index d35a1390988..bd1a85534e4 100644 --- a/dojo/endpoint/views.py +++ b/dojo/endpoint/views.py @@ -12,12 +12,12 @@ from django.db.models import Q, QuerySet, Count from dojo.endpoint.utils import clean_hosts_run, endpoint_meta_import -from dojo.filters import EndpointFilter +from dojo.filters import EndpointFilter, EndpointFilterWithoutObjectLookups from dojo.forms import EditEndpointForm, \ DeleteEndpointForm, AddEndpointForm, DojoMetaDataForm, ImportEndpointMetaForm from dojo.models import Product, Endpoint, Finding, DojoMeta, Endpoint_Status from dojo.utils import get_page_items, add_breadcrumb, get_period_counts, Product_Tab, calculate_grade, redirect, \ - add_error_message_to_response, is_scan_file_too_large + add_error_message_to_response, is_scan_file_too_large, get_system_setting from dojo.notifications.helper import create_notification from dojo.authorization.authorization_decorators import user_is_authorized from dojo.authorization.roles_permissions import Permissions @@ -42,12 +42,13 @@ def process_endpoints_view(request, host_view=False, vulnerable=False): endpoints = endpoints.prefetch_related('product', 'product__tags', 'tags').distinct() endpoints = get_authorized_endpoints(Permissions.Endpoint_View, endpoints, request.user) - + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EndpointFilterWithoutObjectLookups if filter_string_matching else EndpointFilter if host_view: - ids = get_endpoint_ids(EndpointFilter(request.GET, queryset=endpoints, user=request.user).qs) - endpoints = EndpointFilter(request.GET, queryset=endpoints.filter(id__in=ids), user=request.user) + ids = get_endpoint_ids(filter_class(request.GET, queryset=endpoints, user=request.user).qs) + endpoints = filter_class(request.GET, queryset=endpoints.filter(id__in=ids), user=request.user) else: - endpoints = EndpointFilter(request.GET, queryset=endpoints, user=request.user) + endpoints = filter_class(request.GET, queryset=endpoints, user=request.user) paged_endpoints = get_page_items(request, endpoints.qs, 25) diff --git a/dojo/engagement/views.py b/dojo/engagement/views.py index 230c18cc3a0..bdf4c2cb143 100644 --- a/dojo/engagement/views.py +++ b/dojo/engagement/views.py @@ -1,6 +1,7 @@ import logging import csv import re +from typing import List from django.views import View from openpyxl import Workbook from openpyxl.styles import Font @@ -14,7 +15,7 @@ from django.core.exceptions import ValidationError, PermissionDenied from django.urls import reverse, Resolver404 from django.db.models import Q, Count -from django.http import HttpResponseRedirect, StreamingHttpResponse, HttpResponse, FileResponse, QueryDict +from django.http import HttpResponseRedirect, StreamingHttpResponse, HttpResponse, FileResponse, QueryDict, HttpRequest from django.shortcuts import render, get_object_or_404 from django.views.decorators.cache import cache_page from django.utils import timezone @@ -23,7 +24,14 @@ from django.db import DEFAULT_DB_ALIAS from dojo.engagement.services import close_engagement, reopen_engagement -from dojo.filters import EngagementFilter, EngagementDirectFilter, EngagementTestFilter +from dojo.filters import ( + EngagementFilter, + EngagementFilterWithoutObjectLookups, + EngagementDirectFilter, + EngagementDirectFilterWithoutObjectLookups, + EngagementTestFilter, + EngagementTestFilterWithoutObjectLookups +) from dojo.forms import CheckForm, \ UploadThreatForm, RiskAcceptanceForm, NoteForm, DoneForm, \ EngForm, TestForm, ReplaceRiskAcceptanceProofForm, AddFindingsRiskAcceptanceForm, DeleteEngagementForm, ImportScanForm, \ @@ -112,7 +120,9 @@ def get_filtered_engagements(request, view): 'product__jira_project_set__jira_instance' ) - engagements = EngagementDirectFilter(request.GET, queryset=engagements) + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EngagementDirectFilterWithoutObjectLookups if filter_string_matching else EngagementDirectFilter + engagements = filter_class(request.GET, queryset=engagements) return engagements @@ -181,8 +191,9 @@ def engagements_all(request): 'engagement_set__jira_project__jira_instance', 'jira_project_set__jira_instance' ) - - filtered = EngagementFilter( + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EngagementFilterWithoutObjectLookups if filter_string_matching else EngagementFilter + filtered = filter_class( request.GET, queryset=filter_qs ) @@ -384,11 +395,21 @@ def get_risks_accepted(self, eng): risks_accepted = eng.risk_acceptance.all().select_related('owner').annotate(accepted_findings_count=Count('accepted_findings__id')) return risks_accepted + def get_filtered_tests( + self, + request: HttpRequest, + queryset: List[Test], + engagement: Engagement, + ): + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EngagementTestFilterWithoutObjectLookups if filter_string_matching else EngagementTestFilter + return filter_class(request.GET, queryset=queryset, engagement=engagement) + def get(self, request, eid, *args, **kwargs): eng = get_object_or_404(Engagement, id=eid) tests = eng.test_set.all().order_by('test_type__name', '-updated') default_page_num = 10 - tests_filter = EngagementTestFilter(request.GET, queryset=tests, engagement=eng) + tests_filter = self.get_filtered_tests(request, tests, eng) paged_tests = get_page_items(request, tests_filter.qs, default_page_num) paged_tests.object_list = prefetch_for_view_tests(paged_tests.object_list) prod = eng.product @@ -458,7 +479,7 @@ def post(self, request, eid, *args, **kwargs): default_page_num = 10 - tests_filter = EngagementTestFilter(request.GET, queryset=tests, engagement=eng) + tests_filter = self.get_filtered_tests(request, tests, eng) paged_tests = get_page_items(request, tests_filter.qs, default_page_num) # prefetch only after creating the filters to avoid https://code.djangoproject.com/ticket/23771 and https://code.djangoproject.com/ticket/25375 paged_tests.object_list = prefetch_for_view_tests(paged_tests.object_list) diff --git a/dojo/filters.py b/dojo/filters.py index 4f1f3c539ad..8dc61379104 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -289,7 +289,7 @@ def get_tags_label_from_model(model): return 'Tags (Unknown)' -def get_finding_filterset_fields(metrics=False, similar=False): +def get_finding_filterset_fields(metrics=False, similar=False, filter_string_matching=False): fields = [] if similar: @@ -315,10 +315,28 @@ def get_finding_filterset_fields(metrics=False, similar=False): 'mitigated', 'reporter', 'reviewers', - 'test__engagement__product__prod_type', - 'test__engagement__product', - 'test__engagement', - 'test', + ]) + + if filter_string_matching: + fields.extend([ + 'reporter', + 'reviewers', + 'test__engagement__product__prod_type__name', + 'test__engagement__product__name', + 'test__engagement__name', + 'test__title', + ]) + else: + fields.extend([ + 'reporter', + 'reviewers', + 'test__engagement__product__prod_type', + 'test__engagement__product', + 'test__engagement', + 'test', + ]) + + fields.extend([ 'test__test_type', 'test__engagement__version', 'test__version', @@ -348,9 +366,9 @@ def get_finding_filterset_fields(metrics=False, similar=False): ]) fields.extend([ - 'param', - 'payload', - 'risk_acceptance', + 'param', + 'payload', + 'risk_acceptance', ]) if get_system_setting('enable_jira'): @@ -362,10 +380,16 @@ def get_finding_filterset_fields(metrics=False, similar=False): ]) if is_finding_groups_enabled(): - fields.extend([ - 'has_finding_group', - 'finding_group', - ]) + if filter_string_matching: + fields.extend([ + 'has_finding_group', + 'finding_group__name', + ]) + else: + fields.extend([ + 'has_finding_group', + 'finding_group', + ]) if get_system_setting('enable_jira'): fields.extend([ @@ -375,73 +399,155 @@ def get_finding_filterset_fields(metrics=False, similar=False): return fields -class FindingFilterWithTags(DojoFilter): +class FindingTagFilter(DojoFilter): + tag = CharFilter( + field_name="tags__name", + lookup_expr="icontains", + label="Tag name contains", + help_text="Search for tags on a Finding that contain a given pattern") tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Finding.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - + field_name="tags__name", + to_field_name="name", + queryset=Finding.tags.tag_model.objects.all().order_by("name"), + help_text="Filter Findings by the selected tags") test__tags = ModelMultipleChoiceFilter( - field_name='test__tags__name', - to_field_name='name', - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - + field_name="test__tags__name", + to_field_name="name", + queryset=Test.tags.tag_model.objects.all().order_by("name"), + help_text="Filter Tests by the selected tags") test__engagement__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__tags__name', - to_field_name='name', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - + field_name="test__engagement__tags__name", + to_field_name="name", + queryset=Engagement.tags.tag_model.objects.all().order_by("name"), + help_text="Filter Engagements by the selected tags") test__engagement__product__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__product__tags__name', - to_field_name='name', - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') + field_name="test__engagement__product__tags__name", + to_field_name="name", + queryset=Product.tags.tag_model.objects.all().order_by("name"), + help_text="Filter Products by the selected tags") not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Finding.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - + field_name="tags__name", + to_field_name="name", + queryset=Finding.tags.tag_model.objects.all().order_by("name"), + help_text="Search for tags on a Finding that contain a given pattern, and exclude them", + exclude=True) not_test__tags = ModelMultipleChoiceFilter( - field_name='test__tags__name', - to_field_name='name', - exclude=True, - label='Test without tags', - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - + field_name="test__tags__name", + to_field_name="name", + label="Test without tags", + queryset=Test.tags.tag_model.objects.all().order_by("name"), + help_text="Search for tags on a Test that contain a given pattern, and exclude them", + exclude=True) not_test__engagement__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__tags__name', - to_field_name='name', - exclude=True, - label='Engagement without tags', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - + field_name="test__engagement__tags__name", + to_field_name="name", + label="Engagement without tags", + queryset=Engagement.tags.tag_model.objects.all().order_by("name"), + help_text="Search for tags on a Engagement that contain a given pattern, and exclude them", + exclude=True) not_test__engagement__product__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__product__tags__name', - to_field_name='name', - exclude=True, - label='Product without tags', - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) + field_name="test__engagement__product__tags__name", + to_field_name="name", + label="Product without tags", + queryset=Product.tags.tag_model.objects.all().order_by("name"), + help_text="Search for tags on a Product that contain a given pattern, and exclude them", + exclude=True) - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class FindingTagStringFilter(FilterSet): + tags_contains = CharFilter( + label="Finding Tag Contains", + field_name="tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + tags = CharFilter( + label="Finding Tag", + field_name="tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + test__tags_contains = CharFilter( + label="Test Tag Contains", + field_name="test__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + test__tags = CharFilter( + label="Test Tag", + field_name="test__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + test__engagement__tags_contains = CharFilter( + label="Engagement Tag Contains", + field_name="test__engagement__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + test__engagement__tags = CharFilter( + label="Engagement Tag", + field_name="test__engagement__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + test__engagement__product__tags_contains = CharFilter( + label="Product Tag Contains", + field_name="test__engagement__product__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + test__engagement__product__tags = CharFilter( + label="Product Tag", + field_name="test__engagement__product__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + + not_tags_contains = CharFilter( + label="Finding Tag Does Not Contain", + field_name="tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern, and exclude them", + exclude=True) + not_tags = CharFilter( + label="Not Finding Tag", + field_name="tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match, and exclude them", + exclude=True) + not_test__tags_contains = CharFilter( + label="Test Tag Does Not Contain", + field_name="test__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Test that contain a given pattern, and exclude them", + exclude=True) + not_test__tags = CharFilter( + label="Not Test Tag", + field_name="test__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Test that are an exact match, and exclude them", + exclude=True) + not_test__engagement__tags_contains = CharFilter( + label="Engagement Tag Does Not Contain", + field_name="test__engagement__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Engagement that contain a given pattern, and exclude them", + exclude=True) + not_test__engagement__tags = CharFilter( + label="Not Engagement Tag", + field_name="test__engagement__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Engagement that are an exact match, and exclude them", + exclude=True) + not_test__engagement__product__tags_contains = CharFilter( + label="Product Tag Does Not Contain", + field_name="test__engagement__product__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Product that contain a given pattern, and exclude them", + exclude=True) + not_test__engagement__product__tags = CharFilter( + label="Not Product Tag", + field_name="test__engagement__product__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Product that are an exact match, and exclude them", + exclude=True) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -720,6 +826,29 @@ class ProductComponentFilter(DojoFilter): ) +class ComponentFilterWithoutObjectLookups(ProductComponentFilter): + test__engagement__product__prod_type__name = CharFilter( + field_name="test__engagement__product__prod_type__name", + lookup_expr="iexact", + label="Product Type Name", + help_text="Search for Product Type names that are an exact match") + test__engagement__product__prod_type__name_contains = CharFilter( + field_name="test__engagement__product__prod_type__name", + lookup_expr="icontains", + label="Product Type Name Contains", + help_text="Search for Product Type names that contain a given pattern") + test__engagement__product__name = CharFilter( + field_name="test__engagement__product__name", + lookup_expr="iexact", + label="Product Name", + help_text="Search for Product names that are an exact match") + test__engagement__product__name_contains = CharFilter( + field_name="test__engagement__product__name", + lookup_expr="icontains", + label="Product Name Contains", + help_text="Search for Product names that contain a given pattern") + + class ComponentFilter(ProductComponentFilter): test__engagement__product__prod_type = ModelMultipleChoiceFilter( queryset=Product_Type.objects.none(), @@ -736,125 +865,140 @@ def __init__(self, *args, **kwargs): 'test__engagement__product'].queryset = get_authorized_products(Permissions.Product_View) -class EngagementDirectFilter(DojoFilter): - name = CharFilter(lookup_expr='icontains', label='Engagement name contains') - lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") - version = CharFilter(field_name='version', lookup_expr='icontains', label='Engagement version') - test__version = CharFilter(field_name='test__version', lookup_expr='icontains', label='Test version') - - product__name = CharFilter(lookup_expr='icontains', label='Product name contains') - product__prod_type = ModelMultipleChoiceFilter( - queryset=Product_Type.objects.none(), - label="Product Type") +class EngagementDirectFilterHelper(FilterSet): + name = CharFilter(lookup_expr="icontains", label="Engagement name contains") + version = CharFilter(field_name="version", lookup_expr="icontains", label="Engagement version") + test__version = CharFilter(field_name="test__version", lookup_expr="icontains", label="Test version") + product__name = CharFilter(lookup_expr="icontains", label="Product name contains") + status = MultipleChoiceFilter(choices=ENGAGEMENT_STATUS_CHOICES, label="Status") + tag = CharFilter(field_name="tags__name", lookup_expr="icontains", label="Tag name contains") + not_tag = CharFilter(field_name="tags__name", lookup_expr="icontains", label="Not tag name contains", exclude=True) + has_tags = BooleanFilter(field_name="tags", lookup_expr="isnull", exclude=True, label="Has tags") test__engagement__product__lifecycle = MultipleChoiceFilter( - choices=Product.LIFECYCLE_CHOICES, label='Product lifecycle', null_label='Empty') - status = MultipleChoiceFilter(choices=ENGAGEMENT_STATUS_CHOICES, - label="Status") - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') - - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - - has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') - + choices=Product.LIFECYCLE_CHOICES, + label="Product lifecycle", + null_label="Empty") o = OrderingFilter( # tuple-mapping retains order fields=( - ('target_start', 'target_start'), - ('name', 'name'), - ('product__name', 'product__name'), - ('product__prod_type__name', 'product__prod_type__name'), - ('lead__first_name', 'lead__first_name'), + ("target_start", "target_start"), + ("name", "name"), + ("product__name", "product__name"), + ("product__prod_type__name", "product__prod_type__name"), + ("lead__first_name", "lead__first_name"), ), field_labels={ - 'target_start': 'Start date', - 'name': 'Engagement', - 'product__name': 'Product Name', - 'product__prod_type__name': 'Product Type', - 'lead__first_name': 'Lead', + "target_start": "Start date", + "name": "Engagement", + "product__name": "Product Name", + "product__prod_type__name": "Product Type", + "lead__first_name": "Lead", } - ) + +class EngagementDirectFilter(EngagementDirectFilterHelper, DojoFilter): + lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") + product__prod_type = ModelMultipleChoiceFilter( + queryset=Product_Type.objects.none(), + label="Product Type") + tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + queryset=Engagement.tags.tag_model.objects.all().order_by("name")) + not_tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + exclude=True, + queryset=Engagement.tags.tag_model.objects.all().order_by("name")) + def __init__(self, *args, **kwargs): super(EngagementDirectFilter, self).__init__(*args, **kwargs) - self.form.fields['product__prod_type'].queryset = get_authorized_product_types(Permissions.Product_Type_View) - self.form.fields['lead'].queryset = get_authorized_users(Permissions.Product_Type_View) \ + self.form.fields["product__prod_type"].queryset = get_authorized_product_types(Permissions.Product_Type_View) + self.form.fields["lead"].queryset = get_authorized_users(Permissions.Product_Type_View) \ .filter(engagement__lead__isnull=False).distinct() class Meta: model = Engagement - fields = ['product__name', 'product__prod_type'] + fields = ["product__name", "product__prod_type"] + + +class EngagementDirectFilterWithoutObjectLookups(EngagementDirectFilterHelper): + lead = CharFilter( + field_name="lead__username", + lookup_expr="iexact", + label="Lead Username", + help_text="Search for Lead username that are an exact match") + lead_contains = CharFilter( + field_name="lead__username", + lookup_expr="icontains", + label="Lead Username Contains", + help_text="Search for Lead username that contain a given pattern") + product__prod_type__name = CharFilter( + field_name="product__prod_type__name", + lookup_expr="iexact", + label="Product Type Name", + help_text="Search for Product Type names that are an exact match") + product__prod_type__name_contains = CharFilter( + field_name="product__prod_type__name", + lookup_expr="icontains", + label="Product Type Name Contains", + help_text="Search for Product Type names that contain a given pattern") + class Meta: + model = Engagement + fields = ["product__name"] -class EngagementFilter(DojoFilter): - engagement__name = CharFilter(lookup_expr='icontains', label='Engagement name contains') - engagement__lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") - engagement__version = CharFilter(field_name='engagement__version', lookup_expr='icontains', label='Engagement version') - engagement__test__version = CharFilter(field_name='engagement__test__version', lookup_expr='icontains', label='Test version') - name = CharFilter(lookup_expr='icontains', label='Product name contains') - prod_type = ModelMultipleChoiceFilter( - queryset=Product_Type.objects.none(), - label="Product Type") +class EngagementFilterHelper(FilterSet): + name = CharFilter(lookup_expr="icontains", label="Product name contains") + tag = CharFilter(field_name="tags__name", lookup_expr="icontains", label="Tag name contains") + not_tag = CharFilter(field_name="tags__name", lookup_expr="icontains", label="Not tag name contains", exclude=True) + has_tags = BooleanFilter(field_name="tags", lookup_expr="isnull", exclude=True, label="Has tags") + engagement__name = CharFilter(lookup_expr="icontains", label="Engagement name contains") + engagement__version = CharFilter(field_name="engagement__version", lookup_expr="icontains", label="Engagement version") + engagement__test__version = CharFilter(field_name="engagement__test__version", lookup_expr="icontains", label="Test version") engagement__product__lifecycle = MultipleChoiceFilter( - choices=Product.LIFECYCLE_CHOICES, label='Product lifecycle', null_label='Empty') - engagement__status = MultipleChoiceFilter(choices=ENGAGEMENT_STATUS_CHOICES, - label="Status") - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') - - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - - has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') - + choices=Product.LIFECYCLE_CHOICES, + label="Product lifecycle", + null_label="Empty") + engagement__status = MultipleChoiceFilter( + choices=ENGAGEMENT_STATUS_CHOICES, + label="Status") o = OrderingFilter( # tuple-mapping retains order fields=( - ('name', 'name'), - ('prod_type__name', 'prod_type__name'), + ("name", "name"), + ("prod_type__name", "prod_type__name"), ), field_labels={ - 'name': 'Product Name', - 'prod_type__name': 'Product Type', + "name": "Product Name", + "prod_type__name": "Product Type", } - ) + +class EngagementFilter(EngagementFilterHelper, DojoFilter): + engagement__lead = ModelChoiceFilter( + queryset=Dojo_User.objects.none(), + label="Lead") + prod_type = ModelMultipleChoiceFilter( + queryset=Product_Type.objects.none(), + label="Product Type") + tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + queryset=Engagement.tags.tag_model.objects.all().order_by("name")) + not_tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + exclude=True, + queryset=Engagement.tags.tag_model.objects.all().order_by("name")) + def __init__(self, *args, **kwargs): super(EngagementFilter, self).__init__(*args, **kwargs) - self.form.fields['prod_type'].queryset = get_authorized_product_types(Permissions.Product_Type_View) - self.form.fields['engagement__lead'].queryset = get_authorized_users(Permissions.Product_Type_View) \ + self.form.fields["prod_type"].queryset = get_authorized_product_types(Permissions.Product_Type_View) + self.form.fields["engagement__lead"].queryset = get_authorized_users(Permissions.Product_Type_View) \ .filter(engagement__lead__isnull=False).distinct() class Meta: @@ -862,37 +1006,42 @@ class Meta: fields = ['name', 'prod_type'] -class ProductEngagementFilter(DojoFilter): - lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") +class EngagementFilterWithoutObjectLookups(EngagementFilterHelper): + engagement__lead = CharFilter( + field_name="engagement__lead__username", + lookup_expr="iexact", + label="Lead Username", + help_text="Search for Lead username that are an exact match") + engagement__lead_contains = CharFilter( + field_name="engagement__lead__username", + lookup_expr="icontains", + label="Lead Username Contains", + help_text="Search for Lead username that contain a given pattern") + prod_type__name = CharFilter( + field_name="prod_type__name", + lookup_expr="iexact", + label="Product Type Name", + help_text="Search for Product Type names that are an exact match") + prod_type__name_contains = CharFilter( + field_name="prod_type__name", + lookup_expr="icontains", + label="Product Type Name Contains", + help_text="Search for Product Type names that contain a given pattern") + + class Meta: + model = Product + fields = ['name'] + + +class ProductEngagementFilterHelper(FilterSet): version = CharFilter(lookup_expr='icontains', label='Engagement version') test__version = CharFilter(field_name='test__version', lookup_expr='icontains', label='Test version') - name = CharFilter(lookup_expr='icontains') - status = MultipleChoiceFilter(choices=ENGAGEMENT_STATUS_CHOICES, - label="Status") - + status = MultipleChoiceFilter(choices=ENGAGEMENT_STATUS_CHOICES, label="Status") target_start = DateRangeFilter() target_end = DateRangeFilter() - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') - - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - o = OrderingFilter( # tuple-mapping retains order fields=( @@ -906,19 +1055,44 @@ class ProductEngagementFilter(DojoFilter): field_labels={ 'name': 'Engagement Name', } - ) - def __init__(self, *args, **kwargs): - super(ProductEngagementFilter, self).__init__(*args, **kwargs) - self.form.fields['lead'].queryset = get_authorized_users(Permissions.Product_Type_View) \ - .filter(engagement__lead__isnull=False).distinct() - class Meta: model = Product fields = ['name'] +class ProductEngagementFilter(ProductEngagementFilterHelper, DojoFilter): + lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") + tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + queryset=Engagement.tags.tag_model.objects.all().order_by("name")) + not_tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + exclude=True, + queryset=Engagement.tags.tag_model.objects.all().order_by("name")) + + def __init__(self, *args, **kwargs): + super(ProductEngagementFilter, self).__init__(*args, **kwargs) + self.form.fields["lead"].queryset = get_authorized_users( + Permissions.Product_Type_View).filter(engagement__lead__isnull=False).distinct() + + +class ProductEngagementFilterWithoutObjectLookups(ProductEngagementFilterHelper, DojoFilter): + lead = CharFilter( + field_name="lead__username", + lookup_expr="iexact", + label="Lead Username", + help_text="Search for Lead username that are an exact match") + lead_contains = CharFilter( + field_name="lead__username", + lookup_expr="icontains", + label="Lead Username Contains", + help_text="Search for Lead username that contain a given pattern") + + class ApiEngagementFilter(DojoFilter): product__prod_type = NumberInFilter(field_name='product__prod_type', lookup_expr='in') tag = CharFilter(field_name='tags__name', lookup_expr='icontains', help_text='Tag name contains') @@ -963,107 +1137,19 @@ class Meta: 'pen_test', 'status', 'product', 'name', 'version', 'tags'] -class ProductFilter(DojoFilter): +class ProductFilterHelper(FilterSet): name = CharFilter(lookup_expr='icontains', label="Product Name") name_exact = CharFilter(field_name='name', lookup_expr='iexact', label="Exact Product Name") - prod_type = ModelMultipleChoiceFilter( - queryset=Product_Type.objects.none(), - label="Product Type") business_criticality = MultipleChoiceFilter(choices=Product.BUSINESS_CRITICALITY_CHOICES, null_label="Empty") platform = MultipleChoiceFilter(choices=Product.PLATFORM_CHOICES, null_label="Empty") lifecycle = MultipleChoiceFilter(choices=Product.LIFECYCLE_CHOICES, null_label="Empty") origin = MultipleChoiceFilter(choices=Product.ORIGIN_CHOICES, null_label="Empty") external_audience = BooleanFilter(field_name='external_audience') internet_accessible = BooleanFilter(field_name='internet_accessible') - - # not specifying anything for tags will render a multiselect input functioning as OR - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - # tags_and = ModelMultipleChoiceFilter( - # field_name='tags__name', - # to_field_name='name', - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags (AND)', - # conjoined=True, - # ) - - # tags__name = ModelMultipleChoiceFilter( - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label="tags (AND)" - # ) - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label="Tag contains") - - # tags__name = CharFilter( - # lookup_expr='icontains', - # label="Tag contains", - # ) - - # tags__all = ModelMultipleChoiceFilter( - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # field_name='tags__name', - # label="tags (AND)" - # ) - - # not working below - - # tags = ModelMultipleChoiceFilter( - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label="tags_widget", widget=TagWidget, tag_options=tagulous.models.TagOptions( - # force_lowercase=True,) - # ,) - - # tags__name = CharFilter(lookup_expr='icontains') - - # tags__and = ModelMultipleChoiceFilter( - # field_name='tags__name', - # to_field_name='name', - # lookup_expr='in', - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label="tags (AND)" - # ) - - # tags = ModelMultipleChoiceFilter( - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label="tags (OR)" - # ) - - # tags = ModelMultipleChoiceFilter( - # field_name='tags__name', - # to_field_name='name', - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label="tags (OR2)", - # ) - - # tags = ModelMultipleChoiceFilter( - # field_name='tags', - # to_field_name='name', - # # lookup_expr='icontains', # nor working - # # without lookup_expr we get an error: ValueError: invalid literal for int() with base 10: 'magento' - # queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label="tags (OR3)", - # ) - - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - outside_of_sla = ProductSLAFilter(label="Outside of SLA") - has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') - o = OrderingFilter( # tuple-mapping retains order fields=( @@ -1088,24 +1174,61 @@ class ProductFilter(DojoFilter): 'external_audience': 'External Audience ', 'internet_accessible': 'Internet Accessible ', } - ) - # tags = CharFilter(lookup_expr='icontains', label="Tags") + +class ProductFilter(ProductFilterHelper, DojoFilter): + prod_type = ModelMultipleChoiceFilter( + queryset=Product_Type.objects.none(), + label="Product Type") + tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + queryset=Product.tags.tag_model.objects.all().order_by("name")) + not_tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + exclude=True, + queryset=Product.tags.tag_model.objects.all().order_by("name")) def __init__(self, *args, **kwargs): self.user = None - if 'user' in kwargs: - self.user = kwargs.pop('user') - + if "user" in kwargs: + self.user = kwargs.pop("user") super(ProductFilter, self).__init__(*args, **kwargs) + self.form.fields["prod_type"].queryset = get_authorized_product_types(Permissions.Product_Type_View) + + class Meta: + model = Product + fields = [ + "name", "name_exact", "prod_type", "business_criticality", + "platform", "lifecycle", "origin", "external_audience", + "internet_accessible", "tags" + ] + + +class ProductFilterWithoutObjectLookups(ProductFilterHelper): + prod_type__name = CharFilter( + field_name="prod_type__name", + lookup_expr="iexact", + label="Product Type Name", + help_text="Search for Product Type names that are an exact match") + prod_type__name_contains = CharFilter( + field_name="prod_type__name", + lookup_expr="icontains", + label="Product Type Name Contains", + help_text="Search for Product Type names that contain a given pattern") - self.form.fields['prod_type'].queryset = get_authorized_product_types(Permissions.Product_Type_View) + def __init__(self, *args, **kwargs): + kwargs.pop("user", None) + super(ProductFilterWithoutObjectLookups, self).__init__(*args, **kwargs) class Meta: model = Product - fields = ['name', 'name_exact', 'prod_type', 'business_criticality', 'platform', 'lifecycle', 'origin', 'external_audience', - 'internet_accessible', 'tags'] + fields = [ + "name", "name_exact", "business_criticality", "platform", + "lifecycle", "origin", "external_audience", "internet_accessible", + ] class ApiProductFilter(DojoFilter): @@ -1338,156 +1461,91 @@ def filter(self, qs, value): return super().filter(qs, value) -class FindingFilter(FindingFilterWithTags): - # tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') - title = CharFilter(lookup_expr='icontains') +class FindingFilterHelper(FilterSet): + title = CharFilter(lookup_expr="icontains") date = DateRangeFilter() - on = DateFilter(field_name='date', lookup_expr='exact', label='On') - before = DateFilter(field_name='date', lookup_expr='lt', label='Before') - after = DateFilter(field_name='date', lookup_expr='gt', label='After') + on = DateFilter(field_name="date", lookup_expr="exact", label="On") + before = DateFilter(field_name="date", lookup_expr="lt", label="Before") + after = DateFilter(field_name="date", lookup_expr="gt", label="After") last_reviewed = DateRangeFilter() last_status_update = DateRangeFilter() cwe = MultipleChoiceFilter(choices=[]) - vulnerability_id = CharFilter(method=vulnerability_id_filter, label='Vulnerability Id') + vulnerability_id = CharFilter(method=vulnerability_id_filter, label="Vulnerability Id") severity = MultipleChoiceFilter(choices=SEVERITY_CHOICES) - test__test_type = ModelMultipleChoiceFilter( - queryset=Test_Type.objects.all(), label='Test Type') - duplicate = ReportBooleanFilter() is_mitigated = ReportBooleanFilter() mitigated = DateRangeFilter(label="Mitigated Date") - planned_remediation_date = DateRangeOmniFilter() - planned_remediation_version = CharFilter(lookup_expr='icontains', label=_('Planned remediation version')) - - file_path = CharFilter(lookup_expr='icontains') - param = CharFilter(lookup_expr='icontains') - payload = CharFilter(lookup_expr='icontains') - - reporter = ModelMultipleChoiceFilter( - queryset=Dojo_User.objects.none()) - - reviewers = ModelMultipleChoiceFilter( - queryset=Dojo_User.objects.none()) - - test__engagement__product__prod_type = ModelMultipleChoiceFilter( - queryset=Product_Type.objects.none(), - label="Product Type") - - test__engagement__product__lifecycle = MultipleChoiceFilter( - choices=Product.LIFECYCLE_CHOICES, label='Product lifecycle') - - test__engagement__product = ModelMultipleChoiceFilter( - queryset=Product.objects.none(), - label="Product") - test__engagement = ModelMultipleChoiceFilter( - queryset=Engagement.objects.none(), - label="Engagement") - - endpoints__host = CharFilter(lookup_expr='icontains', label="Endpoint Host") - - service = CharFilter(lookup_expr='icontains') - - test = ModelMultipleChoiceFilter( - queryset=Test.objects.none(), - label="Test") - - test__engagement__version = CharFilter(lookup_expr='icontains', label="Engagement Version") - test__version = CharFilter(lookup_expr='icontains', label="Test Version") - - status = FindingStatusFilter(label='Status') - - if is_finding_groups_enabled(): - finding_group = ModelMultipleChoiceFilter( - queryset=Finding_Group.objects.none(), - label="Finding Group") - - has_finding_group = BooleanFilter(field_name='finding_group', - lookup_expr='isnull', - exclude=True, - label='Is Grouped') - - risk_acceptance = ReportRiskAcceptanceFilter( - label="Risk Accepted") - + planned_remediation_version = CharFilter(lookup_expr="icontains", label=_("Planned remediation version")) + file_path = CharFilter(lookup_expr="icontains") + param = CharFilter(lookup_expr="icontains") + payload = CharFilter(lookup_expr="icontains") + test__test_type = ModelMultipleChoiceFilter(queryset=Test_Type.objects.all(), label='Test Type') + endpoints__host = CharFilter(lookup_expr="icontains", label="Endpoint Host") + service = CharFilter(lookup_expr="icontains") + test__engagement__version = CharFilter(lookup_expr="icontains", label="Engagement Version") + test__version = CharFilter(lookup_expr="icontains", label="Test Version") + risk_acceptance = ReportRiskAcceptanceFilter(label="Risk Accepted") effort_for_fixing = MultipleChoiceFilter(choices=EFFORT_FOR_FIXING_CHOICES) - test_import_finding_action__test_import = NumberFilter(widget=HiddenInput()) endpoints = NumberFilter(widget=HiddenInput()) - - if get_system_setting('enable_jira'): - has_jira_issue = BooleanFilter(field_name='jira_issue', - lookup_expr='isnull', - exclude=True, - label='Has JIRA') - jira_creation = DateRangeFilter(field_name='jira_issue__jira_creation', label='JIRA Creation') - jira_change = DateRangeFilter(field_name='jira_issue__jira_change', label='JIRA Updated') - jira_issue__jira_key = CharFilter(field_name='jira_issue__jira_key', lookup_expr='icontains', label="JIRA issue") - - if is_finding_groups_enabled(): - has_jira_group_issue = BooleanFilter(field_name='finding_group__jira_issue', - lookup_expr='isnull', - exclude=True, - label='Has Group JIRA') - - has_component = BooleanFilter(field_name='component_name', - lookup_expr='isnull', - exclude=True, - label='Has Component') - - has_notes = BooleanFilter(field_name='notes', - lookup_expr='isnull', - exclude=True, - label='Has notes') - - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Finding.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_test__tags = ModelMultipleChoiceFilter( - field_name='test__tags__name', - to_field_name='name', + status = FindingStatusFilter(label="Status") + has_component = BooleanFilter( + field_name="component_name", + lookup_expr="isnull", exclude=True, - label='Test without tags', - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_test__engagement__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__tags__name', - to_field_name='name', + label="Has Component") + has_notes = BooleanFilter( + field_name="notes", + lookup_expr="isnull", exclude=True, - label='Engagement without tags', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) + label="Has notes") - not_test__engagement__product__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__product__tags__name', - to_field_name='name', - exclude=True, - label='Product without tags', - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) + if is_finding_groups_enabled(): + has_finding_group = BooleanFilter( + field_name='finding_group', + lookup_expr='isnull', + exclude=True, + label='Is Grouped') + + if get_system_setting("enable_jira"): + has_jira_issue = BooleanFilter( + field_name="jira_issue", + lookup_expr="isnull", + exclude=True, + label="Has JIRA") + jira_creation = DateRangeFilter(field_name="jira_issue__jira_creation", label="JIRA Creation") + jira_change = DateRangeFilter(field_name="jira_issue__jira_change", label="JIRA Updated") + jira_issue__jira_key = CharFilter(field_name="jira_issue__jira_key", lookup_expr="icontains", label="JIRA issue") - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) + if is_finding_groups_enabled(): + has_jira_group_issue = BooleanFilter( + field_name="finding_group__jira_issue", + lookup_expr="isnull", + exclude=True, + label="Has Group JIRA") outside_of_sla = FindingSLAFilter(label="Outside of SLA") - - has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') - - epss_score = PercentageFilter(field_name='epss_score', label='EPSS score') - epss_score_range = PercentageRangeFilter(field_name='epss_score', label='EPSS score range', - help_text='The range of EPSS score percentages to filter on; the left input is a lower bound, the right is an upper bound. Leaving one empty will skip that bound (e.g., leaving the lower bound input empty will filter only on the upper bound -- filtering on "less than or equal").') - - epss_percentile = PercentageFilter(field_name='epss_percentile', label='EPSS percentile') - epss_percentile_range = PercentageRangeFilter(field_name='epss_percentile', label='EPSS percentile range', - help_text='The range of EPSS percentiles to filter on; the left input is a lower bound, the right is an upper bound. Leaving one empty will skip that bound (e.g., leaving the lower bound input empty will filter only on the upper bound -- filtering on "less than or equal").') + has_tags = BooleanFilter(field_name="tags", lookup_expr="isnull", exclude=True, label="Has tags") + epss_score = PercentageFilter(field_name="epss_score", label="EPSS score") + epss_score_range = PercentageRangeFilter( + field_name="epss_score", + label="EPSS score range", + help_text=( + "The range of EPSS score percentages to filter on; the left input is a lower bound, " + "the right is an upper bound. Leaving one empty will skip that bound (e.g., leaving " + "the lower bound input empty will filter only on the upper bound -- filtering on " + "\"less than or equal\")." + )) + epss_percentile = PercentageFilter(field_name="epss_percentile", label="EPSS percentile") + epss_percentile_range = PercentageRangeFilter( + field_name="epss_percentile", + label="EPSS percentile range", + help_text=( + "The range of EPSS percentiles to filter on; the left input is a lower bound, the right " + "is an upper bound. Leaving one empty will skip that bound (e.g., leaving the lower bound " + "input empty will filter only on the upper bound -- filtering on \"less than or equal\")." + )) o = OrderingFilter( # tuple-mapping retains order @@ -1517,9 +1575,94 @@ class FindingFilter(FindingFilterWithTags): } ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def set_date_fields(self, *args: list, **kwargs: dict): + date_input_widget = forms.DateInput(attrs={'class': 'datepicker', 'placeholder': 'YYYY-MM-DD'}, format="%Y-%m-%d") + self.form.fields['on'].widget = date_input_widget + self.form.fields['before'].widget = date_input_widget + self.form.fields['after'].widget = date_input_widget + self.form.fields['cwe'].choices = cwe_options(self.queryset) + + +class FindingFilterWithoutObjectLookups(FindingFilterHelper, FindingTagStringFilter): + reporter = CharFilter( + field_name="reporter__username", + lookup_expr="iexact", + label="Reporter Username", + help_text="Search for Reporter names that are an exact match") + reporter_contains = CharFilter( + field_name="reporter__username", + lookup_expr="icontains", + label="Reporter Username Contains", + help_text="Search for Reporter names that contain a given pattern") + reviewer = CharFilter( + field_name="reviewer__username", + lookup_expr="iexact", + label="Reviewer Username", + help_text="Search for Reviewer names that are an exact match") + reviewer_contains = CharFilter( + field_name="reviewer__username", + lookup_expr="icontains", + label="Reviewer Username Contains", + help_text="Search for Reviewer usernames that contain a given pattern") + test__engagement__product__prod_type__name = CharFilter( + field_name="test__engagement__product__prod_type__name", + lookup_expr="iexact", + label="Product Type Name", + help_text="Search for Product Type names that are an exact match") + test__engagement__product__prod_type__name_contains = CharFilter( + field_name="test__engagement__product__prod_type__name", + lookup_expr="icontains", + label="Product Type Name Contains", + help_text="Search for Product Type names that contain a given pattern") + test__engagement__product__name = CharFilter( + field_name="test__engagement__product__name", + lookup_expr="iexact", + label="Product Name", + help_text="Search for Product names that are an exact match") + test__engagement__product__name_contains = CharFilter( + field_name="test__engagement__product__name", + lookup_expr="icontains", + label="Product name Contains", + help_text="Search for Product Typ names that contain a given pattern") + test__engagement__name = CharFilter( + field_name="test__engagement__name", + lookup_expr="iexact", + label="Engagement Name", + help_text="Search for Engagement names that are an exact match") + test__engagement__name_contains = CharFilter( + field_name="test__engagement__name", + lookup_expr="icontains", + label="Engagement name Contains", + help_text="Search for Engagement names that contain a given pattern") + test__name = CharFilter( + field_name="test__engagement__name", + lookup_expr="iexact", + label="Test Name", + help_text="Search for Test names that are an exact match") + test__name_contains = CharFilter( + field_name="test__engagement__name", + lookup_expr="icontains", + label="Test name Contains", + help_text="Search for Test names that contain a given pattern") + + if is_finding_groups_enabled(): + finding_group__name = CharFilter( + field_name="finding_group__name", + lookup_expr="iexact", + label="Finding Group Name", + help_text="Search for Finding Group names that are an exact match") + finding_group__name_contains = CharFilter( + field_name="finding_group__name", + lookup_expr="icontains", + label="Finding Group Name Contains", + help_text="Search for Finding Group names that contain a given pattern") + class Meta: model = Finding - fields = get_finding_filterset_fields() + fields = get_finding_filterset_fields(filter_string_matching=True) exclude = ['url', 'description', 'mitigation', 'impact', 'endpoints', 'references', @@ -1538,15 +1681,71 @@ def __init__(self, *args, **kwargs): if 'pid' in kwargs: self.pid = kwargs.pop('pid') super().__init__(*args, **kwargs) + # Set some date fields + self.set_date_fields(*args, **kwargs) + # Don't show the product filter on the product finding view + if self.pid: + del self.form.fields['test__engagement__product__name'] + del self.form.fields['test__engagement__product__name_contains'] + del self.form.fields['test__engagement__product__prod_type__name'] + del self.form.fields['test__engagement__product__prod_type__name_contains'] + else: + del self.form.fields['test__name'] + del self.form.fields['test__name_contains'] - self.form.fields['cwe'].choices = cwe_options(self.queryset) - date_input_widget = forms.DateInput(attrs={'class': 'datepicker', 'placeholder': 'YYYY-MM-DD'}, format="%Y-%m-%d") - self.form.fields['on'].widget = date_input_widget - self.form.fields['before'].widget = date_input_widget - self.form.fields['after'].widget = date_input_widget +class FindingFilter(FindingFilterHelper, FindingTagFilter): + reporter = ModelMultipleChoiceFilter(queryset=Dojo_User.objects.none()) + reviewers = ModelMultipleChoiceFilter(queryset=Dojo_User.objects.none()) + test__engagement__product__prod_type = ModelMultipleChoiceFilter( + queryset=Product_Type.objects.none(), + label="Product Type") + test__engagement__product__lifecycle = MultipleChoiceFilter( + choices=Product.LIFECYCLE_CHOICES, + label='Product lifecycle') + test__engagement__product = ModelMultipleChoiceFilter( + queryset=Product.objects.none(), + label="Product") + test__engagement = ModelMultipleChoiceFilter( + queryset=Engagement.objects.none(), + label="Engagement") + test = ModelMultipleChoiceFilter( + queryset=Test.objects.none(), + label="Test") + + if is_finding_groups_enabled(): + finding_group = ModelMultipleChoiceFilter( + queryset=Finding_Group.objects.none(), + label="Finding Group") + + class Meta: + model = Finding + fields = get_finding_filterset_fields() + + exclude = ['url', 'description', 'mitigation', 'impact', + 'endpoints', 'references', + 'thread_id', 'notes', 'scanner_confidence', + 'numerical_severity', 'line', 'duplicate_finding', + 'hash_code', 'reviewers', 'created', 'files', + 'sla_start_date', 'sla_expiration_date', 'cvssv3', + 'severity_justification', 'steps_to_reproduce',] + + def __init__(self, *args, **kwargs): + self.user = None + self.pid = None + if 'user' in kwargs: + self.user = kwargs.pop('user') + + if 'pid' in kwargs: + self.pid = kwargs.pop('pid') + super().__init__(*args, **kwargs) + # Set some date fields + self.set_date_fields(*args, **kwargs) # Don't show the product filter on the product finding view - if self.pid: + self.set_related_object_fields(*args, **kwargs) + + def set_related_object_fields(self, *args: list, **kwargs: dict): + if self.pid is not None: del self.form.fields['test__engagement__product'] del self.form.fields['test__engagement__product__prod_type'] # TODO add authorized check to be sure @@ -1569,18 +1768,13 @@ def __init__(self, *args, **kwargs): class AcceptedFindingFilter(FindingFilter): - risk_acceptance__created__date = \ - DateRangeFilter(label="Acceptance Date") - - risk_acceptance__owner = \ - ModelMultipleChoiceFilter( + risk_acceptance__created__date = DateRangeFilter(label="Acceptance Date") + risk_acceptance__owner = ModelMultipleChoiceFilter( queryset=Dojo_User.objects.none(), label="Risk Acceptance Owner") - risk_acceptance = ModelMultipleChoiceFilter( queryset=Risk_Acceptance.objects.none(), - label="Accepted By" - ) + label="Accepted By") def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1588,24 +1782,35 @@ def __init__(self, *args, **kwargs): self.form.fields['risk_acceptance'].queryset = get_authorized_risk_acceptances(Permissions.Risk_Acceptance) -class SimilarFindingFilter(FindingFilter): +class AcceptedFindingFilterWithoutObjectLookups(FindingFilterWithoutObjectLookups): + risk_acceptance__created__date = DateRangeFilter(label="Acceptance Date") + risk_acceptance__owner = CharFilter( + field_name="risk_acceptance__owner__username", + lookup_expr="iexact", + label="Risk Acceptance Owner Username", + help_text="Search for Risk Acceptance Owners username that are an exact match") + risk_acceptance__owner_contains = CharFilter( + field_name="risk_acceptance__owner__username", + lookup_expr="icontains", + label="Risk Acceptance Owner Username Contains", + help_text="Search for Risk Acceptance Owners username that contain a given pattern") + risk_acceptance__name = CharFilter( + field_name="risk_acceptance__name", + lookup_expr="iexact", + label="Risk Acceptance Name", + help_text="Search for Risk Acceptance name that are an exact match") + risk_acceptance__name_contains = CharFilter( + field_name="risk_acceptance__name", + lookup_expr="icontains", + label="Risk Acceptance Name", + help_text="Search for Risk Acceptance name contain a given pattern") + + +class SimilarFindingHelper(FilterSet): hash_code = MultipleChoiceFilter() vulnerability_ids = CharFilter(method=custom_vulnerability_id_filter, label='Vulnerability Ids') - class Meta(FindingFilter.Meta): - model = Finding - # slightly different fields from FindingFilter, but keep the same ordering for UI consistency - fields = get_finding_filterset_fields(similar=True) - - def __init__(self, data=None, *args, **kwargs): - self.user = None - if 'user' in kwargs: - self.user = kwargs.pop('user') - - self.finding = None - if 'finding' in kwargs: - self.finding = kwargs.pop('finding') - + def update_data(self, data: dict, *args: list, **kwargs: dict): # if filterset is bound, use initial values as defaults # because of this, we can't rely on the self.form.has_changed self.has_changed = True @@ -1624,18 +1829,53 @@ def __init__(self, data=None, *args, **kwargs): self.has_changed = False - super().__init__(data, *args, **kwargs) - + def set_hash_codes(self, *args: list, **kwargs: dict): if self.finding and self.finding.hash_code: self.form.fields['hash_code'] = forms.MultipleChoiceField(choices=[(self.finding.hash_code, self.finding.hash_code[:24] + '...')], required=False, initial=[]) - def filter_queryset(self, *args, **kwargs): + def filter_queryset(self, *args: list, **kwargs: dict): queryset = super().filter_queryset(*args, **kwargs) queryset = get_authorized_findings(Permissions.Finding_View, queryset, self.user) queryset = queryset.exclude(pk=self.finding.pk) return queryset +class SimilarFindingFilter(FindingFilter, SimilarFindingHelper): + class Meta(FindingFilter.Meta): + model = Finding + # slightly different fields from FindingFilter, but keep the same ordering for UI consistency + fields = get_finding_filterset_fields(similar=True) + + def __init__(self, data=None, *args, **kwargs): + self.user = None + if 'user' in kwargs: + self.user = kwargs.pop('user') + self.finding = None + if 'finding' in kwargs: + self.finding = kwargs.pop('finding') + self.update_data(data, *args, **kwargs) + super().__init__(data, *args, **kwargs) + self.set_hash_codes(*args, **kwargs) + + +class SimilarFindingFilterWithoutObjectLookups(FindingFilterWithoutObjectLookups, SimilarFindingHelper): + class Meta(FindingFilterWithoutObjectLookups.Meta): + model = Finding + # slightly different fields from FindingFilter, but keep the same ordering for UI consistency + fields = get_finding_filterset_fields(similar=True, filter_string_matching=True) + + def __init__(self, data=None, *args, **kwargs): + self.user = None + if 'user' in kwargs: + self.user = kwargs.pop('user') + self.finding = None + if 'finding' in kwargs: + self.finding = kwargs.pop('finding') + self.update_data(data, *args, **kwargs) + super().__init__(data, *args, **kwargs) + self.set_hash_codes(*args, **kwargs) + + class TemplateFindingFilter(DojoFilter): title = CharFilter(lookup_expr='icontains') cwe = MultipleChoiceFilter(choices=[]) @@ -1772,30 +2012,11 @@ class Meta(FindingFilter.Meta): fields = get_finding_filterset_fields(metrics=True) -class MetricsEndpointFilter(FilterSet): +class MetricsFindingFilterWithoutObjectLookups(FindingFilterWithoutObjectLookups): start_date = DateFilter(field_name='date', label='Start Date', lookup_expr=('gt')) end_date = DateFilter(field_name='date', label='End Date', lookup_expr=('lt')) date = MetricsDateRangeFilter() - finding__test__engagement__product__prod_type = ModelMultipleChoiceFilter( - queryset=Product_Type.objects.none(), - label="Product Type") - finding__test__engagement = ModelMultipleChoiceFilter( - queryset=Engagement.objects.none(), - label="Engagement") - finding__test__engagement__version = CharFilter(lookup_expr='icontains', label="Engagement Version") - finding__severity = MultipleChoiceFilter(choices=SEVERITY_CHOICES, label="Severity") - - endpoint__host = CharFilter(lookup_expr='icontains', label="Endpoint Host") - finding_title = CharFilter(lookup_expr='icontains', label="Finding Title") - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Endpoint.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') + vulnerability_id = CharFilter(method=vulnerability_id_filter, label='Vulnerability Id') not_tags = ModelMultipleChoiceFilter( field_name='tags__name', @@ -1807,78 +2028,276 @@ class MetricsEndpointFilter(FilterSet): not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', + def __init__(self, *args, **kwargs): + if args[0]: + if args[0].get('start_date', '') != '' or args[0].get('end_date', '') != '': + args[0]._mutable = True + args[0]['date'] = 8 + args[0]._mutable = False + + super().__init__(*args, **kwargs) + + class Meta(FindingFilterWithoutObjectLookups.Meta): + model = Finding + fields = get_finding_filterset_fields(metrics=True, filter_string_matching=True) + + +class MetricsEndpointFilterHelper(FilterSet): + start_date = DateFilter(field_name="date", label="Start Date", lookup_expr=("gt")) + end_date = DateFilter(field_name="date", label="End Date", lookup_expr=("lt")) + date = MetricsDateRangeFilter() + finding__test__engagement__version = CharFilter(lookup_expr="icontains", label="Engagement Version") + finding__severity = MultipleChoiceFilter(choices=SEVERITY_CHOICES, label="Severity") + endpoint__host = CharFilter(lookup_expr="icontains", label="Endpoint Host") + finding_title = CharFilter(lookup_expr="icontains", label="Finding Title") + tag = CharFilter(field_name="tags__name", lookup_expr="icontains", label="Tag name contains") + not_tag = CharFilter(field_name="tags__name", lookup_expr="icontains", label="Not tag name contains", exclude=True) + + +class MetricsEndpointFilter(MetricsEndpointFilterHelper): + finding__test__engagement__product__prod_type = ModelMultipleChoiceFilter( + queryset=Product_Type.objects.none(), + label="Product Type") + finding__test__engagement = ModelMultipleChoiceFilter( + queryset=Engagement.objects.none(), + label="Engagement") + endpoint__tags = ModelMultipleChoiceFilter( + field_name='endpoint__tags__name', + to_field_name='name', + label='Endpoint tags', + queryset=Endpoint.tags.tag_model.objects.all().order_by('name')) + finding__tags = ModelMultipleChoiceFilter( + field_name='finding__tags__name', + to_field_name='name', + label='Finding tags', + queryset=Finding.tags.tag_model.objects.all().order_by('name')) + finding__test__tags = ModelMultipleChoiceFilter( + field_name='finding__test__tags__name', + to_field_name='name', + label='Test tags', + queryset=Test.tags.tag_model.objects.all().order_by('name')) + finding__test__engagement__tags = ModelMultipleChoiceFilter( + field_name='finding__test__engagement__tags__name', + to_field_name='name', + label='Engagement tags', + queryset=Engagement.tags.tag_model.objects.all().order_by('name')) + finding__test__engagement__product__tags = ModelMultipleChoiceFilter( + field_name='finding__test__engagement__product__tags__name', + to_field_name='name', + label='Product tags', + queryset=Product.tags.tag_model.objects.all().order_by('name')) + not_endpoint__tags = ModelMultipleChoiceFilter( + field_name='endpoint__tags__name', to_field_name='name', exclude=True, - queryset=Finding.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_test__tags = ModelMultipleChoiceFilter( - field_name='test__tags__name', + label='Endpoint without tags', + queryset=Endpoint.tags.tag_model.objects.all().order_by('name')) + not_finding__tags = ModelMultipleChoiceFilter( + field_name='finding__tags__name', + to_field_name='name', + exclude=True, + label='Finding without tags', + queryset=Finding.tags.tag_model.objects.all().order_by('name')) + not_finding__test__tags = ModelMultipleChoiceFilter( + field_name='finding__test__tags__name', to_field_name='name', exclude=True, label='Test without tags', - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_test__engagement__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__tags__name', + queryset=Test.tags.tag_model.objects.all().order_by('name')) + not_finding__test__engagement__tags = ModelMultipleChoiceFilter( + field_name='finding__test__engagement__tags__name', to_field_name='name', exclude=True, label='Engagement without tags', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_test__engagement__product__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__product__tags__name', + queryset=Engagement.tags.tag_model.objects.all().order_by('name')) + not_finding__test__engagement__product__tags = ModelMultipleChoiceFilter( + field_name='finding__test__engagement__product__tags__name', to_field_name='name', exclude=True, label='Product without tags', - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') + queryset=Product.tags.tag_model.objects.all().order_by('name')) def __init__(self, *args, **kwargs): if args[0]: - if args[0].get('start_date', '') != '' or args[0].get('end_date', '') != '': + if args[0].get("start_date", "") != "" or args[0].get("end_date", "") != "": args[0]._mutable = True - args[0]['date'] = 8 + args[0]["date"] = 8 args[0]._mutable = False self.pid = None - if 'pid' in kwargs: - self.pid = kwargs.pop('pid') + if "pid" in kwargs: + self.pid = kwargs.pop("pid") super().__init__(*args, **kwargs) if self.pid: - del self.form.fields['finding__test__engagement__product__prod_type'] - self.form.fields['finding__test__engagement'].queryset = Engagement.objects.filter( + del self.form.fields["finding__test__engagement__product__prod_type"] + self.form.fields["finding__test__engagement"].queryset = Engagement.objects.filter( product_id=self.pid ).all() else: - self.form.fields['finding__test__engagement'].queryset = get_authorized_engagements(Permissions.Engagement_View).order_by('name') + self.form.fields["finding__test__engagement"].queryset = get_authorized_engagements(Permissions.Engagement_View).order_by("name") - if 'finding__test__engagement__product__prod_type' in self.form.fields: + if "finding__test__engagement__product__prod_type" in self.form.fields: self.form.fields[ - 'finding__test__engagement__product__prod_type'].queryset = get_authorized_product_types(Permissions.Product_Type_View) + "finding__test__engagement__product__prod_type"].queryset = get_authorized_product_types(Permissions.Product_Type_View) class Meta: model = Endpoint_Status - exclude = ['last_modified', 'endpoint', 'finding'] + exclude = ["last_modified", "endpoint", "finding"] + + +class MetricsEndpointFilterWithoutObjectLookups(MetricsEndpointFilterHelper, FindingTagStringFilter): + finding__test__engagement__product__prod_type = CharFilter( + field_name="finding__test__engagement__product__prod_type", + lookup_expr="iexact", + label="Product Type Name", + help_text="Search for Product Type names that are an exact match") + finding__test__engagement__product__prod_type_contains = CharFilter( + field_name="finding__test__engagement__product__prod_type", + lookup_expr="icontains", + label="Product Type Name Contains", + help_text="Search for Product Type names that contain a given pattern") + finding__test__engagement = CharFilter( + field_name="finding__test__engagement", + lookup_expr="iexact", + label="Engagement Name", + help_text="Search for Engagement names that are an exact match") + finding__test__engagement_contains = CharFilter( + field_name="finding__test__engagement", + lookup_expr="icontains", + label="Engagement Name Contains", + help_text="Search for Engagement names that contain a given pattern") + endpoint__tags_contains = CharFilter( + label="Endpoint Tag Contains", + field_name="endpoint__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Endpoint that contain a given pattern") + endpoint__tags = CharFilter( + label="Endpoint Tag", + field_name="endpoint__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Endpoint that are an exact match") + finding__tags_contains = CharFilter( + label="Finding Tag Contains", + field_name="finding__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + finding__tags = CharFilter( + label="Finding Tag", + field_name="finding__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + finding__test__tags_contains = CharFilter( + label="Test Tag Contains", + field_name="finding__test__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + finding__test__tags = CharFilter( + label="Test Tag", + field_name="finding__test__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + finding__test__engagement__tags_contains = CharFilter( + label="Engagement Tag Contains", + field_name="finding__test__engagement__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + finding__test__engagement__tags = CharFilter( + label="Engagement Tag", + field_name="finding__test__engagement__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + finding__test__engagement__product__tags_contains = CharFilter( + label="Product Tag Contains", + field_name="finding__test__engagement__product__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + finding__test__engagement__product__tags = CharFilter( + label="Product Tag", + field_name="finding__test__engagement__product__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + + not_endpoint__tags_contains = CharFilter( + label="Endpoint Tag Does Not Contain", + field_name="endpoint__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Endpoint that contain a given pattern, and exclude them", + exclude=True) + not_endpoint__tags = CharFilter( + label="Not Endpoint Tag", + field_name="endpoint__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Endpoint that are an exact match, and exclude them", + exclude=True) + not_finding__tags_contains = CharFilter( + label="Finding Tag Does Not Contain", + field_name="finding__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern, and exclude them", + exclude=True) + not_finding__tags = CharFilter( + label="Not Finding Tag", + field_name="finding__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match, and exclude them", + exclude=True) + not_finding__test__tags_contains = CharFilter( + label="Test Tag Does Not Contain", + field_name="finding__test__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Test that contain a given pattern, and exclude them", + exclude=True) + not_finding__test__tags = CharFilter( + label="Not Test Tag", + field_name="finding__test__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Test that are an exact match, and exclude them", + exclude=True) + not_finding__test__engagement__tags_contains = CharFilter( + label="Engagement Tag Does Not Contain", + field_name="finding__test__engagement__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Engagement that contain a given pattern, and exclude them", + exclude=True) + not_finding__test__engagement__tags = CharFilter( + label="Not Engagement Tag", + field_name="finding__test__engagement__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Engagement that are an exact match, and exclude them", + exclude=True) + not_finding__test__engagement__product__tags_contains = CharFilter( + label="Product Tag Does Not Contain", + field_name="finding__test__engagement__product__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Product that contain a given pattern, and exclude them", + exclude=True) + not_finding__test__engagement__product__tags = CharFilter( + label="Not Product Tag", + field_name="finding__test__engagement__product__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Product that are an exact match, and exclude them", + exclude=True) + def __init__(self, *args, **kwargs): + if args[0]: + if args[0].get("start_date", "") != "" or args[0].get("end_date", "") != "": + args[0]._mutable = True + args[0]["date"] = 8 + args[0]._mutable = False + self.pid = None + if "pid" in kwargs: + self.pid = kwargs.pop("pid") + super().__init__(*args, **kwargs) + if self.pid: + del self.form.fields["finding__test__engagement__product__prod_type"] -class EndpointFilter(DojoFilter): - product = ModelMultipleChoiceFilter( - queryset=Product.objects.none(), - label="Product") + class Meta: + model = Endpoint_Status + exclude = ["last_modified", "endpoint", "finding"] + + +class EndpointFilterHelper(FilterSet): protocol = CharFilter(lookup_expr='icontains') userinfo = CharFilter(lookup_expr='icontains') host = CharFilter(lookup_expr='icontains') @@ -1886,65 +2305,77 @@ class EndpointFilter(DojoFilter): path = CharFilter(lookup_expr='icontains') query = CharFilter(lookup_expr='icontains') fragment = CharFilter(lookup_expr='icontains') - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Endpoint.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below + tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') + not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) + has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') + o = OrderingFilter( + # tuple-mapping retains order + fields=( + ('product', 'product'), + ('host', 'host'), + ), ) - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') +class EndpointFilter(EndpointFilterHelper, DojoFilter): + product = ModelMultipleChoiceFilter( + queryset=Product.objects.none(), + label="Product") tags = ModelMultipleChoiceFilter( field_name='tags__name', to_field_name='name', - queryset=Finding.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - test__tags = ModelMultipleChoiceFilter( - field_name='test__tags__name', + label="Endpoint Tags", + queryset=Endpoint.tags.tag_model.objects.all().order_by('name')) + findings__tags = ModelMultipleChoiceFilter( + field_name='findings__tags__name', to_field_name='name', - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - test__engagement__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__tags__name', + label="Finding Tags", + queryset=Finding.tags.tag_model.objects.all().order_by('name')) + findings__test__tags = ModelMultipleChoiceFilter( + field_name='findings__test__tags__name', to_field_name='name', - queryset=Engagement.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - test__engagement__product__tags = ModelMultipleChoiceFilter( - field_name='test__engagement__product__tags__name', + label="Test Tags", + queryset=Test.tags.tag_model.objects.all().order_by('name')) + findings__test__engagement__tags = ModelMultipleChoiceFilter( + field_name='findings__test__engagement__tags__name', to_field_name='name', - queryset=Product.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') - + label="Engagement Tags", + queryset=Engagement.tags.tag_model.objects.all().order_by('name')) + findings__test__engagement__product__tags = ModelMultipleChoiceFilter( + field_name='findings__test__engagement__product__tags__name', + to_field_name='name', + label="Product Tags", + queryset=Product.tags.tag_model.objects.all().order_by('name')) not_tags = ModelMultipleChoiceFilter( field_name='tags__name', to_field_name='name', + label="Not Endpoint Tags", exclude=True, - queryset=Finding.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - - has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') - - o = OrderingFilter( - # tuple-mapping retains order - fields=( - ('product', 'product'), - ('host', 'host'), - ), - ) + queryset=Endpoint.tags.tag_model.objects.all().order_by('name')) + not_findings__tags = ModelMultipleChoiceFilter( + field_name='findings__tags__name', + to_field_name='name', + label="Not Finding Tags", + exclude=True, + queryset=Finding.tags.tag_model.objects.all().order_by('name')) + not_findings__test__tags = ModelMultipleChoiceFilter( + field_name='findings__test__tags__name', + to_field_name='name', + label="Not Test Tags", + exclude=True, + queryset=Test.tags.tag_model.objects.all().order_by('name')) + not_findings__test__engagement__tags = ModelMultipleChoiceFilter( + field_name='findings__test__engagement__tags__name', + to_field_name='name', + label="Not Engagement Tags", + exclude=True, + queryset=Engagement.tags.tag_model.objects.all().order_by('name')) + not_findings__test__engagement__product__tags = ModelMultipleChoiceFilter( + field_name='findings__test__engagement__product__tags__name', + to_field_name='name', + label="Not Product Tags", + exclude=True, + queryset=Product.tags.tag_model.objects.all().order_by('name')) def __init__(self, *args, **kwargs): self.user = None @@ -1960,7 +2391,147 @@ def qs(self): class Meta: model = Endpoint - exclude = ['findings'] + exclude = ["findings", "inherited_tags"] + + +class EndpointFilterWithoutObjectLookups(EndpointFilterHelper): + product__name = CharFilter( + field_name="product__name", + lookup_expr="iexact", + label="Product Name", + help_text="Search for Product names that are an exact match") + product__name_contains = CharFilter( + field_name="product__name", + lookup_expr="icontains", + label="Product Name Contains", + help_text="Search for Product names that contain a given pattern") + + tags_contains = CharFilter( + label="Endpoint Tag Contains", + field_name="tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Endpoint that contain a given pattern") + tags = CharFilter( + label="Endpoint Tag", + field_name="tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Endpoint that are an exact match") + findings__tags_contains = CharFilter( + label="Finding Tag Contains", + field_name="findings__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + findings__tags = CharFilter( + label="Finding Tag", + field_name="findings__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + findings__test__tags_contains = CharFilter( + label="Test Tag Contains", + field_name="findings__test__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + findings__test__tags = CharFilter( + label="Test Tag", + field_name="findings__test__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + findings__test__engagement__tags_contains = CharFilter( + label="Engagement Tag Contains", + field_name="findings__test__engagement__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + findings__test__engagement__tags = CharFilter( + label="Engagement Tag", + field_name="findings__test__engagement__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + findings__test__engagement__product__tags_contains = CharFilter( + label="Product Tag Contains", + field_name="findings__test__engagement__product__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern") + findings__test__engagement__product__tags = CharFilter( + label="Product Tag", + field_name="findings__test__engagement__product__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match") + + not_tags_contains = CharFilter( + label="Endpoint Tag Does Not Contain", + field_name="tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Endpoint that contain a given pattern, and exclude them", + exclude=True) + not_tags = CharFilter( + label="Not Endpoint Tag", + field_name="tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Endpoint that are an exact match, and exclude them", + exclude=True) + not_findings__tags_contains = CharFilter( + label="Finding Tag Does Not Contain", + field_name="findings__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Finding that contain a given pattern, and exclude them", + exclude=True) + not_findings__tags = CharFilter( + label="Not Finding Tag", + field_name="findings__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Finding that are an exact match, and exclude them", + exclude=True) + not_findings__test__tags_contains = CharFilter( + label="Test Tag Does Not Contain", + field_name="findings__test__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Test that contain a given pattern, and exclude them", + exclude=True) + not_findings__test__tags = CharFilter( + label="Not Test Tag", + field_name="findings__test__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Test that are an exact match, and exclude them", + exclude=True) + not_findings__test__engagement__tags_contains = CharFilter( + label="Engagement Tag Does Not Contain", + field_name="findings__test__engagement__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Engagement that contain a given pattern, and exclude them", + exclude=True) + not_findings__test__engagement__tags = CharFilter( + label="Not Engagement Tag", + field_name="findings__test__engagement__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Engagement that are an exact match, and exclude them", + exclude=True) + not_findings__test__engagement__product__tags_contains = CharFilter( + label="Product Tag Does Not Contain", + field_name="findings__test__engagement__product__tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Product that contain a given pattern, and exclude them", + exclude=True) + not_findings__test__engagement__product__tags = CharFilter( + label="Not Product Tag", + field_name="findings__test__engagement__product__tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Product that are an exact match, and exclude them", + exclude=True) + + def __init__(self, *args, **kwargs): + self.user = None + if 'user' in kwargs: + self.user = kwargs.pop('user') + super(EndpointFilterWithoutObjectLookups, self).__init__(*args, **kwargs) + + @property + def qs(self): + parent = super(EndpointFilterWithoutObjectLookups, self).qs + return get_authorized_endpoints(Permissions.Endpoint_View, parent) + + class Meta: + model = Endpoint + exclude = ["findings", "inherited_tags", "product"] class ApiEndpointFilter(DojoFilter): @@ -2004,37 +2575,15 @@ class Meta: ] -class EngagementTestFilter(DojoFilter): - lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") +class EngagementTestFilterHelper(FilterSet): version = CharFilter(lookup_expr='icontains', label='Version') - if settings.TRACK_IMPORT_HISTORY: test_import__version = CharFilter(field_name='test_import__version', lookup_expr='icontains', label='Reimported Version') - target_start = DateRangeFilter() target_end = DateRangeFilter() - - tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Tag name contains') - - not_tags = ModelMultipleChoiceFilter( - field_name='tags__name', - to_field_name='name', - exclude=True, - queryset=Test.tags.tag_model.objects.all().order_by('name'), - # label='tags', # doesn't work with tagulous, need to set in __init__ below - ) - not_tag = CharFilter(field_name='tags__name', lookup_expr='icontains', label='Not tag name contains', exclude=True) - has_tags = BooleanFilter(field_name='tags', lookup_expr='isnull', exclude=True, label='Has tags') - o = OrderingFilter( # tuple-mapping retains order fields=( @@ -2048,14 +2597,31 @@ class EngagementTestFilter(DojoFilter): field_labels={ 'name': 'Test Name', } - ) + +class EngagementTestFilter(EngagementTestFilterHelper, DojoFilter): + lead = ModelChoiceFilter(queryset=Dojo_User.objects.none(), label="Lead") + api_scan_configuration = ModelChoiceFilter( + queryset=Product_API_Scan_Configuration.objects.none(), + label="API Scan Configuration") + tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + queryset=Test.tags.tag_model.objects.all().order_by("name")) + not_tags = ModelMultipleChoiceFilter( + field_name="tags__name", + to_field_name="name", + exclude=True, + queryset=Test.tags.tag_model.objects.all().order_by("name")) + class Meta: model = Test - fields = ['id', 'title', 'test_type', 'target_start', - 'target_end', 'percent_complete', - 'version', 'api_scan_configuration'] + fields = [ + "title", "test_type", "target_start", + "target_end", "percent_complete", + "version", "api_scan_configuration", + ] def __init__(self, *args, **kwargs): self.engagement = kwargs.pop('engagement') @@ -2066,6 +2632,63 @@ def __init__(self, *args, **kwargs): .filter(test__lead__isnull=False).distinct() +class EngagementTestFilterWithoutObjectLookups(EngagementTestFilterHelper): + lead = CharFilter( + field_name="lead__username", + lookup_expr="iexact", + label="Lead Username", + help_text="Search for Lead username that are an exact match") + lead_contains = CharFilter( + field_name="lead__username", + lookup_expr="icontains", + label="Lead Username Contains", + help_text="Search for Lead username that contain a given pattern") + api_scan_configuration__tool_configuration__name = CharFilter( + field_name="api_scan_configuration__tool_configuration__name", + lookup_expr="iexact", + label="API Scan Configuration Name", + help_text="Search for Lead username that are an exact match") + api_scan_configuration__tool_configuration__name_contains = CharFilter( + field_name="api_scan_configuration__tool_configuration__name", + lookup_expr="icontains", + label="API Scan Configuration Name Contains", + help_text="Search for Lead username that contain a given pattern") + tags_contains = CharFilter( + label="Test Tag Contains", + field_name="tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Test that contain a given pattern") + tags = CharFilter( + label="Test Tag", + field_name="tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Test that are an exact match") + not_tags_contains = CharFilter( + label="Test Tag Does Not Contain", + field_name="tags__name", + lookup_expr="icontains", + help_text="Search for tags on a Test that contain a given pattern, and exclude them", + exclude=True) + not_tags = CharFilter( + label="Not Test Tag", + field_name="tags__name", + lookup_expr="iexact", + help_text="Search for tags on a Test that are an exact match, and exclude them", + exclude=True) + + class Meta: + model = Test + fields = [ + "title", "test_type", "target_start", + "target_end", "percent_complete", "version", + ] + + def __init__(self, *args, **kwargs): + self.engagement = kwargs.pop('engagement') + super(EngagementTestFilterWithoutObjectLookups, self).__init__(*args, **kwargs) + self.form.fields['test_type'].queryset = Test_Type.objects.filter(test__engagement=self.engagement).distinct().order_by('name') + + class ApiTestFilter(DojoFilter): tag = CharFilter(field_name='tags__name', lookup_expr='icontains', help_text='Tag name contains') tags = CharFieldInFilter(field_name='tags__name', lookup_expr='in', @@ -2175,7 +2798,7 @@ class Meta: exclude = ['product'] -class ReportFindingFilter(FindingFilterWithTags): +class ReportFindingFilter(FindingTagFilter): title = CharFilter(lookup_expr='icontains', label='Name') test__engagement__product = ModelMultipleChoiceFilter( queryset=Product.objects.none(), label="Product") diff --git a/dojo/finding/views.py b/dojo/finding/views.py index 08ce201491d..9faef2cffc5 100644 --- a/dojo/finding/views.py +++ b/dojo/finding/views.py @@ -39,8 +39,11 @@ from dojo.filters import ( TemplateFindingFilter, SimilarFindingFilter, + SimilarFindingFilterWithoutObjectLookups, FindingFilter, + FindingFilterWithoutObjectLookups, AcceptedFindingFilter, + AcceptedFindingFilterWithoutObjectLookups, TestImportFindingActionFilter, TestImportFilter, ) @@ -345,10 +348,13 @@ def filter_findings_by_form(self, request: HttpRequest, findings: QuerySet[Findi "pid": self.get_product_id(), } + filter_string_matching = get_system_setting("filter_string_matching", False) + finding_filter_class = FindingFilterWithoutObjectLookups if filter_string_matching else FindingFilter + accepted_finding_filter_class = AcceptedFindingFilterWithoutObjectLookups if filter_string_matching else AcceptedFindingFilter return ( - AcceptedFindingFilter(*args, **kwargs) + accepted_finding_filter_class(*args, **kwargs) if self.get_filter_name() == "Accepted" - else FindingFilter(*args, **kwargs) + else finding_filter_class(*args, **kwargs) ) def get_filtered_findings(self): @@ -596,6 +602,14 @@ def get_test_import_data(self, request: HttpRequest, finding: Finding): } def get_similar_findings(self, request: HttpRequest, finding: Finding): + similar_findings_enabled = get_system_setting("enable_similar_findings", True) + if similar_findings_enabled is False: + return { + "similar_findings_enabled": similar_findings_enabled, + "duplicate_cluster": duplicate_cluster(request, finding), + "similar_findings": None, + "similar_findings_filter": None, + } # add related actions for non-similar and non-duplicate cluster members finding.related_actions = calculate_possible_related_actions_for_similar_finding( request, finding, finding @@ -606,7 +620,9 @@ def get_similar_findings(self, request: HttpRequest, finding: Finding): request, finding, finding.duplicate_finding ) ) - similar_findings_filter = SimilarFindingFilter( + filter_string_matching = get_system_setting("filter_string_matching", False) + finding_filter_class = SimilarFindingFilterWithoutObjectLookups if filter_string_matching else SimilarFindingFilter + similar_findings_filter = finding_filter_class( request.GET, queryset=get_authorized_findings(Permissions.Finding_View), user=request.user, @@ -630,6 +646,7 @@ def get_similar_findings(self, request: HttpRequest, finding: Finding): ) return { + "similar_findings_enabled": similar_findings_enabled, "duplicate_cluster": duplicate_cluster(request, finding), "similar_findings": similar_findings, "similar_findings_filter": similar_findings_filter, diff --git a/dojo/finding_group/views.py b/dojo/finding_group/views.py index e6f92a71d27..2a5df5a775b 100644 --- a/dojo/finding_group/views.py +++ b/dojo/finding_group/views.py @@ -1,8 +1,8 @@ -from dojo.utils import Product_Tab, add_breadcrumb, get_words_for_field, get_page_items +from dojo.utils import Product_Tab, add_breadcrumb, get_words_for_field, get_page_items, get_system_setting from dojo.forms import DeleteFindingGroupForm, EditFindingGroupForm, FindingBulkUpdateForm from dojo.notifications.helper import create_notification from dojo.finding.views import prefetch_for_findings -from dojo.filters import FindingFilter +from dojo.filters import FindingFilter, FindingFilterWithoutObjectLookups from django.contrib import messages from django.contrib.admin.utils import NestedObjects from django.db.utils import DEFAULT_DB_ALIAS @@ -25,13 +25,14 @@ def view_finding_group(request, fgid): finding_group = get_object_or_404(Finding_Group, pk=fgid) findings = finding_group.findings.all() edit_finding_group_form = EditFindingGroupForm(instance=finding_group) + filter_string_matching = get_system_setting("filter_string_matching", False) + finding_filter_class = FindingFilterWithoutObjectLookups if filter_string_matching else FindingFilter show_product_column = True custom_breadcrumb = None product_tab = None jira_project = None github_config = None - if finding_group.test.engagement.product.id: pid = finding_group.test.engagement.product.id product = get_object_or_404(Product, id=pid) @@ -39,7 +40,7 @@ def view_finding_group(request, fgid): product_tab = Product_Tab(product, title="Findings", tab="findings") jira_project = jira_helper.get_jira_project(product) github_config = GITHUB_PKey.objects.filter(product=pid).first() - findings_filter = FindingFilter(request.GET, findings, user=request.user, pid=pid) + findings_filter = finding_filter_class(request.GET, findings, user=request.user, pid=pid) elif finding_group.test.engagement.id: eid = finding_group.test.engagement.id engagement = get_object_or_404(Engagement, id=eid) @@ -47,7 +48,7 @@ def view_finding_group(request, fgid): product_tab = Product_Tab(engagement.product, title=engagement.name, tab="engagements") jira_project = jira_helper.get_jira_project(engagement) github_config = GITHUB_PKey.objects.filter(product__engagement=eid).first() - findings_filter = FindingFilter(request.GET, findings, user=request.user, eid=eid) + findings_filter = finding_filter_class(request.GET, findings, user=request.user, eid=eid) title_words = get_words_for_field(Finding, 'title') component_words = get_words_for_field(Finding, 'component_name') diff --git a/dojo/jira_link/views.py b/dojo/jira_link/views.py index 2f3ff1d3c30..e652b8703e4 100644 --- a/dojo/jira_link/views.py +++ b/dojo/jira_link/views.py @@ -7,7 +7,7 @@ from django.contrib.admin.utils import NestedObjects from django.urls import reverse from django.db import DEFAULT_DB_ALIAS -from django.http import HttpResponseRedirect, HttpResponse, Http404, HttpResponseBadRequest +from django.http import HttpResponseRedirect, HttpResponse, Http404 from django.shortcuts import render, get_object_or_404 from django.utils import timezone from django.utils.dateparse import parse_datetime @@ -15,8 +15,8 @@ from django.core.exceptions import PermissionDenied # Local application/library imports from dojo.forms import JIRAForm, DeleteJIRAInstanceForm, ExpressJIRAForm -from dojo.models import User, JIRA_Instance, JIRA_Issue, Notes -from dojo.utils import add_breadcrumb, add_error_message_to_response, get_system_setting +from dojo.models import System_Settings, User, JIRA_Instance, JIRA_Issue, Notes +from dojo.utils import add_breadcrumb, add_error_message_to_response from dojo.notifications.helper import create_notification from django.views.decorators.http import require_POST import dojo.jira_link.helper as jira_helper @@ -26,114 +26,140 @@ logger = logging.getLogger(__name__) -# for examples of incoming json, see the unit tests for the webhook: https://github.com/DefectDojo/django-DefectDojo/blob/master/unittests/test_jira_webhook.py -# or the officials docs (which are not always clear): https://developer.atlassian.com/server/jira/platform/webhooks/ +def webhook_responser_handler( + log_level: str, + message: str, +) -> HttpResponse: + # These represent an error and will be sent to the debugger + # for development purposes + if log_level == "info": + logger.info(message) + # These are more common in misconfigurations and have a better + # chance of being seen by a user + elif log_level == "debug": + logger.debug(message) + # Return the response with the code + return HttpResponse(message, status=200) + + @csrf_exempt @require_POST def webhook(request, secret=None): - if not get_system_setting('enable_jira'): - logger.debug('ignoring incoming webhook as JIRA is disabled.') - raise Http404('JIRA disabled') - elif not get_system_setting('enable_jira_web_hook'): - logger.debug('ignoring incoming webhook as JIRA Webhook is disabled.') - raise Http404('JIRA Webhook disabled') - elif not get_system_setting('disable_jira_webhook_secret'): - if not get_system_setting('jira_webhook_secret'): - logger.warning('ignoring incoming webhook as JIRA Webhook secret is empty in Defect Dojo system settings.') - raise PermissionDenied('JIRA Webhook secret cannot be empty') - if secret != get_system_setting('jira_webhook_secret'): - logger.warning('invalid secret provided to JIRA Webhook') - raise PermissionDenied('invalid or no secret provided to JIRA Webhook') + """ + for examples of incoming json, see the unit tests for the webhook: + https://github.com/DefectDojo/django-DefectDojo/blob/master/unittests/test_jira_webhook.py + or the officials docs (which are not always clear): + https://developer.atlassian.com/server/jira/platform/webhooks/ + All responses here will return a 201 so that we may have control over the + logging level + """ + # Make sure the request is a POST, otherwise, we reject + if request.method != "POST": + return webhook_responser_handler("debug", "Only POST requests are supported") + # Determine if th webhook is in use or not + system_settings = System_Settings.objects.get() + # If the jira integration is not enabled, then return a 404 + if not system_settings.enable_jira: + return webhook_responser_handler("info", "Ignoring incoming webhook as JIRA is disabled.") + # If the webhook is not enabled, then return a 404 + elif not system_settings.enable_jira_web_hook: + return webhook_responser_handler("info", "Ignoring incoming webhook as JIRA Webhook is disabled.") + # Determine if the request should be "authenticated" + elif not system_settings.disable_jira_webhook_secret: + # Make sure there is a value for the webhook secret before making a comparison + if not system_settings.jira_webhook_secret: + return webhook_responser_handler("info", "Ignoring incoming webhook as JIRA Webhook secret is empty in Defect Dojo system settings.") + # Make sure the secret supplied in the path of the webhook request matches the + # secret supplied in the system settings + if secret != system_settings.jira_webhook_secret: + return webhook_responser_handler("info", "Invalid or no secret provided to JIRA Webhook") # if webhook secret is disabled in system_settings, we ignore the incoming secret, even if it doesn't match - # example json bodies at the end of this file - - if request.content_type != 'application/json': - return HttpResponseBadRequest("only application/json supported") - - if request.method == 'POST': - try: - parsed = json.loads(request.body.decode('utf-8')) - if parsed.get('webhookEvent') == 'jira:issue_updated': - # xml examples at the end of file - jid = parsed['issue']['id'] - jissue = get_object_or_404(JIRA_Issue, jira_id=jid) - - findings = None - if jissue.finding: - logging.info(f"Received issue update for {jissue.jira_key} for finding {jissue.finding.id}") - findings = [jissue.finding] - elif jissue.finding_group: - logging.info(f"Received issue update for {jissue.jira_key} for finding group {jissue.finding_group}") - findings = jissue.finding_group.findings.all() - elif jissue.engagement: - # if parsed['issue']['fields']['resolution'] != None: - # eng.active = False - # eng.status = 'Completed' - # eng.save() - return HttpResponse('Update for engagement ignored') - else: - logging.info(f"Received issue update for {jissue.jira_key} for unknown object") - raise Http404(f'No finding, finding_group or engagement found for JIRA issue {jissue.jira_key}') - - assignee = parsed['issue']['fields'].get('assignee') - assignee_name = 'Jira User' - if assignee is not None: - # First look for the 'name' field. If not present, try 'displayName'. Else put None - assignee_name = assignee.get('name', assignee.get('displayName')) - - resolution = parsed['issue']['fields']['resolution'] - - # "resolution":{ - # "self":"http://www.testjira.com/rest/api/2/resolution/11", - # "id":"11", - # "description":"Cancelled by the customer.", - # "name":"Cancelled" - # }, - - # or - # "resolution": null - - # or - # "resolution": "None" - - resolution = resolution if resolution and resolution != "None" else None - resolution_id = resolution['id'] if resolution else None - resolution_name = resolution['name'] if resolution else None - jira_now = parse_datetime(parsed['issue']['fields']['updated']) - - if findings: - for finding in findings: - jira_helper.process_resolution_from_jira(finding, resolution_id, resolution_name, assignee_name, jira_now, jissue) - # Check for any comment that could have come along with the resolution - if (error_response := check_for_and_create_comment(parsed)) is not None: - return error_response - - if parsed.get('webhookEvent') == 'comment_created': - if (error_response := check_for_and_create_comment(parsed)) is not None: - return error_response - - if parsed.get('webhookEvent') not in ['comment_created', 'jira:issue_updated']: - logger.info(f"Unrecognized JIRA webhook event received: {parsed.get('webhookEvent')}") - - except Exception as e: - if isinstance(e, Http404): - logger.warning('404 error processing JIRA webhook') - logger.warning(str(e)) - else: - logger.exception(e) - + if request.content_type != "application/json": + return webhook_responser_handler("debug", "only application/json supported") + # Time to process the request + try: + parsed = json.loads(request.body.decode("utf-8")) + # Check if the events supplied are supported + if parsed.get('webhookEvent') not in ['comment_created', 'jira:issue_updated']: + return webhook_responser_handler("info", f"Unrecognized JIRA webhook event received: {parsed.get('webhookEvent')}") + + if parsed.get('webhookEvent') == 'jira:issue_updated': + # xml examples at the end of file + jid = parsed['issue']['id'] + # This may raise a 404, but it will be handled in the exception response try: - logger.debug('jira_webhook_body_parsed:') - logger.debug(json.dumps(parsed, indent=4)) - except Exception: - logger.debug('jira_webhook_body:') - logger.debug(request.body.decode('utf-8')) + jissue = JIRA_Issue.objects.get(jira_id=jid) + except JIRA_Instance.DoesNotExist: + return webhook_responser_handler("info", f"JIRA issue {jid} is not linked to a DefectDojo Finding") + findings = None + # Determine what type of object we will be working with + if jissue.finding: + logging.debug(f"Received issue update for {jissue.jira_key} for finding {jissue.finding.id}") + findings = [jissue.finding] + elif jissue.finding_group: + logging.debug(f"Received issue update for {jissue.jira_key} for finding group {jissue.finding_group}") + findings = jissue.finding_group.findings.all() + elif jissue.engagement: + return webhook_responser_handler("debug", "Update for engagement ignored") + else: + return webhook_responser_handler("info", f"Received issue update for {jissue.jira_key} for unknown object") + # Process the assignee if present + assignee = parsed['issue']['fields'].get('assignee') + assignee_name = 'Jira User' + if assignee is not None: + # First look for the 'name' field. If not present, try 'displayName'. Else put None + assignee_name = assignee.get('name', assignee.get('displayName')) + + # "resolution":{ + # "self":"http://www.testjira.com/rest/api/2/resolution/11", + # "id":"11", + # "description":"Cancelled by the customer.", + # "name":"Cancelled" + # }, + + # or + # "resolution": null + + # or + # "resolution": "None" + + resolution = parsed['issue']['fields']['resolution'] + resolution = resolution if resolution and resolution != "None" else None + resolution_id = resolution['id'] if resolution else None + resolution_name = resolution['name'] if resolution else None + jira_now = parse_datetime(parsed['issue']['fields']['updated']) + + if findings: + for finding in findings: + jira_helper.process_resolution_from_jira(finding, resolution_id, resolution_name, assignee_name, jira_now, jissue) + # Check for any comment that could have come along with the resolution + if (error_response := check_for_and_create_comment(parsed)) is not None: + return error_response + + if parsed.get('webhookEvent') == 'comment_created': + if (error_response := check_for_and_create_comment(parsed)) is not None: + return error_response + + except Exception as e: + # Check if the issue is originally a 404 + if isinstance(e, Http404): + return webhook_responser_handler("debug", str(e)) + # Try to get a little more information on the exact exception + try: + message = ( + f"Original Exception: {e}\n" + f"jira webhook body parsed:\n{json.dumps(parsed, indent=4)}" + ) + except Exception: + message = ( + f"Original Exception: {e}\n" + f"jira webhook body :\n{request.body.decode('utf-8')}" + ) + return webhook_responser_handler("debug", message) - # reraise to make sure we don't silently swallow things - raise - return HttpResponse('') + return webhook_responser_handler("No logging here", "Success!") def check_for_and_create_comment(parsed_json): @@ -194,31 +220,30 @@ def check_for_and_create_comment(parsed_json): commenter_display_name = comment.get('updateAuthor', {}).get('displayName') # example: body['comment']['self'] = "http://www.testjira.com/jira_under_a_path/rest/api/2/issue/666/comment/456843" jid = comment.get('self', '').split('/')[-3] - jissue = get_object_or_404(JIRA_Issue, jira_id=jid) - logging.info(f"Received issue comment for {jissue.jira_key}") + try: + jissue = JIRA_Issue.objects.get(jira_id=jid) + except JIRA_Instance.DoesNotExist: + return webhook_responser_handler("info", f"JIRA issue {jid} is not linked to a DefectDojo Finding") + logging.debug(f"Received issue comment for {jissue.jira_key}") logger.debug('jissue: %s', vars(jissue)) jira_usernames = JIRA_Instance.objects.values_list('username', flat=True) for jira_user_id in jira_usernames: # logger.debug('incoming username: %s jira config username: %s', commenter.lower(), jira_user_id.lower()) if jira_user_id.lower() == commenter.lower(): - logger.debug('skipping incoming JIRA comment as the user id of the comment in JIRA (%s) matches the JIRA username in DefectDojo (%s)', commenter.lower(), jira_user_id.lower()) - return HttpResponse('') + return webhook_responser_handler("debug", f"skipping incoming JIRA comment as the user id of the comment in JIRA {commenter.lower()} matches the JIRA username in DefectDojo {jira_user_id.lower()}") findings = None if jissue.finding: findings = [jissue.finding] create_notification(event='other', title=f'JIRA incoming comment - {jissue.finding}', finding=jissue.finding, url=reverse("view_finding", args=(jissue.finding.id,)), icon='check') - elif jissue.finding_group: findings = [jissue.finding_group.findings.all()] create_notification(event='other', title=f'JIRA incoming comment - {jissue.finding}', finding=jissue.finding, url=reverse("view_finding_group", args=(jissue.finding_group.id,)), icon='check') - elif jissue.engagement: - return HttpResponse('Comment for engagement ignored') + return webhook_responser_handler("debug", "Comment for engagement ignored") else: - raise Http404(f'No finding or engagement found for JIRA issue {jissue.jira_key}') - + return webhook_responser_handler("info", f"Received issue update for {jissue.jira_key} for unknown object") # Set the fields for the notes author, _ = User.objects.get_or_create(username='JIRA') entry = f'({commenter_display_name} ({commenter})): {comment_text}' diff --git a/dojo/metrics/views.py b/dojo/metrics/views.py index 5f34dd7717c..865fa3a0e7b 100644 --- a/dojo/metrics/views.py +++ b/dojo/metrics/views.py @@ -20,7 +20,13 @@ from django.views.decorators.cache import cache_page from django.utils import timezone -from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointFilter +from dojo.filters import ( + MetricsEndpointFilter, + MetricsEndpointFilterWithoutObjectLookups, + MetricsFindingFilter, + MetricsFindingFilterWithoutObjectLookups, + UserFilter, +) from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm, ProductTagCountsForm from dojo.models import Product_Type, Finding, Product, Engagement, Test, \ Risk_Acceptance, Dojo_User, Endpoint_Status @@ -141,7 +147,10 @@ def finding_querys(prod_type, request): 'test__engagement__risk_acceptance', 'test__test_type', ) - findings = MetricsFindingFilter(request.GET, queryset=findings_query) + filter_string_matching = get_system_setting("filter_string_matching", False) + finding_filter_class = MetricsFindingFilterWithoutObjectLookups if filter_string_matching else MetricsFindingFilter + findings = finding_filter_class(request.GET, queryset=findings_query) + form = findings.form findings_qs = queryset_check(findings) # Quick check to determine if the filters were too tight and filtered everything away if not findings_qs and not findings_query: @@ -205,6 +214,7 @@ def finding_querys(prod_type, request): 'weeks_between': weeks_between, 'start_date': start_date, 'end_date': end_date, + 'form': form, } @@ -218,8 +228,10 @@ def endpoint_querys(prod_type, request): 'finding__reporter') endpoints_query = get_authorized_endpoint_status(Permissions.Endpoint_View, endpoints_query, request.user) - endpoints = MetricsEndpointFilter(request.GET, queryset=endpoints_query) - + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = MetricsEndpointFilterWithoutObjectLookups if filter_string_matching else MetricsEndpointFilter + endpoints = filter_class(request.GET, queryset=endpoints_query) + form = endpoints.form endpoints_qs = queryset_check(endpoints) if not endpoints_qs: @@ -295,6 +307,7 @@ def endpoint_querys(prod_type, request): 'weeks_between': weeks_between, 'start_date': start_date, 'end_date': end_date, + 'form': form, } @@ -445,6 +458,7 @@ def metrics(request, mtype): 'closed_in_period_details': closed_in_period_details, 'punchcard': punchcard, 'ticks': ticks, + 'form': filters.get('form', None), 'show_pt_filter': show_pt_filter, }) diff --git a/dojo/models.py b/dojo/models.py index ab456ae4830..74b8da26888 100755 --- a/dojo/models.py +++ b/dojo/models.py @@ -421,6 +421,12 @@ class System_Settings(models.Model): verbose_name=_('Enable Remediation Advice'), help_text=_("Enables global remediation advice and matching on CWE and Title. The text will be replaced for mitigation, impact and references on a finding. Useful for providing consistent impact and remediation advice regardless of the scanner.")) + enable_similar_findings = models.BooleanField( + default=True, + blank=False, + verbose_name=_("Enable Similar Findings"), + help_text=_("Enable the query of similar findings on the view finding page. This feature can involve potentially large queries and negatively impact performance")) + engagement_auto_close = models.BooleanField( default=False, blank=False, @@ -571,6 +577,14 @@ class System_Settings(models.Model): blank=False, verbose_name=_("API expose error details"), help_text=_("When turned on, the API will expose error details in the response.")) + filter_string_matching = models.BooleanField( + default=False, + blank=False, + verbose_name=_("Filter String Matching Optimization"), + help_text=_( + "When turned on, all filter operations in the UI will require string matches rather than ID. " + "This is a performance enhancement to avoid fetching objects unnecessarily." + )) from dojo.middleware import System_Settings_Manager objects = System_Settings_Manager() diff --git a/dojo/product/views.py b/dojo/product/views.py index c4b45ae329d..0724f7d4f35 100755 --- a/dojo/product/views.py +++ b/dojo/product/views.py @@ -25,8 +25,19 @@ from django.views import View from dojo.templatetags.display_tags import asvs_calc_level -from dojo.filters import ProductEngagementFilter, ProductFilter, EngagementFilter, MetricsEndpointFilter, \ - MetricsFindingFilter, ProductComponentFilter +from dojo.filters import ( + ProductEngagementFilter, + ProductEngagementFilterWithoutObjectLookups, + ProductFilter, + ProductFilterWithoutObjectLookups, + EngagementFilter, + EngagementFilterWithoutObjectLookups, + MetricsEndpointFilter, + MetricsEndpointFilterWithoutObjectLookups, + MetricsFindingFilter, + MetricsFindingFilterWithoutObjectLookups, + ProductComponentFilter, +) from dojo.forms import ProductForm, EngForm, DeleteProductForm, DojoMetaDataForm, JIRAProjectForm, JIRAFindingForm, \ AdHocFindingForm, \ EngagementPresetsForm, DeleteEngagementPresetsForm, ProductNotificationsForm, \ @@ -39,7 +50,7 @@ Endpoint, Engagement_Presets, DojoMeta, Notifications, BurpRawRequestResponse, Product_Member, \ Product_Group, Product_API_Scan_Configuration from dojo.utils import add_external_issue, add_error_message_to_response, add_field_errors_to_response, get_page_items, \ - add_breadcrumb, async_delete, \ + add_breadcrumb, async_delete, calculate_finding_age, \ get_system_setting, get_setting, Product_Tab, get_punchcard_data, queryset_check, is_title_in_breadcrumbs, \ get_enabled_notifications_list, get_zero_severity_level, sum_by_severity_level, get_open_findings_burndown @@ -67,8 +78,9 @@ def product(request): # otherwise the paginator will perform all the annotations/prefetching already only to count the total number of records # see https://code.djangoproject.com/ticket/23771 and https://code.djangoproject.com/ticket/25375 name_words = prods.values_list('name', flat=True) - - prod_filter = ProductFilter(request.GET, queryset=prods, user=request.user) + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = ProductFilterWithoutObjectLookups if filter_string_matching else ProductFilter + prod_filter = filter_class(request.GET, queryset=prods, user=request.user) prod_list = get_page_items(request, prod_filter.qs, 25) @@ -304,7 +316,9 @@ def finding_querys(request, prod): # 'test__test_type', # 'risk_acceptance_set', 'reporter') - findings = MetricsFindingFilter(request.GET, queryset=findings_query, pid=prod) + filter_string_matching = get_system_setting("filter_string_matching", False) + finding_filter_class = MetricsFindingFilterWithoutObjectLookups if filter_string_matching else MetricsFindingFilter + findings = finding_filter_class(request.GET, queryset=findings_query, pid=prod) findings_qs = queryset_check(findings) filters['form'] = findings.form @@ -368,7 +382,9 @@ def endpoint_querys(request, prod): 'finding__test__engagement__risk_acceptance', 'finding__risk_acceptance_set', 'finding__reporter').annotate(severity=F('finding__severity')) - endpoints = MetricsEndpointFilter(request.GET, queryset=endpoints_query) + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = MetricsEndpointFilterWithoutObjectLookups if filter_string_matching else MetricsEndpointFilter + endpoints = filter_class(request.GET, queryset=endpoints_query) endpoints_qs = queryset_check(endpoints) filters['form'] = endpoints.form @@ -448,7 +464,9 @@ def view_product_metrics(request, pid): engs = Engagement.objects.filter(product=prod, active=True) view = identify_view(request) - result = EngagementFilter( + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EngagementFilterWithoutObjectLookups if filter_string_matching else EngagementFilter + result = filter_class( request.GET, queryset=Engagement.objects.filter(product=prod, active=False).order_by('-target_end')) @@ -460,16 +478,9 @@ def view_product_metrics(request, pid): elif view == 'Endpoint': filters = endpoint_querys(request, prod) - start_date = filters['start_date'] + start_date = timezone.make_aware(datetime.combine(filters['start_date'], datetime.min.time())) end_date = filters['end_date'] - tests = Test.objects.filter(engagement__product=prod).prefetch_related('finding_set', 'test_type') - tests = tests.annotate(verified_finding_count=Count('finding__id', filter=Q(finding__verified=True))) - - open_vulnerabilities = filters['open_vulns'] - all_vulnerabilities = filters['all_vulns'] - - start_date = timezone.make_aware(datetime.combine(start_date, datetime.min.time())) r = relativedelta(end_date, start_date) weeks_between = int(ceil((((r.years * 12) + r.months) * 4.33) + (r.days / 7))) if weeks_between <= 0: @@ -485,19 +496,45 @@ def view_product_metrics(request, pid): critical_weekly = OrderedDict() high_weekly = OrderedDict() medium_weekly = OrderedDict() + open_objs_by_age = {} open_objs_by_severity = get_zero_severity_level() closed_objs_by_severity = get_zero_severity_level() accepted_objs_by_severity = get_zero_severity_level() - for finding in filters.get("all", []): - iso_cal = finding.date.isocalendar() + # Optimization: Make all queries lists, and only pull values of fields for metrics based calculations + open_vulnerabilities = list(filters['open_vulns'].values('cwe', 'count')) + all_vulnerabilities = list(filters['all_vulns'].values('cwe', 'count')) + + verified_objs_by_severity = list(filters.get('verified').values('severity')) + inactive_objs_by_severity = list(filters.get('inactive').values('severity')) + false_positive_objs_by_severity = list(filters.get('false_positive').values('severity')) + out_of_scope_objs_by_severity = list(filters.get('out_of_scope').values('severity')) + new_objs_by_severity = list(filters.get('new_verified').values('severity')) + all_objs_by_severity = list(filters.get('all').values('severity')) + + all_findings = list(filters.get("all", []).values('id', 'date', 'severity')) + open_findings = list(filters.get("open", []).values('id', 'date', 'mitigated', 'severity')) + closed_findings = list(filters.get("closed", []).values('id', 'date', 'severity')) + accepted_findings = list(filters.get("accepted", []).values('id', 'date', 'severity')) + + ''' + Optimization: Create dictionaries in the structure of { finding_id: True } for index based search + Previously the for-loop below used "if finding in open_findings" -- an average O(n^2) time complexity + This allows for "if open_findings.get(finding_id, None)" -- an average O(n) time complexity + ''' + open_findings_dict = {f.get('id'): True for f in open_findings} + closed_findings_dict = {f.get('id'): True for f in closed_findings} + accepted_findings_dict = {f.get('id'): True for f in accepted_findings} + + for finding in all_findings: + iso_cal = finding.get('date').isocalendar() date = iso_to_gregorian(iso_cal[0], iso_cal[1], 1) html_date = date.strftime("%m/%d
%Y") unix_timestamp = (tcalendar.timegm(date.timetuple()) * 1000) # Open findings - if finding in filters.get("open", []): + if open_findings_dict.get(finding.get('id', None), None): if unix_timestamp not in critical_weekly: critical_weekly[unix_timestamp] = {'count': 0, 'week': html_date} if unix_timestamp not in high_weekly: @@ -512,9 +549,15 @@ def view_product_metrics(request, pid): open_close_weekly[unix_timestamp]['week'] = html_date if view == 'Finding': - severity = finding.severity + severity = finding.get('severity') elif view == 'Endpoint': - severity = finding.finding.severity + severity = finding.finding.get('severity') + + finding_age = calculate_finding_age(finding) + if open_objs_by_age.get(finding_age, None): + open_objs_by_age[finding_age] += 1 + else: + open_objs_by_age[finding_age] = 1 if unix_timestamp in severity_weekly: if severity in severity_weekly[unix_timestamp]: @@ -542,28 +585,33 @@ def view_product_metrics(request, pid): else: medium_weekly[unix_timestamp] = {'count': 1, 'week': html_date} # Optimization: count severity level on server side - if open_objs_by_severity.get(finding.severity) is not None: - open_objs_by_severity[finding.severity] += 1 + if open_objs_by_severity.get(finding.get('severity')) is not None: + open_objs_by_severity[finding.get('severity')] += 1 + # Close findings - if finding in filters.get("closed", []): + elif closed_findings_dict.get(finding.get('id', None), None): if unix_timestamp in open_close_weekly: open_close_weekly[unix_timestamp]['closed'] += 1 else: open_close_weekly[unix_timestamp] = {'closed': 1, 'open': 0, 'accepted': 0} open_close_weekly[unix_timestamp]['week'] = html_date # Optimization: count severity level on server side - if closed_objs_by_severity.get(finding.severity) is not None: - closed_objs_by_severity[finding.severity] += 1 + if closed_objs_by_severity.get(finding.get('severity')) is not None: + closed_objs_by_severity[finding.get('severity')] += 1 + # Risk Accepted findings - if finding in filters.get("accepted", []): + if accepted_findings_dict.get(finding.get('id', None), None): if unix_timestamp in open_close_weekly: open_close_weekly[unix_timestamp]['accepted'] += 1 else: open_close_weekly[unix_timestamp] = {'closed': 0, 'open': 0, 'accepted': 1} open_close_weekly[unix_timestamp]['week'] = html_date # Optimization: count severity level on server side - if accepted_objs_by_severity.get(finding.severity) is not None: - accepted_objs_by_severity[finding.severity] += 1 + if accepted_objs_by_severity.get(finding.get('severity')) is not None: + accepted_objs_by_severity[finding.get('severity')] += 1 + + tests = Test.objects.filter(engagement__product=prod).prefetch_related('finding_set', 'test_type') + tests = tests.annotate(verified_finding_count=Count('finding__id', filter=Q(finding__verified=True))) test_data = {} for t in tests: @@ -572,9 +620,11 @@ def view_product_metrics(request, pid): else: test_data[t.test_type.name] = t.verified_finding_count - product_tab = Product_Tab(prod, title=_("Product"), tab="metrics") + # Optimization: Format Open/Total CWE vulnerabilities graph data here, instead of template + open_vulnerabilities = [['CWE-' + str(f.get('cwe')), f.get('count')] for f in open_vulnerabilities] + all_vulnerabilities = [['CWE-' + str(f.get('cwe')), f.get('count')] for f in all_vulnerabilities] - open_objs_by_age = {x: len([_ for _ in filters.get('open') if _.age == x]) for x in set([_.age for _ in filters.get('open')])} + product_tab = Product_Tab(prod, title=_("Product"), tab="metrics") return render(request, 'dojo/product_metrics.html', { 'prod': prod, @@ -582,28 +632,30 @@ def view_product_metrics(request, pid): 'engs': engs, 'inactive_engs': inactive_engs_page, 'view': view, - 'verified_objs': filters.get('verified', None), - 'verified_objs_by_severity': sum_by_severity_level(filters.get('verified')), - 'open_objs': filters.get('open', None), + 'verified_objs': len(verified_objs_by_severity), + 'verified_objs_by_severity': sum_by_severity_level(verified_objs_by_severity), + 'open_objs': len(open_findings), 'open_objs_by_severity': open_objs_by_severity, 'open_objs_by_age': open_objs_by_age, - 'inactive_objs': filters.get('inactive', None), - 'inactive_objs_by_severity': sum_by_severity_level(filters.get('inactive')), - 'closed_objs': filters.get('closed', None), + 'inactive_objs': len(inactive_objs_by_severity), + 'inactive_objs_by_severity': sum_by_severity_level(inactive_objs_by_severity), + 'closed_objs': len(closed_findings), 'closed_objs_by_severity': closed_objs_by_severity, - 'false_positive_objs': filters.get('false_positive', None), - 'false_positive_objs_by_severity': sum_by_severity_level(filters.get('false_positive')), - 'out_of_scope_objs': filters.get('out_of_scope', None), - 'out_of_scope_objs_by_severity': sum_by_severity_level(filters.get('out_of_scope')), - 'accepted_objs': filters.get('accepted', None), + 'false_positive_objs': len(false_positive_objs_by_severity), + 'false_positive_objs_by_severity': sum_by_severity_level(false_positive_objs_by_severity), + 'out_of_scope_objs': len(out_of_scope_objs_by_severity), + 'out_of_scope_objs_by_severity': sum_by_severity_level(out_of_scope_objs_by_severity), + 'accepted_objs': len(accepted_findings), 'accepted_objs_by_severity': accepted_objs_by_severity, - 'new_objs': filters.get('new_verified', None), - 'new_objs_by_severity': sum_by_severity_level(filters.get('new_verified')), - 'all_objs': filters.get('all', None), - 'all_objs_by_severity': sum_by_severity_level(filters.get('all')), + 'new_objs': len(new_objs_by_severity), + 'new_objs_by_severity': sum_by_severity_level(new_objs_by_severity), + 'all_objs': len(all_objs_by_severity), + 'all_objs_by_severity': sum_by_severity_level(all_objs_by_severity), 'form': filters.get('form', None), 'reset_link': reverse('view_product_metrics', args=(prod.id,)) + '?type=' + view, + 'open_vulnerabilities_count': len(open_vulnerabilities), 'open_vulnerabilities': open_vulnerabilities, + 'all_vulnerabilities_count': len(all_vulnerabilities), 'all_vulnerabilities': all_vulnerabilities, 'start_date': start_date, 'punchcard': punchcard, @@ -636,31 +688,36 @@ def async_burndown_metrics(request, pid): @user_is_authorized(Product, Permissions.Engagement_View, 'pid') def view_engagements(request, pid): prod = get_object_or_404(Product, id=pid) - default_page_num = 10 recent_test_day_count = 7 - + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = ProductEngagementFilterWithoutObjectLookups if filter_string_matching else ProductEngagementFilter # In Progress Engagements engs = Engagement.objects.filter(product=prod, active=True, status="In Progress").order_by('-updated') - active_engs_filter = ProductEngagementFilter(request.GET, queryset=engs, prefix='active') + active_engs_filter = filter_class(request.GET, queryset=engs, prefix='active') result_active_engs = get_page_items(request, active_engs_filter.qs, default_page_num, prefix="engs") - # prefetch only after creating the filters to avoid https://code.djangoproject.com/ticket/23771 and https://code.djangoproject.com/ticket/25375 - result_active_engs.object_list = prefetch_for_view_engagements(result_active_engs.object_list, - recent_test_day_count) - + # prefetch only after creating the filters to avoid https://code.djangoproject.com/ticket/23771 + # and https://code.djangoproject.com/ticket/25375 + result_active_engs.object_list = prefetch_for_view_engagements( + result_active_engs.object_list, + recent_test_day_count, + ) # Engagements that are queued because they haven't started or paused engs = Engagement.objects.filter(~Q(status="In Progress"), product=prod, active=True).order_by('-updated') - queued_engs_filter = ProductEngagementFilter(request.GET, queryset=engs, prefix='queued') + queued_engs_filter = filter_class(request.GET, queryset=engs, prefix='queued') result_queued_engs = get_page_items(request, queued_engs_filter.qs, default_page_num, prefix="queued_engs") - result_queued_engs.object_list = prefetch_for_view_engagements(result_queued_engs.object_list, - recent_test_day_count) - + result_queued_engs.object_list = prefetch_for_view_engagements( + result_queued_engs.object_list, + recent_test_day_count, + ) # Cancelled or Completed Engagements engs = Engagement.objects.filter(product=prod, active=False).order_by('-target_end') - inactive_engs_filter = ProductEngagementFilter(request.GET, queryset=engs, prefix='closed') + inactive_engs_filter = filter_class(request.GET, queryset=engs, prefix='closed') result_inactive_engs = get_page_items(request, inactive_engs_filter.qs, default_page_num, prefix="inactive_engs") - result_inactive_engs.object_list = prefetch_for_view_engagements(result_inactive_engs.object_list, - recent_test_day_count) + result_inactive_engs.object_list = prefetch_for_view_engagements( + result_inactive_engs.object_list, + recent_test_day_count, + ) product_tab = Product_Tab(prod, title=_("All Engagements"), tab="engagements") return render(request, 'dojo/view_engagements.html', { diff --git a/dojo/reports/views.py b/dojo/reports/views.py index 2eea646b74d..b6c3024bb0e 100644 --- a/dojo/reports/views.py +++ b/dojo/reports/views.py @@ -17,7 +17,7 @@ from django.views import View from dojo.filters import ReportFindingFilter, EndpointReportFilter, \ - EndpointFilter + EndpointFilter, EndpointFilterWithoutObjectLookups from dojo.forms import ReportOptionsForm from dojo.models import Product_Type, Finding, Product, Engagement, Test, \ Dojo_User, Endpoint, Risk_Acceptance @@ -63,8 +63,9 @@ def report_builder(request): finding__duplicate=False, finding__out_of_scope=False, ).distinct() - - endpoints = EndpointFilter(request.GET, queryset=endpoints, user=request.user) + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EndpointFilterWithoutObjectLookups if filter_string_matching else EndpointFilter + endpoints = filter_class(request.GET, queryset=endpoints, user=request.user) in_use_widgets = [ReportOptions(request=request)] available_widgets = [CoverPage(request=request), diff --git a/dojo/reports/widgets.py b/dojo/reports/widgets.py index 36831c4ad0c..fea53696673 100644 --- a/dojo/reports/widgets.py +++ b/dojo/reports/widgets.py @@ -11,10 +11,10 @@ from django.utils.html import format_html from django.utils.safestring import mark_safe -from dojo.filters import EndpointFilter, ReportFindingFilter +from dojo.filters import EndpointFilter, EndpointFilterWithoutObjectLookups, ReportFindingFilter from dojo.forms import CustomReportOptionsForm from dojo.models import Endpoint, Finding -from dojo.utils import get_page_items, get_words_for_field +from dojo.utils import get_page_items, get_words_for_field, get_system_setting """ Widgets are content sections that can be included on reports. The report builder will allow any number of widgets @@ -407,7 +407,9 @@ def report_widget_factory(json_data=None, request=None, user=None, finding_notes d[item['name']] = item['value'] endpoints = Endpoint.objects.filter(id__in=endpoints) - endpoints = EndpointFilter(d, queryset=endpoints, user=request.user) + filter_string_matching = get_system_setting("filter_string_matching", False) + filter_class = EndpointFilterWithoutObjectLookups if filter_string_matching else EndpointFilter + endpoints = filter_class(d, queryset=endpoints, user=request.user) user_id = user.id if user is not None else None endpoints = EndpointList(request=request, endpoints=endpoints, finding_notes=finding_notes, finding_images=finding_images, host=host, user_id=user_id) diff --git a/dojo/search/views.py b/dojo/search/views.py index b2a474eb26b..717ced20594 100644 --- a/dojo/search/views.py +++ b/dojo/search/views.py @@ -6,11 +6,11 @@ from django.db.models import Q from dojo.forms import SimpleSearchForm from dojo.models import Finding, Finding_Template, Product, Test, Engagement, Languages -from dojo.utils import add_breadcrumb, get_page_items, get_words_for_field +from dojo.utils import add_breadcrumb, get_page_items, get_words_for_field, get_system_setting import re from dojo.finding.views import prefetch_for_findings from dojo.endpoint.views import prefetch_for_endpoints -from dojo.filters import FindingFilter +from dojo.filters import FindingFilter, FindingFilterWithoutObjectLookups from django.conf import settings import shlex import itertools @@ -117,8 +117,9 @@ def simple_search(request): elif search_findings: logger.debug('searching findings') - - findings_filter = FindingFilter(request.GET, queryset=findings, user=request.user, pid=None, prefix='finding') + filter_string_matching = get_system_setting("filter_string_matching", False) + finding_filter_class = FindingFilterWithoutObjectLookups if filter_string_matching else FindingFilter + findings_filter = finding_filter_class(request.GET, queryset=findings, user=request.user, pid=None, prefix='finding') # setting initial values for filters is not supported and discouraged: https://django-filter.readthedocs.io/en/stable/guide/tips.html#using-initial-values-as-defaults # we could try to modify request.GET before generating the filter, but for now we'll leave it as is diff --git a/dojo/templates/dojo/edit_finding.html b/dojo/templates/dojo/edit_finding.html index dc52e0fd2f0..60e29494b11 100644 --- a/dojo/templates/dojo/edit_finding.html +++ b/dojo/templates/dojo/edit_finding.html @@ -208,7 +208,7 @@

GitHub

if ($("#id_duplicate").prop("checked")) { $("#id_duplicate").parent().parent().append(original_finding) } else { - $("#id_duplicate").click(function(){ alert('findings can only be marked as duplicates from the view finding screen'); return false; }); + $("#id_duplicate").click(function(){ alert('findings can only be marked as duplicates from the view finding screen. Similar Findings must be enabled for this operation.'); return false; }); } }; diff --git a/dojo/templates/dojo/metrics.html b/dojo/templates/dojo/metrics.html index fd32fb06876..f44b469a8be 100644 --- a/dojo/templates/dojo/metrics.html +++ b/dojo/templates/dojo/metrics.html @@ -150,7 +150,7 @@

- {% include "dojo/filter_snippet.html" with form=findings.form clear_link="/metrics/product/type" %} + {% include "dojo/filter_snippet.html" with form=form clear_link="/metrics/product/type" %}
diff --git a/dojo/templates/dojo/product_metrics.html b/dojo/templates/dojo/product_metrics.html index dc9d447f833..e50ea32d33f 100644 --- a/dojo/templates/dojo/product_metrics.html +++ b/dojo/templates/dojo/product_metrics.html @@ -50,11 +50,11 @@

- {{ verified_objs|length }} + {{ verified_objs }} Verified - {{ view }}{{ verified_objs|length|pluralize }}
@@ -79,10 +79,10 @@

- {{ open_objs|length }} + {{ open_objs }} Open - {{ view }}{{ open_objs|length|pluralize }}
@@ -107,7 +107,7 @@

- {{ accepted_objs|length }} + {{ accepted_objs }} Risk Accepted @@ -135,10 +135,10 @@

- {{ closed_objs|length }} + {{ closed_objs }} Closed - {{ view }}{{ closed_objs|length|pluralize }}
@@ -163,10 +163,10 @@

- {{ false_positive_objs|length }} + {{ false_positive_objs }} False-postive - {{ view }}{{ false_positive_objs|length|pluralize }}
@@ -191,11 +191,11 @@

- {{ out_of_scope_objs|length }} + {{ out_of_scope_objs }} Out Of Scope - {{ view }}{{ out_of_scope_objs|length|pluralize }}
@@ -220,10 +220,10 @@

- {{ all_objs|length }} + {{ all_objs }} Total - {{ view }}{{ all_objs|length|pluralize }}
@@ -248,10 +248,10 @@

- {{ inactive_objs|length }} + {{ inactive_objs }} Inactive - {{ view }}{{ inactive_objs|length|pluralize }}
@@ -406,8 +406,8 @@

- Finding age of {{ open_objs|length|apnumber }} open - finding{{ open_objs|length|pluralize }} + Finding age of {{ open_objs|apnumber }} open + finding{{ open_objs|pluralize }}

@@ -456,8 +456,8 @@

- Showing {{ open_vulnerabilities|length|apnumber }} - vulnerabilit{{ open_vulnerabilities|length|pluralize:"y,ies"}} from Open findings + Showing {{ open_vulnerabilities_count|apnumber }} + vulnerabilit{{ open_vulnerabilities_count|pluralize:"y,ies"}} from Open findings
@@ -471,8 +471,8 @@

- Showing {{ all_vulnerabilities|length|apnumber }} - vulnerabilit{{ all_vulnerabilities|length|pluralize:"y,ies"}} from all findings + Showing {{ all_vulnerabilities_count|apnumber }} + vulnerabilit{{ all_vulnerabilities_count|pluralize:"y,ies"}} from all findings
@@ -701,17 +701,8 @@

}); finding_age(data_2, ticks); - data = []; - {% for x in open_vulnerabilities %} - data.push(['CWE-{{x.cwe}}', {{x.count}}]); - {% endfor %} - draw_vulnerabilities_graph("#open_vulnerabilities", data); - - data = []; - {% for x in all_vulnerabilities %} - data.push(['CWE-{{x.cwe}}', {{x.count}}]); - {% endfor %} - draw_vulnerabilities_graph("#all_vulnerabilities", data); + draw_vulnerabilities_graph("#open_vulnerabilities", {{ open_vulnerabilities|safe }}); + draw_vulnerabilities_graph("#all_vulnerabilities", {{ all_vulnerabilities|safe }}); //$(".product-graphs").hide(); $("#meta_accordion").accordion(); diff --git a/dojo/templates/dojo/view_finding.html b/dojo/templates/dojo/view_finding.html index 5b2a7cc081a..f623f16d6aa 100755 --- a/dojo/templates/dojo/view_finding.html +++ b/dojo/templates/dojo/view_finding.html @@ -729,6 +729,7 @@

Duplicate Cluster ({{ finding|finding_duplicate_cluster_size }}) {% endif %} + {% if similar_findings_enabled %}

Similar Findings ({{ similar_findings.paginator.count }}) @@ -759,8 +760,8 @@

Similar Findings ({{ similar_findings.paginator.count }}

{% endif %} -
- + + {% endif %} {% comment %} Add a form to (ab)use to submit any actions related to similar/duplicates as POST requests {% endcomment %}