add metrics page: "Product Tag Count" (fixes #9151)

docs/content/en/usage/features.md

@@ -557,6 +557,9 @@ Product Type Counts
 
     ![Product Type Counts](../../images/met_2.png)
 
+Product Tag Counts
+:   Same as above, but for a group of products sharing a tag.
+
 Simple Metrics
 :   Provides tabular data for all Product Types. The data displayed in
     this view is the total number of S0, S1, S2, S3, S4, Opened This

dojo/forms.py

@@ -1,4 +1,4 @@
 import os
 import re
 from datetime import datetime, date
 import pickle
@@ -2111,21 +2111,37 @@
     return [(now.year, now.year), (now.year - 1, now.year - 1), (now.year - 2, now.year - 2)]
 
 
-class ProductTypeCountsForm(forms.Form):
+class ProductCountsFormBase(forms.Form):
     month = forms.ChoiceField(choices=list(MONTHS.items()), required=True, error_messages={
         'required': '*'})
     year = forms.ChoiceField(choices=get_years, required=True, error_messages={
         'required': '*'})
+
+
+class ProductTypeCountsForm(ProductCountsFormBase):
     product_type = forms.ModelChoiceField(required=True,
                                           queryset=Product_Type.objects.none(),
                                           error_messages={
                                               'required': '*'})
 
     def __init__(self, *args, **kwargs):
-        super(ProductTypeCountsForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.fields['product_type'].queryset = get_authorized_product_types(Permissions.Product_Type_View)
 
 
+class ProductTagCountsForm(ProductCountsFormBase):
+    product_tag = forms.ModelChoiceField(required=True,
+                                         queryset=Product.tags.tag_model.objects.none().order_by('name'),
+                                         error_messages={
+                                             'required': '*'})
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        prods = get_authorized_products(Permissions.Product_View)
+        tags_available_to_user = Product.tags.tag_model.objects.filter(product__in=prods)
+        self.fields['product_tag'].queryset = tags_available_to_user
+
+
 class APIKeyForm(forms.ModelForm):
     id = forms.IntegerField(required=True,
                             widget=forms.widgets.HiddenInput())

dojo/locale/en/LC_MESSAGES/django.po

@@ -2692,6 +2692,10 @@ msgstr ""
 msgid "Product Type Counts"
 msgstr ""
 
+#: dojo/templates/base.html
+msgid "Product Tag Counts"
+msgstr ""
+
 #: dojo/templates/base.html
 msgid "Users"
 msgstr ""

dojo/metrics/urls.py

@@ -1,4 +1,4 @@
 from django.urls import re_path
 
 from dojo.metrics import views
 
@@ -18,6 +18,8 @@
         views.metrics, name='product_type_metrics'),
     re_path(r'^metrics/product/type/counts$',
         views.product_type_counts, name='product_type_counts'),
+    re_path(r'^metrics/product/tag/counts$',
+        views.product_tag_counts, name='product_tag_counts'),
     re_path(r'^metrics/engineer$', views.engineer_metrics,
         name='engineer_metrics'),
     re_path(r'^metrics/engineer/(?P<eid>\d+)$', views.view_engineer,

dojo/metrics/views.py

@@ -1,4 +1,4 @@
 # #  metrics
 import collections
 import logging
 import operator
@@ -21,7 +21,7 @@
 from django.utils import timezone
 
 from dojo.filters import MetricsFindingFilter, UserFilter, MetricsEndpointFilter
-from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm
+from dojo.forms import SimpleMetricsForm, ProductTypeCountsForm, ProductTagCountsForm
 from dojo.models import Product_Type, Finding, Product, Engagement, Test, \
     Risk_Acceptance, Dojo_User, Endpoint_Status
 from dojo.utils import get_page_items, add_breadcrumb, findings_this_period, opened_in_period, count_findings, \
@@ -586,13 +586,13 @@
                                 end_date.month, end_date.day,
                                 tzinfo=timezone.get_current_timezone())
 
-            oip = opened_in_period(start_date, end_date, pt)
+            oip = opened_in_period(start_date, end_date, test__engagement__product__prod_type=pt)
 
             # trending data - 12 months
             for x in range(12, 0, -1):
                 opened_in_period_list.append(
                     opened_in_period(start_date + relativedelta(months=-x), end_of_month + relativedelta(months=-x),
-                                     pt))
+                                     test__engagement__product__prod_type=pt))
 
             opened_in_period_list.append(oip)
 
@@ -697,6 +697,164 @@
                   )
 
 
+def product_tag_counts(request):
+    form = ProductTagCountsForm()
+    opened_in_period_list = []
+    oip = None
+    cip = None
+    aip = None
+    all_current_in_pt = None
+    top_ten = None
+    pt = None
+    today = timezone.now()
+    first_of_month = today.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+    mid_month = first_of_month.replace(day=15, hour=23, minute=59, second=59, microsecond=999999)
+    end_of_month = mid_month.replace(day=monthrange(today.year, today.month)[1], hour=23, minute=59, second=59,
+                                     microsecond=999999)
+    start_date = first_of_month
+    end_date = end_of_month
+
+    if request.method == 'GET' and 'month' in request.GET and 'year' in request.GET and 'product_tag' in request.GET:
+        form = ProductTagCountsForm(request.GET)
+        if form.is_valid():
+            prods = get_authorized_products(Permissions.Product_View)
+
+            pt = form.cleaned_data['product_tag']
+            month = int(form.cleaned_data['month'])
+            year = int(form.cleaned_data['year'])
+            first_of_month = first_of_month.replace(month=month, year=year)
+
+            month_requested = datetime(year, month, 1)
+
+            end_of_month = month_requested.replace(day=monthrange(month_requested.year, month_requested.month)[1],
+                                                   hour=23, minute=59, second=59, microsecond=999999)
+            start_date = first_of_month
+            start_date = datetime(start_date.year,
+                                  start_date.month, start_date.day,
+                                  tzinfo=timezone.get_current_timezone())
+            end_date = end_of_month
+            end_date = datetime(end_date.year,
+                                end_date.month, end_date.day,
+                                tzinfo=timezone.get_current_timezone())
+
+            oip = opened_in_period(start_date, end_date,
+                test__engagement__product__tags__name=pt,
+                test__engagement__product__in=prods)
+
+            # trending data - 12 months
+            for x in range(12, 0, -1):
+                opened_in_period_list.append(
+                    opened_in_period(start_date + relativedelta(months=-x), end_of_month + relativedelta(months=-x),
+                                     test__engagement__product__tags__name=pt, test__engagement__product__in=prods))
+
+            opened_in_period_list.append(oip)
+
+            closed_in_period = Finding.objects.filter(mitigated__date__range=[start_date, end_date],
+                                                      test__engagement__product__tags__name=pt,
+                                                      test__engagement__product__in=prods,
+                                                      severity__in=('Critical', 'High', 'Medium', 'Low')).values(
+                'numerical_severity').annotate(Count('numerical_severity')).order_by('numerical_severity')
+
+            total_closed_in_period = Finding.objects.filter(mitigated__date__range=[start_date, end_date],
+                                                            test__engagement__product__tags__name=pt,
+                                                            test__engagement__product__in=prods,
+                                                            severity__in=(
+                                                                'Critical', 'High', 'Medium', 'Low')).aggregate(
+                total=Sum(
+                    Case(When(severity__in=('Critical', 'High', 'Medium', 'Low'),
+                              then=Value(1)),
+                         output_field=IntegerField())))['total']
+
+            overall_in_pt = Finding.objects.filter(date__lt=end_date,
+                                                   verified=True,
+                                                   false_p=False,
+                                                   duplicate=False,
+                                                   out_of_scope=False,
+                                                   mitigated__isnull=True,
+                                                   test__engagement__product__tags__name=pt,
+                                                   test__engagement__product__in=prods,
+                                                   severity__in=('Critical', 'High', 'Medium', 'Low')).values(
+                'numerical_severity').annotate(Count('numerical_severity')).order_by('numerical_severity')
+
+            total_overall_in_pt = Finding.objects.filter(date__lte=end_date,
+                                                         verified=True,
+                                                         false_p=False,
+                                                         duplicate=False,
+                                                         out_of_scope=False,
+                                                         mitigated__isnull=True,
+                                                         test__engagement__product__tags__name=pt,
+                                                         test__engagement__product__in=prods,
+                                                         severity__in=('Critical', 'High', 'Medium', 'Low')).aggregate(
+                total=Sum(
+                    Case(When(severity__in=('Critical', 'High', 'Medium', 'Low'),
+                              then=Value(1)),
+                         output_field=IntegerField())))['total']
+
+            all_current_in_pt = Finding.objects.filter(date__lte=end_date,
+                                                       verified=True,
+                                                       false_p=False,
+                                                       duplicate=False,
+                                                       out_of_scope=False,
+                                                       mitigated__isnull=True,
+                                                       test__engagement__product__tags__name=pt,
+                                                       test__engagement__product__in=prods,
+                                                       severity__in=(
+                                                           'Critical', 'High', 'Medium', 'Low')).prefetch_related(
+                'test__engagement__product',
+                'test__engagement__product__prod_type',
+                'test__engagement__risk_acceptance',
+                'reporter').order_by(
+                'numerical_severity')
+
+            top_ten = Product.objects.filter(engagement__test__finding__date__lte=end_date,
+                                             engagement__test__finding__verified=True,
+                                             engagement__test__finding__false_p=False,
+                                             engagement__test__finding__duplicate=False,
+                                             engagement__test__finding__out_of_scope=False,
+                                             engagement__test__finding__mitigated__isnull=True,
+                                             engagement__test__finding__severity__in=(
+                                                 'Critical', 'High', 'Medium', 'Low'),
+                                             tags__name=pt, engagement__product__in=prods)
+            top_ten = severity_count(top_ten, 'annotate', 'engagement__test__finding__severity').order_by('-critical', '-high', '-medium', '-low')[:10]
+
+            cip = {'S0': 0,
+                   'S1': 0,
+                   'S2': 0,
+                   'S3': 0,
+                   'Total': total_closed_in_period}
+
+            aip = {'S0': 0,
+                   'S1': 0,
+                   'S2': 0,
+                   'S3': 0,
+                   'Total': total_overall_in_pt}
+
+            for o in closed_in_period:
+                cip[o['numerical_severity']] = o['numerical_severity__count']
+
+            for o in overall_in_pt:
+                aip[o['numerical_severity']] = o['numerical_severity__count']
+        else:
+            messages.add_message(request, messages.ERROR, _("Please choose month and year and the Product Tag."),
+                                 extra_tags='alert-danger')
+
+    add_breadcrumb(title=_("Bi-Weekly Metrics"), top_level=True, request=request)
+
+    return render(request,
+                  'dojo/pt_counts.html',
+                  {'form': form,
+                   'start_date': start_date,
+                   'end_date': end_date,
+                   'opened_in_period': oip,
+                   'trending_opened': opened_in_period_list,
+                   'closed_in_period': cip,
+                   'overall_in_pt': aip,
+                   'all_current_in_pt': all_current_in_pt,
+                   'top_ten': top_ten,
+                   'pt': pt}
+                  )
+
+
 def engineer_metrics(request):
     # only superusers can select other users to view
     if request.user.is_superuser:

dojo/templates/base.html

@@ -1,4 +1,4 @@
 {% load navigation_tags %}
 {% load display_tags %}
 {% load authorization_tags %}
 {% load i18n %}
@@ -413,6 +413,11 @@
                                                         {% trans "Product Type Counts" %}
                                                     </a>
                                                 </li>
+                                                <li>
+                                                    <a href="{% url 'product_tag_counts' %}">
+                                                        {% trans "Product Tag Counts" %}
+                                                    </a>
+                                                </li>
                                                 <li>
                                                     <a href="{% url 'simple_metrics' %}">
                                                         {% trans "Simple Metrics" %}

dojo/templates/dojo/pt_counts.html

@@ -1,4 +1,4 @@
 {% extends "base.html" %}
 {% load i18n %}
 {% load display_tags %}
 {% block add_styles %}
@@ -12,16 +12,20 @@
 {% block content %}
     {{ block.super }}
 
-    <form class="biweekly-metrics" action="{% url 'product_type_counts' %}" method="get">
+    <form class="biweekly-metrics" method="get">
         {{ form.as_p }}
         <input class="btn btn-sm btn-primary" type="submit" value="{% trans "Generate Metrics For Selected Period" %}"/>
     </form>
     <br/>
     {% if pt %}
         <h2>{% blocktrans with start_date=start_date.date end_date=end_date.date%}Finding Information For Period of {{ start_date }} - {{ end_date }}
             {% endblocktrans %}</h2>
-        <h3 class="inline-block">{{ pt.name }}</h3> [
-        <a href="{% url 'product_type_metrics' pt.id %}" class="inline-block">{% trans "View Details" %}</a>]
+        <h3 class="inline-block">{{ pt.name }}</h3>
+        {% if pt|class_name == "Product_Type" %}
+        [<a href="{% url 'product_type_metrics' pt.id %}" class="inline-block">{% trans "View Details" %}</a>]
+        {% elif pt|class_name == "Tagulous_Product_tags" %}
+        [<a href="{% url 'product' %}?tags={{ pt }}" class="inline-block">{% trans "View Details" %}</a>]
+        {% endif %}
         <div class="panel panel-default table-responsive">
             <div class="panel-heading">
                 <h4>{% trans "Total Security Bug Count In Period" %}</h4>

dojo/utils.py

@@ -1,4 +1,4 @@
 from dojo.authorization.roles_permissions import Permissions
 from dojo.finding.queries import get_authorized_findings
 import re
 import binascii
@@ -1082,7 +1082,7 @@
     }
 
 
-def opened_in_period(start_date, end_date, pt):
+def opened_in_period(start_date, end_date, **kwargs):
     start_date = datetime(
         start_date.year,
         start_date.month,
@@ -1095,7 +1095,7 @@
         tzinfo=timezone.get_current_timezone())
     opened_in_period = Finding.objects.filter(
         date__range=[start_date, end_date],
-        test__engagement__product__prod_type=pt,
+        **kwargs,
         verified=True,
         false_p=False,
         duplicate=False,
@@ -1107,7 +1107,7 @@
                 Count('numerical_severity')).order_by('numerical_severity')
     total_opened_in_period = Finding.objects.filter(
         date__range=[start_date, end_date],
-        test__engagement__product__prod_type=pt,
+        **kwargs,
         verified=True,
         false_p=False,
         duplicate=False,
@@ -1139,7 +1139,7 @@
         'closed':
         Finding.objects.filter(
             mitigated__date__range=[start_date, end_date],
-            test__engagement__product__prod_type=pt,
+            **kwargs,
             severity__in=('Critical', 'High', 'Medium', 'Low')).aggregate(
                 total=Sum(
                     Case(
@@ -1155,7 +1155,7 @@
             duplicate=False,
             out_of_scope=False,
             mitigated__isnull=True,
-            test__engagement__product__prod_type=pt,
+            **kwargs,
             severity__in=('Critical', 'High', 'Medium', 'Low')).count()
     }