diff --git a/dojo/product/ui/filters.py b/dojo/product/ui/filters.py index 89ca863f65..4b9ceb76a6 100644 --- a/dojo/product/ui/filters.py +++ b/dojo/product/ui/filters.py @@ -104,12 +104,16 @@ class ProductFilterHelper(FilterSet): field_name="locations__status", choices=ProductLocationStatus.choices, help_text="Status of the Location from the Products relationship", + method="filter_location_status", ) endpoints__host = CharFilter( field_name="locations__location__url__host", method="filter_endpoints_host", label="Endpoint Host", ) endpoints = NumberFilter(field_name="locations__location", method="filter_endpoints", widget=HiddenInput()) + def filter_location_status(self, queryset, name, value): + return queryset.filter(locations__status__in=value).distinct() + def filter_endpoints_host(self, queryset, name, value): return filter_endpoints_host_base( queryset, @@ -117,7 +121,7 @@ def filter_endpoints_host(self, queryset, name, value): value, endpoint_id=self.data.get("endpoints"), statuses=self.data.getlist("location_status"), - ) + ).distinct() def filter_endpoints(self, queryset, name, value): return filter_endpoints_base( @@ -126,7 +130,7 @@ def filter_endpoints(self, queryset, name, value): value, statuses=self.data.getlist("location_status"), host=self.data.get("endpoints__host"), - ) + ).distinct() o = OrderingFilter( # tuple-mapping retains order diff --git a/dojo/product/ui/views.py b/dojo/product/ui/views.py index 775146fcdc..54f608fb9e 100644 --- a/dojo/product/ui/views.py +++ b/dojo/product/ui/views.py @@ -13,7 +13,7 @@ from django.contrib.postgres.aggregates import StringAgg from django.core.exceptions import PermissionDenied, ValidationError from django.db import DEFAULT_DB_ALIAS, connection -from django.db.models import Count, DateField, F, OuterRef, Prefetch, Q, Subquery, Sum, Value +from django.db.models import Count, DateField, F, IntegerField, OuterRef, Prefetch, Q, Subquery, Sum, Value from django.db.models.functions import Coalesce from django.db.models.query import QuerySet from django.http import Http404, HttpRequest, HttpResponseRedirect, JsonResponse @@ -62,6 +62,7 @@ from dojo.jira import services as jira_services from dojo.labels import get_labels from dojo.location.feature import locations_enabled +from dojo.location.models import LocationProductReference from dojo.models import ( App_Analysis, Benchmark_Product_Summary, @@ -132,6 +133,35 @@ labels = get_labels() +def annotate_product_findings_count(prods): + base_findings = Finding.objects.filter(test__engagement__product_id=OuterRef("pk"), active=True) + return prods.annotate( + findings_count=Coalesce( + build_count_subquery(base_findings, group_field="test__engagement__product_id"), Value(0), + ), + ) + + +def annotate_product_location_counts(prods): + location_refs = LocationProductReference.objects.filter(product_id=OuterRef("pk")) + return prods.annotate( + location_count=Coalesce( + build_count_subquery(location_refs, group_field="product_id"), Value(0), + ), + location_host_count=Coalesce( + Subquery( + location_refs.order_by() + .values("product_id") + .annotate(c=Count("location__url__host", distinct=True)) + .order_by("product_id") + .values("c")[:1], + output_field=IntegerField(), + ), + Value(0), + ), + ) + + def product(request): prods = get_authorized_products("view") # perform all stuff for filtering and pagination first, before annotation/prefetching @@ -139,17 +169,9 @@ def product(request): # see https://code.djangoproject.com/ticket/23771 and https://code.djangoproject.com/ticket/25375 name_words = prods.values_list("name", flat=True) - base_findings = Finding.objects.filter(test__engagement__product_id=OuterRef("pk"), active=True) - prods = prods.annotate( - findings_count=Coalesce( - build_count_subquery(base_findings, group_field="test__engagement__product_id"), Value(0), - ), - ) + prods = annotate_product_findings_count(prods) if locations_enabled(): - prods = prods.annotate( - location_host_count=Count("locations__location__url__host", distinct=True), - location_count=Count("locations", distinct=True), - ) + prods = annotate_product_location_counts(prods) filter_string_matching = get_system_setting("filter_string_matching", False) filter_class = ProductFilterWithoutObjectLookups if filter_string_matching else ProductFilter diff --git a/unittests/test_product_list_pagination.py b/unittests/test_product_list_pagination.py new file mode 100644 index 0000000000..965b7d5574 --- /dev/null +++ b/unittests/test_product_list_pagination.py @@ -0,0 +1,89 @@ +from django.http import QueryDict +from django.test import override_settings + +from dojo.filters import filter_endpoints_host_base +from dojo.location.models import LocationProductReference +from dojo.location.status import ProductLocationStatus +from dojo.models import Product, Product_Type +from dojo.product.ui.filters import ProductFilter +from dojo.product.ui.views import annotate_product_location_counts +from dojo.url.models import URL +from unittests.dojo_test_case import DojoTestCase, versioned_fixtures + + +@versioned_fixtures +class ProductListPaginationAnnotationTests(DojoTestCase): + fixtures = ["dojo_testdata.json"] + + @staticmethod + def _make_product_with_locations(): + product_type = Product_Type.objects.create(name="Product list location counts") + product = Product.objects.create( + name="Product with duplicate location hosts", + prod_type=product_type, + description="Regression test product", + ) + urls = [ + URL.create_location_from_value("https://duplicate.example/one"), + URL.create_location_from_value("https://duplicate.example/two"), + URL.create_location_from_value("https://other.example/"), + ] + for url in urls: + LocationProductReference.objects.create( + location=url.location, + product=product, + status=ProductLocationStatus.Active, + ) + return product + + @override_settings(V3_FEATURE_LOCATIONS=True) + def test_location_counts_use_subqueries_without_product_group_by(self): + product = self._make_product_with_locations() + + queryset = annotate_product_location_counts(Product.objects.filter(id=product.id)) + sql = str(queryset.query) + annotated = queryset.get() + + self.assertEqual(annotated.location_count, 3) + self.assertEqual(annotated.location_host_count, 2) + self.assertNotIn('LEFT OUTER JOIN "dojo_locationproductreference"', sql) + self.assertNotIn('GROUP BY "dojo_product"', sql) + + @override_settings(V3_FEATURE_LOCATIONS=True) + def test_product_endpoint_host_filter_deduplicates_products(self): + product = self._make_product_with_locations() + + data = QueryDict(mutable=True) + data["endpoints__host"] = "duplicate.example" + prod_filter = ProductFilter(data, queryset=Product.objects.all(), user=self.get_test_admin()) + + self.assertEqual(list(prod_filter.qs.filter(id=product.id).values_list("id", flat=True)), [product.id]) + + @override_settings(V3_FEATURE_LOCATIONS=True) + def test_product_endpoint_id_filter_deduplicates_products(self): + product = self._make_product_with_locations() + location_id = product.locations.first().location_id + + data = QueryDict(mutable=True) + data["endpoints"] = str(location_id) + prod_filter = ProductFilter(data, queryset=Product.objects.all(), user=self.get_test_admin()) + + self.assertEqual(list(prod_filter.qs.filter(id=product.id).values_list("id", flat=True)), [product.id]) + + @override_settings(V3_FEATURE_LOCATIONS=True) + def test_product_location_status_filter_deduplicates_products(self): + product = self._make_product_with_locations() + + data = QueryDict(mutable=True) + data.setlist("location_status", [ProductLocationStatus.Active]) + prod_filter = ProductFilter(data, queryset=Product.objects.all(), user=self.get_test_admin()) + + self.assertEqual(list(prod_filter.qs.filter(id=product.id).values_list("id", flat=True)), [product.id]) + + @override_settings(V3_FEATURE_LOCATIONS=True) + def test_shared_endpoint_host_helper_does_not_deduplicate_findings_scope(self): + product = self._make_product_with_locations() + + filtered = filter_endpoints_host_base(Product.objects.all(), "endpoints__host", "duplicate.example") + + self.assertEqual(list(filtered.filter(id=product.id).values_list("id", flat=True)), [product.id, product.id])