fix(product): defer list count annotations

dojo/product/ui/filters.py

@@ -104,20 +104,24 @@ class ProductFilterHelper(FilterSet):
             field_name="locations__status",
             choices=ProductLocationStatus.choices,
             help_text="Status of the Location from the Products relationship",
+            method="filter_location_status",
         )
         endpoints__host = CharFilter(
             field_name="locations__location__url__host", method="filter_endpoints_host", label="Endpoint Host",
         )
         endpoints = NumberFilter(field_name="locations__location", method="filter_endpoints", widget=HiddenInput())
 
+        def filter_location_status(self, queryset, name, value):
+            return queryset.filter(locations__status__in=value).distinct()
+
         def filter_endpoints_host(self, queryset, name, value):
             return filter_endpoints_host_base(
                 queryset,
                 name,
                 value,
                 endpoint_id=self.data.get("endpoints"),
                 statuses=self.data.getlist("location_status"),
-            )
+            ).distinct()
 
         def filter_endpoints(self, queryset, name, value):
             return filter_endpoints_base(
@@ -126,7 +130,7 @@ def filter_endpoints(self, queryset, name, value):
                 value,
                 statuses=self.data.getlist("location_status"),
                 host=self.data.get("endpoints__host"),
-            )
+            ).distinct()
 
     o = OrderingFilter(
         # tuple-mapping retains order

dojo/product/ui/views.py

@@ -13,7 +13,7 @@
 from django.contrib.postgres.aggregates import StringAgg
 from django.core.exceptions import PermissionDenied, ValidationError
 from django.db import DEFAULT_DB_ALIAS, connection
-from django.db.models import Count, DateField, F, OuterRef, Prefetch, Q, Subquery, Sum, Value
+from django.db.models import Count, DateField, F, IntegerField, OuterRef, Prefetch, Q, Subquery, Sum, Value
 from django.db.models.functions import Coalesce
 from django.db.models.query import QuerySet
 from django.http import Http404, HttpRequest, HttpResponseRedirect, JsonResponse
@@ -62,6 +62,7 @@
 from dojo.jira import services as jira_services
 from dojo.labels import get_labels
 from dojo.location.feature import locations_enabled
+from dojo.location.models import LocationProductReference
 from dojo.models import (
     App_Analysis,
     Benchmark_Product_Summary,
@@ -132,24 +133,45 @@
 labels = get_labels()
 
 
+def annotate_product_findings_count(prods):
+    base_findings = Finding.objects.filter(test__engagement__product_id=OuterRef("pk"), active=True)
+    return prods.annotate(
+        findings_count=Coalesce(
+            build_count_subquery(base_findings, group_field="test__engagement__product_id"), Value(0),
+        ),
+    )
+
+
+def annotate_product_location_counts(prods):
+    location_refs = LocationProductReference.objects.filter(product_id=OuterRef("pk"))
+    return prods.annotate(
+        location_count=Coalesce(
+            build_count_subquery(location_refs, group_field="product_id"), Value(0),
+        ),
+        location_host_count=Coalesce(
+            Subquery(
+                location_refs.order_by()
+                .values("product_id")
+                .annotate(c=Count("location__url__host", distinct=True))
+                .order_by("product_id")
+                .values("c")[:1],
+                output_field=IntegerField(),
+            ),
+            Value(0),
+        ),
+    )
+
+
 def product(request):
     prods = get_authorized_products("view")
     # perform all stuff for filtering and pagination first, before annotation/prefetching
     # otherwise the paginator will perform all the annotations/prefetching already only to count the total number of records
     # see https://code.djangoproject.com/ticket/23771 and https://code.djangoproject.com/ticket/25375
 
     name_words = prods.values_list("name", flat=True)
-    base_findings = Finding.objects.filter(test__engagement__product_id=OuterRef("pk"), active=True)
-    prods = prods.annotate(
-        findings_count=Coalesce(
-            build_count_subquery(base_findings, group_field="test__engagement__product_id"), Value(0),
-        ),
-    )
+    prods = annotate_product_findings_count(prods)
     if locations_enabled():
-        prods = prods.annotate(
-            location_host_count=Count("locations__location__url__host", distinct=True),
-            location_count=Count("locations", distinct=True),
-        )
+        prods = annotate_product_location_counts(prods)
 
     filter_string_matching = get_system_setting("filter_string_matching", False)
     filter_class = ProductFilterWithoutObjectLookups if filter_string_matching else ProductFilter

unittests/test_product_list_pagination.py

@@ -0,0 +1,89 @@
+from django.http import QueryDict
+from django.test import override_settings
+
+from dojo.filters import filter_endpoints_host_base
+from dojo.location.models import LocationProductReference
+from dojo.location.status import ProductLocationStatus
+from dojo.models import Product, Product_Type
+from dojo.product.ui.filters import ProductFilter
+from dojo.product.ui.views import annotate_product_location_counts
+from dojo.url.models import URL
+from unittests.dojo_test_case import DojoTestCase, versioned_fixtures
+
+
+@versioned_fixtures
+class ProductListPaginationAnnotationTests(DojoTestCase):
+    fixtures = ["dojo_testdata.json"]
+
+    @staticmethod
+    def _make_product_with_locations():
+        product_type = Product_Type.objects.create(name="Product list location counts")
+        product = Product.objects.create(
+            name="Product with duplicate location hosts",
+            prod_type=product_type,
+            description="Regression test product",
+        )
+        urls = [
+            URL.create_location_from_value("https://duplicate.example/one"),
+            URL.create_location_from_value("https://duplicate.example/two"),
+            URL.create_location_from_value("https://other.example/"),
+        ]
+        for url in urls:
+            LocationProductReference.objects.create(
+                location=url.location,
+                product=product,
+                status=ProductLocationStatus.Active,
+            )
+        return product
+
+    @override_settings(V3_FEATURE_LOCATIONS=True)
+    def test_location_counts_use_subqueries_without_product_group_by(self):
+        product = self._make_product_with_locations()
+
+        queryset = annotate_product_location_counts(Product.objects.filter(id=product.id))
+        sql = str(queryset.query)
+        annotated = queryset.get()
+
+        self.assertEqual(annotated.location_count, 3)
+        self.assertEqual(annotated.location_host_count, 2)
+        self.assertNotIn('LEFT OUTER JOIN "dojo_locationproductreference"', sql)
+        self.assertNotIn('GROUP BY "dojo_product"', sql)
+
+    @override_settings(V3_FEATURE_LOCATIONS=True)
+    def test_product_endpoint_host_filter_deduplicates_products(self):
+        product = self._make_product_with_locations()
+
+        data = QueryDict(mutable=True)
+        data["endpoints__host"] = "duplicate.example"
+        prod_filter = ProductFilter(data, queryset=Product.objects.all(), user=self.get_test_admin())
+
+        self.assertEqual(list(prod_filter.qs.filter(id=product.id).values_list("id", flat=True)), [product.id])
+
+    @override_settings(V3_FEATURE_LOCATIONS=True)
+    def test_product_endpoint_id_filter_deduplicates_products(self):
+        product = self._make_product_with_locations()
+        location_id = product.locations.first().location_id
+
+        data = QueryDict(mutable=True)
+        data["endpoints"] = str(location_id)
+        prod_filter = ProductFilter(data, queryset=Product.objects.all(), user=self.get_test_admin())
+
+        self.assertEqual(list(prod_filter.qs.filter(id=product.id).values_list("id", flat=True)), [product.id])
+
+    @override_settings(V3_FEATURE_LOCATIONS=True)
+    def test_product_location_status_filter_deduplicates_products(self):
+        product = self._make_product_with_locations()
+
+        data = QueryDict(mutable=True)
+        data.setlist("location_status", [ProductLocationStatus.Active])
+        prod_filter = ProductFilter(data, queryset=Product.objects.all(), user=self.get_test_admin())
+
+        self.assertEqual(list(prod_filter.qs.filter(id=product.id).values_list("id", flat=True)), [product.id])
+
+    @override_settings(V3_FEATURE_LOCATIONS=True)
+    def test_shared_endpoint_host_helper_does_not_deduplicate_findings_scope(self):
+        product = self._make_product_with_locations()
+
+        filtered = filter_endpoints_host_base(Product.objects.all(), "endpoints__host", "duplicate.example")
+
+        self.assertEqual(list(filtered.filter(id=product.id).values_list("id", flat=True)), [product.id, product.id])