Community Health: Obsolete resources and corresponding entities

pontoon/administration/forms.py

@@ -123,9 +123,9 @@ def __init__(self, *args, **kwargs):
         # If the project instance is available, filter resources for this project
         if kwargs.get("instance") and kwargs["instance"].project:
             project = kwargs["instance"].project
-            self.fields["resources"].queryset = Resource.objects.filter(
-                project=project
-            ).select_related()
+            self.fields["resources"].queryset = (
+                Resource.objects.current().filter(project=project).select_related()
+            )
 
 
 TagInlineFormSet = inlineformset_factory(Project, Tag, form=TagInlineForm, extra=1)

pontoon/administration/tests/test_views.py

@@ -380,6 +380,32 @@ def test_manage_project_strings_download_csv(client_superuser):
     assert "Mächt’ge".encode() in response.content
 
 
+@pytest.mark.django_db
+def test_manage_project_translate_link_excludes_obsolete_resources(client_superuser):
+    """Test that translate_locale is only set when non-obsolete resources exist."""
+    locale_kl = LocaleFactory.create(code="tlh", name="Klingon")
+    project = ProjectFactory.create(
+        data_source=Project.DataSource.DATABASE,
+        locales=[locale_kl],
+        repositories=[],
+    )
+
+    # add obsolete resource
+    ResourceFactory.create(project=project, obsolete=True)
+
+    url = reverse("pontoon.admin.project", args=(project.slug,))
+    response = client_superuser.get(url)
+    assert response.status_code == 200
+    assert "translate_locale" not in response.context
+
+    # add non-obsolete resource
+    ResourceFactory.create(project=project, obsolete=False)
+
+    response = client_superuser.get(url)
+    assert response.status_code == 200
+    assert response.context["translate_locale"] == "tlh"
+
+
 @pytest.mark.django_db
 def test_project_add_locale(client_superuser):
     locale_kl = LocaleFactory.create(code="kl", name="Klingon")

pontoon/administration/views.py

@@ -306,7 +306,7 @@ def manage_project(request, slug=None, template="admin_project.html"):
     }
 
     # Set locale in Translate link
-    if Resource.objects.filter(project=project).exists() and locales_selected:
+    if Resource.objects.current().filter(project=project).exists() and locales_selected:
         locale = (
             utils.get_project_locale_from_request(request, project.locales)
             or locales_selected[0].code
@@ -373,7 +373,7 @@ def _get_resource_for_database_project(project):
 
     """
     try:
-        return Resource.objects.get(
+        return Resource.objects.current().get(
             project=project,
         )
     except Resource.DoesNotExist:
@@ -492,7 +492,7 @@ def manage_project_strings(request, slug=None):
             # Get all strings, find the ones that changed, update them in the database.
             formset = EntityFormSet(request.POST, queryset=entities)
             if formset.is_valid():
-                resource = Resource.objects.filter(project=project).first()
+                resource = Resource.objects.current().filter(project=project).first()
                 entity_max_order = entities.aggregate(Max("order"))["order__max"]
                 try:
                     # This line can purposefully cause an exception, and that

pontoon/base/models/locale.py

@@ -63,12 +63,16 @@ def available(self):
 
     def stats_data(self, project=None):
         if project is not None:
-            query = self.filter(translatedresources__resource__project=project)
+            query = self.filter(
+                translatedresources__resource__project=project,
+                translatedresources__resource__obsolete=False,
+            )
         else:
             query = self.filter(
                 translatedresources__resource__project__disabled=False,
                 translatedresources__resource__project__system_project=False,
                 translatedresources__resource__project__visibility="public",
+                translatedresources__resource__obsolete=False,
             )
 
         return query.annotate(

pontoon/base/models/project_locale.py

@@ -32,7 +32,8 @@ def visible(self):
     def stats_data(self, project=None, locale=None):
         if project:
             query = self.filter(
-                locale__translatedresources__resource__project=project
+                locale__translatedresources__resource__project=project,
+                locale__translatedresources__resource__obsolete=False,
             ).prefetch_related("locale")
             tr = "locale__translatedresources"
         elif locale:
@@ -41,6 +42,7 @@ def stats_data(self, project=None, locale=None):
                 project__disabled=False,
                 project__system_project=False,
                 project__visibility="public",
+                project__resources__obsolete=False,
             ).prefetch_related("project")
             tr = "project__resources__translatedresources"
         return query.annotate(

pontoon/base/models/resource.py

@@ -3,6 +3,28 @@
 from django.utils import timezone
 
 
+class ResourceQuerySet(models.QuerySet):
+    def mark_as_obsolete(self, now=None):
+        from pontoon.base.models.entity import Entity
+
+        if now is None:
+            now = timezone.now()
+
+        self.update(obsolete=True, date_obsoleted=now)
+        Entity.objects.filter(resource__in=self).update(
+            obsolete=True,
+            date_obsoleted=now,
+            section=None,
+        )
+
+        from pontoon.base.models.translated_resource import TranslatedResource
+
+        TranslatedResource.objects.filter(resource__in=self).delete()
+
+    def current(self):
+        return self.filter(obsolete=False)
+
+
 class Resource(models.Model):
     project = models.ForeignKey("Project", models.CASCADE, related_name="resources")
     path = models.TextField()  # Path to localization file
@@ -42,6 +64,8 @@ class Format(models.TextChoices):
 
     deadline = models.DateField(blank=True, null=True)
 
+    objects = ResourceQuerySet.as_manager()
+
     # Formats that allow empty translations
     EMPTY_TRANSLATION_FORMATS = {
         Format.DTD,

pontoon/base/signals.py

@@ -173,7 +173,7 @@ def add_locale_to_system_projects(sender, instance, created, **kwargs):
         projects = Project.objects.filter(system_project=True)
         for project in projects:
             ProjectLocale.objects.create(project=project, locale=instance)
-            for resource in project.resources.all():
+            for resource in project.resources.current():
                 translated_resource = TranslatedResource.objects.create(
                     resource=resource,
                     locale=instance,
@@ -189,7 +189,7 @@ def add_locale_to_terminology_project(sender, instance, created, **kwargs):
     if created:
         project = Project.objects.get(slug="terminology")
         ProjectLocale.objects.create(project=project, locale=instance)
-        for resource in project.resources.all():
+        for resource in project.resources.current():
             translated_resource = TranslatedResource.objects.create(
                 resource=resource,
                 locale=instance,

pontoon/base/tests/managers/test_user.py

@@ -1,6 +1,9 @@
+from unittest.mock import MagicMock
+
 import pytest
 
 from django.db.models import Q
+from django.utils import timezone
 
 from pontoon.base.utils import aware_datetime
 from pontoon.contributors.utils import users_with_translations_counts
@@ -322,3 +325,66 @@ def test_mgr_user_query_args_filtering(
     assert top_contribs[0].translations_approved_count == 11
     assert top_contribs[0].translations_rejected_count == 0
     assert top_contribs[0].translations_unapproved_count == 3
+
+
+@pytest.mark.django_db
+def test_mgr_user_translation_counts_after_resource_removed(
+    resource_a,
+    locale_a,
+):
+    """
+    Tests that contributor translation counts remain unchanged after
+    a resource is removed via remove_resources.
+
+    Translation counts should include translations from obsolete resources
+    since they represent the contributor's historical work.
+    """
+    from pontoon.sync.core.entities import remove_resources
+
+    contributor = UserFactory.create()
+    entities = EntityFactory.create_batch(size=12, resource=resource_a)
+
+    batch_kwargs = (
+        [dict(approved=True)] * 7
+        + [dict(approved=False, fuzzy=False, rejected=True)] * 3
+        + [dict(fuzzy=True)] * 2
+    )
+
+    for i, kwa in enumerate(batch_kwargs):
+        TranslationFactory.create(
+            locale=locale_a,
+            user=contributor,
+            entity=entities[i],
+            approved=kwa.get("approved", False),
+            rejected=kwa.get("rejected", False),
+            fuzzy=kwa.get("fuzzy", False),
+        )
+
+    top_contribs = users_with_translations_counts()
+    assert len(top_contribs) == 1
+    assert top_contribs[0] == contributor
+    assert top_contribs[0].translations_count == 12
+    assert top_contribs[0].translations_approved_count == 7
+    assert top_contribs[0].translations_rejected_count == 3
+    assert top_contribs[0].translations_unapproved_count == 2
+
+    # Remove resource using remove_resources (simulates sync removing source file)
+    checkout = MagicMock()
+    checkout.path = "/path_1"
+    checkout.removed = [resource_a.path]
+
+    paths = MagicMock()
+    paths.ref_root = "/path_1"
+
+    remove_resources(resource_a.project, paths, checkout, timezone.now())
+
+    resource_a.refresh_from_db()
+    assert resource_a.obsolete is True
+
+    top_contribs = users_with_translations_counts()
+    assert len(top_contribs) == 1
+    assert top_contribs[0] == contributor
+    assert top_contribs[0].translations_count == 12
+    assert top_contribs[0].translations_approved_count == 7
+    assert top_contribs[0].translations_rejected_count == 3
+    assert top_contribs[0].translations_unapproved_count == 2

pontoon/projects/views.py

@@ -39,9 +39,7 @@ def projects(request):
         "projects/projects.html",
         {
             "projects": projects,
-            "all_projects_stats": TranslatedResource.objects.all().string_stats(
-                request.user
-            ),
+            "all_projects_stats": TranslatedResource.objects.string_stats(request.user),
             "project_stats": project_stats,
             "top_instances": get_top_instances(projects, project_stats),
         },

pontoon/sync/core/entities.py

@@ -70,7 +70,7 @@ def sync_resources_from_repo(
 
     with transaction.atomic():
         renamed_paths = rename_resources(project, paths, checkout)
-        removed_paths = remove_resources(project, paths, checkout)
+        removed_paths = remove_resources(project, paths, checkout, now)
         old_res_added_ent_count, changed_paths = update_resources(project, updates, now)
         new_res_added_ent_count, _ = add_resources(project, updates, changed_paths, now)
         update_translated_resources(project, locale_map, paths)
@@ -103,7 +103,10 @@ def rename_resources(
 
 
 def remove_resources(
-    project: Project, paths: L10nConfigPaths | L10nDiscoverPaths, checkout: Checkout
+    project: Project,
+    paths: L10nConfigPaths | L10nDiscoverPaths,
+    checkout: Checkout,
+    now: datetime,
 ) -> set[str]:
     if not checkout.removed:
         return set()
@@ -115,8 +118,7 @@ def remove_resources(
     )
     removed_db_paths = {res.path for res in removed_resources}
     if removed_db_paths:
-        # FIXME: https://github.com/mozilla/pontoon/issues/2133
-        removed_resources.delete()
+        removed_resources.mark_as_obsolete(now)
         rm_count = len(removed_db_paths)
         str_source_files = "source file" if rm_count == 1 else "source files"
         log.info(
@@ -279,11 +281,46 @@ def add_resources(
     changed_paths: set[str],
     now: datetime,
 ) -> tuple[int, set[str]]:
+
+    existing_resources = dict(
+        Resource.objects.filter(project=project, path__in=updates.keys()).values_list(
+            "path", "obsolete"
+        )
+    )
+
+    valid_updates = {}
+    deobsoletion_paths = []
+
+    for db_path, res in updates.items():
+        res_exists = db_path in existing_resources
+        res_obsolete = res_exists and existing_resources[db_path]
+
+        has_entries = False
+        if not res_obsolete:
+            has_entries = next(res.all_entries(), None) is not None
+
+        if (has_entries and db_path not in changed_paths) or res_obsolete:
+            valid_updates[db_path] = res
+
+            if res_obsolete:
+                deobsoletion_paths.append(db_path)
+
+    if not valid_updates:
+        return 0, set()
+
+    # Resource de-obsoletion for existing Resources that are added back in
+    # TODO Entity de-obsoletion needs to accompany Resource de-obsoletion
+    if deobsoletion_paths:
+        Resource.objects.filter(project=project, path__in=deobsoletion_paths).update(
+            obsolete=False
+        )
+
     new_resources = [
         Resource(project=project, path=db_path, format=get_res_format(res))
-        for db_path, res in updates.items()
-        if next(res.all_entries(), None) and db_path not in changed_paths
+        for db_path, res in valid_updates.items()
+        if db_path not in existing_resources
     ]
+
     if not new_resources:
         return 0, set()
 
@@ -345,7 +382,7 @@ def update_translated_resources(
         .iterator()
     )
     add_tr: list[TranslatedResource] = []
-    for resource in Resource.objects.filter(project=project).iterator():
+    for resource in Resource.objects.current().filter(project=project).iterator():
         _, locales = paths.target(resource.path)
         for lc in locales:
             locale = locale_map.get(lc, None)

pontoon/sync/core/translations_from_repo.py

@@ -178,9 +178,9 @@ def find_db_updates(
 
     resources: dict[str, Resource] = {
         res.path: res
-        for res in Resource.objects.filter(
-            project=project, path__in=resource_paths
-        ).iterator()
+        for res in Resource.objects.current()
+        .filter(project=project, path__in=resource_paths)
+        .iterator()
     }
 
     # Exclude translations for which DB & repo already match