From 5fc007b9f423e3258399f4a126a4d10e05d1c79c Mon Sep 17 00:00:00 2001 From: Tejas Saubhage Date: Sat, 14 Mar 2026 02:41:13 -0400 Subject: [PATCH 1/2] Add import history cleanup task and max_import_history setting fixes #13776 --- ...0262_system_settings_max_import_history.py | 22 +++++++++++ dojo/models.py | 3 ++ dojo/settings/settings.dist.py | 2 + dojo/tasks.py | 39 ++++++++++++++++++- 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 dojo/db_migrations/0262_system_settings_max_import_history.py diff --git a/dojo/db_migrations/0262_system_settings_max_import_history.py b/dojo/db_migrations/0262_system_settings_max_import_history.py new file mode 100644 index 00000000000..fd8e917e7f5 --- /dev/null +++ b/dojo/db_migrations/0262_system_settings_max_import_history.py @@ -0,0 +1,22 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("dojo", "0261_remove_url_insert_insert_remove_url_update_update_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="system_settings", + name="max_import_history", + field=models.IntegerField( + blank=True, + null=True, + default=None, + verbose_name="Max Import History", + help_text="When set, the oldest import history records will be deleted when a test exceeds this number of imports. Leave empty to keep all history.", + ), + ), + ] diff --git a/dojo/models.py b/dojo/models.py index 430de3d132e..e787ea7144f 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -325,6 +325,9 @@ class System_Settings(models.Model): "issue reaches the maximum " "number of duplicates, the " "oldest will be deleted. Duplicate will not be deleted when left empty. A value of 0 will remove all duplicates.")) + max_import_history = models.IntegerField(blank=True, null=True, default=None, + verbose_name=_("Max Import History"), + help_text=_("When set, the oldest import history records will be deleted when a test exceeds this number of imports. Leave empty to keep all history.")) email_from = models.CharField(max_length=200, default="no-reply@example.com", blank=True) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 6b5c95b371c..f4886a532f1 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -280,6 +280,7 @@ # we limit the amount of duplicates that can be deleted in a single run of that job # to prevent overlapping runs of that job from occurrring DD_DUPE_DELETE_MAX_PER_RUN=(int, 200), + DD_IMPORT_HISTORY_MAX_PER_OBJECT=(int, 200), # when enabled 'mitigated date' and 'mitigated by' of a finding become editable DD_EDITABLE_MITIGATED_DATA=(bool, False), # new feature that tracks history across multiple reimports for the same test @@ -1773,6 +1774,7 @@ def saml2_attrib_map_format(din): DEDUPLICATION_ALGORITHM_PER_PARSER[key] = value DUPE_DELETE_MAX_PER_RUN = env("DD_DUPE_DELETE_MAX_PER_RUN") +IMPORT_HISTORY_MAX_PER_OBJECT = env("DD_IMPORT_HISTORY_MAX_PER_OBJECT") DISABLE_FINDING_MERGE = env("DD_DISABLE_FINDING_MERGE") diff --git a/dojo/tasks.py b/dojo/tasks.py index 1bbe104783b..3277412d440 100644 --- a/dojo/tasks.py +++ b/dojo/tasks.py @@ -207,7 +207,44 @@ def jira_status_reconciliation_task(*args, **kwargs): return jira_status_reconciliation(*args, **kwargs) -@app.task + +@app.task(bind=True) +def async_import_history_cleanup(*args, **kwargs): + with pghistory.context(source="import_history_cleanup_task"): + _async_import_history_cleanup_impl() + + +def _async_import_history_cleanup_impl(): + """Delete oldest Test_Import records when a test exceeds max_import_history.""" + try: + system_settings = System_Settings.objects.get() + max_history = system_settings.max_import_history + max_per_run = settings.IMPORT_HISTORY_MAX_PER_OBJECT + except System_Settings.DoesNotExist: + return + + if max_history is None: + logger.info("skipping import history cleanup: max_import_history not configured") + return + + logger.info("cleaning up import history (max per test: %s, max deletes per run: %s)", max_history, max_per_run) + + tests_with_excess = Test_Import.objects \ + .values("test") \ + .annotate(import_count=Count("id")) \ + .filter(import_count__gt=max_history)[:max_per_run] + + total_deleted_count = 0 + for entry in tests_with_excess: + test_id = entry["test"] + imports = Test_Import.objects.filter(test_id=test_id).order_by("created") + excess_count = entry["import_count"] - max_history + for test_import in imports[:excess_count]: + logger.debug("deleting Test_Import id %s for test %s", test_import.id, test_id) + test_import.delete() + total_deleted_count += 1 + + logger.info("total import history records deleted: %s", total_deleted_count) def fix_loop_duplicates_task(*args, **kwargs): # Wrap with pghistory context for audit trail with pghistory.context(source="fix_loop_duplicates"): From 0605a5b24ed54d99e2d17a1bbe889f49ebf2ebfb Mon Sep 17 00:00:00 2001 From: Tejas Saubhage Date: Sun, 15 Mar 2026 05:45:42 -0400 Subject: [PATCH 2/2] Fix import history cleanup: add logging for System_Settings.DoesNotExist --- dojo/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dojo/tasks.py b/dojo/tasks.py index 3277412d440..ccc35239f78 100644 --- a/dojo/tasks.py +++ b/dojo/tasks.py @@ -221,6 +221,7 @@ def _async_import_history_cleanup_impl(): max_history = system_settings.max_import_history max_per_run = settings.IMPORT_HISTORY_MAX_PER_OBJECT except System_Settings.DoesNotExist: + logger.error('System_Settings not found, skipping import history cleanup') return if max_history is None: