From c2450080877b535992abea8b18f808cd31c180e8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 12:58:11 -0700 Subject: [PATCH 001/115] #404 add branching support --- netbox_custom_objects/__init__.py | 9 ++ netbox_custom_objects/branching.py | 191 +++++++++++++++++++++++++++ netbox_custom_objects/field_types.py | 5 +- netbox_custom_objects/models.py | 41 ++++-- 4 files changed, 236 insertions(+), 10 deletions(-) create mode 100644 netbox_custom_objects/branching.py diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 2e862e9a..598cda2a 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -175,6 +175,15 @@ def ready(self): pre_migrate.connect(_migration_started) post_migrate.connect(_migration_finished) + # Wire into the netbox-branching lifecycle when that plugin is installed. + # Import is lazy so the plugin remains functional without branching. + try: + from netbox_branching.signals import post_migrate as branching_post_migrate + from netbox_custom_objects.branching import on_branch_migrated + branching_post_migrate.connect(on_branch_migrated) + except ImportError: + pass + # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py new file mode 100644 index 00000000..63832f46 --- /dev/null +++ b/netbox_custom_objects/branching.py @@ -0,0 +1,191 @@ +""" +Branching support for NetBox Custom Objects. + +When netbox-branching runs ``Branch.migrate()``, the Django migration pass only +handles normal app migrations. Custom object field changes (add/alter/remove +column) are applied directly via the schema editor and are therefore invisible +to Django's migration framework. + +To close that gap, ``on_branch_migrated`` fires after the migration pass and +reconciles each custom object type's physical schema in the branch against the +current field definitions in main. The reconciliation mirrors what +``CustomObjectTypeField.save()`` already does when detecting old-vs-new field +state: it compares the branch's ``CustomObjectTypeField`` rows (snapshot from +provision time) with main's current rows, matched by primary key, and calls +``add_field`` / ``alter_field`` / ``remove_field`` for any differences. + +``CustomObjectTypeField`` has ``ChangeLoggingMixin``, so its rows are replicated +into the branch schema at provision time — the same source-of-truth mechanism +used for all other branchable models. +""" + +import logging + +from django.db import connections, models + +logger = logging.getLogger('netbox_custom_objects.branching') + + +def _fields_schema_differ(branch_f, main_f): + """ + Return True if the two ``CustomObjectTypeField`` instances differ in any + attribute that affects the physical DB column, meaning an ALTER TABLE is + needed to bring the branch schema up to date. + """ + return ( + branch_f.name != main_f.name + or branch_f.type != main_f.type + or branch_f.required != main_f.required + or branch_f.default != main_f.default + or branch_f.unique != main_f.unique + or branch_f.related_object_type_id != main_f.related_object_type_id + ) + + +def on_branch_migrated(sender, branch, user, **kwargs): + """ + Reconcile each custom object type's physical schema in the branch against + the current field definitions in main. + + For each ``CustomObjectType`` whose table exists in the branch schema: + - Fields present in main but absent from the branch → ``add_field`` + - Fields absent from main but present in the branch → ``remove_field`` + - Fields present in both with differing definitions → ``alter_field`` + + Matching is done by primary key so renames are detected correctly (the pk + exists in both, with different ``name`` values) rather than being treated + as an unrelated delete + add. + """ + from extras.choices import CustomFieldTypeChoices + from netbox_branching.utilities import activate_branch + from netbox_custom_objects.constants import APP_LABEL + from netbox_custom_objects.field_types import FIELD_TYPE_CLASS + from netbox_custom_objects.models import CustomObjectType + from netbox_custom_objects.utilities import generate_model + + branch_connection = connections[branch.connection_name] + + with branch_connection.cursor() as cursor: + branch_tables = branch_connection.introspection.table_names(cursor) + + for cot in CustomObjectType.objects.all(): + db_table = cot.get_database_table_name() + if db_table not in branch_tables: + # Table absent — COT was created after this branch was provisioned + # and the branch hasn't been synced yet. Skip; the user needs to + # sync first to pull in the new table. + logger.debug('Skipping %s — table %r not in branch schema', cot, db_table) + continue + + # Main's current field definitions (queried from the public schema since + # active_branch is not set here). + main_fields = { + f.pk: f + for f in cot.fields.select_related('related_object_type', 'choice_set').all() + } + + # Branch's field snapshot (as of provision time, or last sync). + with activate_branch(branch): + branch_fields = { + f.pk: f + for f in cot.fields.select_related('related_object_type', 'choice_set').all() + } + + main_pks = set(main_fields) + branch_pks = set(branch_fields) + + to_add = [main_fields[pk] for pk in main_pks - branch_pks] + to_remove = [branch_fields[pk] for pk in branch_pks - main_pks] + to_alter = [ + (branch_fields[pk], main_fields[pk]) # (old, new) + for pk in main_pks & branch_pks + if _fields_schema_differ(branch_fields[pk], main_fields[pk]) + ] + + if not (to_add or to_remove or to_alter): + continue + + logger.info( + 'Migrating branch schema for %s: %d add, %d remove, %d alter', + cot, len(to_add), len(to_remove), len(to_alter), + ) + + model = cot.get_model() + + with branch_connection.schema_editor() as schema_editor: + + # ── add_field ──────────────────────────────────────────────────── + for fi in to_add: + logger.debug('add_field %r on %s', fi.name, cot) + ft = FIELD_TYPE_CLASS[fi.type]() + mf = ft.get_model_field(fi) + mf.contribute_to_class(model, fi.name) + schema_editor.add_field(model, mf) + if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + ft.create_m2m_table(fi, model, fi.name, schema_conn=branch_connection) + + # ── remove_field ───────────────────────────────────────────────── + for fi in to_remove: + logger.debug('remove_field %r on %s', fi.name, cot) + ft = FIELD_TYPE_CLASS[fi.type]() + mf = ft.get_model_field(fi) + mf.contribute_to_class(model, fi.name) + if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + through_table = f'custom_objects_{cot.pk}_{fi.name}' + if through_table in branch_tables: + ThroughMeta = type( + 'Meta', (), + {'db_table': through_table, 'app_label': APP_LABEL, 'managed': True}, + ) + through_model = type( + f'_TempThrough_{through_table}', + (models.Model,), + {'Meta': ThroughMeta, '__module__': 'netbox_custom_objects.models'}, + ) + schema_editor.delete_model(through_model) + schema_editor.remove_field(model, mf) + + # ── alter_field ────────────────────────────────────────────────── + for old_fi, new_fi in to_alter: + logger.debug( + 'alter_field %r → %r on %s', + old_fi.name, new_fi.name, cot, + ) + old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) + new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) + old_mf.contribute_to_class(model, old_fi.name) + new_mf.contribute_to_class(model, new_fi.name) + + # When a MULTIOBJECT field is renamed, the through table must + # be renamed first (same logic as CustomObjectTypeField.save()). + if ( + new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + and old_fi.name != new_fi.name + ): + old_through = f'custom_objects_{cot.pk}_{old_fi.name}' + new_through = f'custom_objects_{cot.pk}_{new_fi.name}' + if old_through in branch_tables: + OldThroughMeta = type( + 'Meta', (), + {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, + ) + old_through_model = generate_model( + f'_TempOldThrough_{old_through}', + (models.Model,), + { + '__module__': 'netbox_custom_objects.models', + 'Meta': OldThroughMeta, + 'id': models.AutoField(primary_key=True), + 'source': models.ForeignKey( + model, on_delete=models.CASCADE, + db_column='source_id', related_name='+', + ), + 'target': models.ForeignKey( + model, on_delete=models.CASCADE, + db_column='target_id', related_name='+', + ), + }, + ) + schema_editor.alter_db_table(old_through_model, old_through, new_through) + + schema_editor.alter_field(model, old_mf, new_mf) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index c89cca3d..83d5e2a6 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -935,11 +935,12 @@ def after_model_generation(self, instance, model, field_name): target_field.remote_field.model = to_model target_field.related_model = to_model - def create_m2m_table(self, instance, model, field_name): + def create_m2m_table(self, instance, model, field_name, schema_conn=None): """ Creates the actual M2M table after models are fully generated """ - from django.db import connection + from django.db import connection as default_connection + connection = schema_conn if schema_conn is not None else default_connection # Get the field instance field = model._meta.get_field(field_name) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index cc0cf58a..577b87ba 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -66,6 +66,25 @@ class UniquenessConstraintTestError(Exception): USER_TABLE_DATABASE_NAME_PREFIX = "custom_objects_" +def _get_schema_connection(): + """ + Return the active branch's DB connection when called within a branch context, + otherwise return the default (main-schema) connection. + + Used so that schema-editor operations (add/alter/remove column) target the + correct PostgreSQL schema without requiring every call-site to be branch-aware. + """ + try: + from netbox_branching.contextvars import active_branch + branch = active_branch.get() + if branch is not None: + from django.db import connections + return connections[branch.connection_name] + except ImportError: + pass + return connection + + class CustomObject( BookmarksMixin, ChangeLoggingMixin, @@ -681,8 +700,6 @@ def create_model(self): # Ensure the ContentType exists and is immediately available features = get_model_features(model) - if 'branching' in features: - features.remove('branching') self.object_type.features = features self.object_type.public = True self.object_type.save() @@ -1608,16 +1625,21 @@ def through_model_name(self): def save(self, *args, **kwargs): is_new = self._state.adding + + # Use the branch connection when operating inside a branch so that schema + # editor operations target the branch schema rather than main. + schema_conn = _get_schema_connection() + field_type = FIELD_TYPE_CLASS[self.type]() model_field = field_type.get_model_field(self) model = self.custom_object_type.get_model() model_field.contribute_to_class(model, self.name) - with connection.schema_editor() as schema_editor: + with schema_conn.schema_editor() as schema_editor: if self._state.adding: schema_editor.add_field(model, model_field) if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - field_type.create_m2m_table(self, model, self.name) + field_type.create_m2m_table(self, model, self.name, schema_conn=schema_conn) else: old_field = field_type.get_model_field(self.original) old_field.contribute_to_class(model, self._original_name) @@ -1632,8 +1654,8 @@ def save(self, *args, **kwargs): new_through_table_name = self.through_table_name # Check if old through table exists - with connection.cursor() as cursor: - tables = connection.introspection.table_names(cursor) + with schema_conn.cursor() as cursor: + tables = schema_conn.introspection.table_names(cursor) old_table_exists = old_through_table_name in tables if old_table_exists: @@ -1709,7 +1731,7 @@ def save(self, *args, **kwargs): ) else: # No old table exists, create the new through table - field_type.create_m2m_table(self, model, self.name) + field_type.create_m2m_table(self, model, self.name, schema_conn=schema_conn) # Alter the field normally (this updates the field definition) schema_editor.alter_field(model, old_field, model_field) @@ -1762,12 +1784,15 @@ def ensure_constraint(): transaction.on_commit(lambda: ReindexCustomObjectTypeJob.enqueue(cot_id=_cot_id)) def delete(self, *args, **kwargs): + # Use the branch connection when operating inside a branch. + schema_conn = _get_schema_connection() + field_type = FIELD_TYPE_CLASS[self.type]() model_field = field_type.get_model_field(self) model = self.custom_object_type.get_model() model_field.contribute_to_class(model, self.name) - with connection.schema_editor() as schema_editor: + with schema_conn.schema_editor() as schema_editor: if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: apps = model._meta.apps through_model = apps.get_model(APP_LABEL, self.through_model_name) From 81565b6154c1a16c7fb6f8b2ff6fcfaaad68337e Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 13:21:21 -0700 Subject: [PATCH 002/115] #404 check for branch drift --- netbox_custom_objects/__init__.py | 14 ++++- netbox_custom_objects/branching.py | 95 ++++++++++++++++++++++++++++++ netbox_custom_objects/models.py | 36 +++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 598cda2a..392aa259 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -178,9 +178,13 @@ def ready(self): # Wire into the netbox-branching lifecycle when that plugin is installed. # Import is lazy so the plugin remains functional without branching. try: + from django.db.models.signals import post_delete, post_save from netbox_branching.signals import post_migrate as branching_post_migrate - from netbox_custom_objects.branching import on_branch_migrated + from netbox_custom_objects.branching import on_branch_migrated, on_custom_object_field_changed + from netbox_custom_objects.models import CustomObjectTypeField branching_post_migrate.connect(on_branch_migrated) + post_save.connect(on_custom_object_field_changed, sender=CustomObjectTypeField) + post_delete.connect(on_custom_object_field_changed, sender=CustomObjectTypeField) except ImportError: pass @@ -213,6 +217,14 @@ def ready(self): super().ready() return + # Scan for branches with pre-existing custom object schema drift + # (covers upgrades and any changes made while the app was offline). + try: + from netbox_custom_objects.branching import check_pending_branch_migrations + check_pending_branch_migrations() + except Exception: + pass + super().ready() def get_model(self, model_name, require_ready=True): diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 63832f46..06db2033 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -26,6 +26,101 @@ logger = logging.getLogger('netbox_custom_objects.branching') +def check_pending_branch_migrations(): + """ + Scan all READY branches at startup and mark any that have custom object + schema drift as PENDING_MIGRATIONS. + + This catches drift that pre-dates the signal handler — for example, after + upgrading the plugin on an instance that already has branches, or if field + changes were applied while the application was not running. + + Called once from ``CustomObjectsPluginConfig.ready()`` after the DB is + confirmed ready. + """ + try: + from netbox_branching.choices import BranchStatusChoices + from netbox_branching.models import Branch + from netbox_custom_objects.models import CustomObjectType + except ImportError: + return + + ready_branches = list(Branch.objects.filter(status=BranchStatusChoices.READY)) + if not ready_branches: + return + + cots = list(CustomObjectType.objects.all()) + if not cots: + return + + to_update = [] + for branch in ready_branches: + branch_connection = connections[branch.connection_name] + with branch_connection.cursor() as cursor: + branch_tables = branch_connection.introspection.table_names(cursor) + + for cot in cots: + if cot.get_database_table_name() not in branch_tables: + continue + if cot.has_branch_schema_drift(branch): + branch.status = BranchStatusChoices.PENDING_MIGRATIONS + to_update.append(branch) + break # One drifted COT is enough — no need to check the rest + + if to_update: + Branch.objects.bulk_update(to_update, ['status']) + logger.info( + 'Marked %d branch(es) as PENDING_MIGRATIONS at startup due to custom object schema drift', + len(to_update), + ) + + +def on_custom_object_field_changed(sender, instance, **kwargs): + """ + Mark any READY branches that contain the affected custom object type's table + as PENDING_MIGRATIONS when a ``CustomObjectTypeField`` is created, modified, + or deleted in the main schema. + + This surfaces the pending state in the branching UI exactly like a normal + Django-migration, prompting users to click "Migrate Branch". That action + calls ``Branch.migrate()``, which fires ``on_branch_migrated`` and reconciles + the physical column differences. + + Skipped when the change happens inside a branch context — the field edit only + affects that branch's schema, not main, so no other branches need updating. + """ + try: + from netbox_branching.contextvars import active_branch + if active_branch.get() is not None: + return + except ImportError: + return + + from netbox_branching.choices import BranchStatusChoices + from netbox_branching.models import Branch + + cot = instance.custom_object_type + db_table = cot.get_database_table_name() + + ready_branches = list(Branch.objects.filter(status=BranchStatusChoices.READY)) + to_update = [] + for branch in ready_branches: + branch_connection = connections[branch.connection_name] + with branch_connection.cursor() as cursor: + branch_tables = branch_connection.introspection.table_names(cursor) + if db_table in branch_tables: + branch.status = BranchStatusChoices.PENDING_MIGRATIONS + to_update.append(branch) + + if to_update: + Branch.objects.bulk_update(to_update, ['status']) + logger.info( + 'Marked %d branch(es) as PENDING_MIGRATIONS due to field changes on %s', + len(to_update), + cot, + ) + + def _fields_schema_differ(branch_f, main_f): """ Return True if the two ``CustomObjectTypeField`` instances differ in any diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 577b87ba..12e9aff6 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -710,6 +710,42 @@ def create_model(self): get_serializer_class(model) self.register_custom_object_search_index(model) + def has_branch_schema_drift(self, branch) -> bool: + """ + Return True if this custom object type's physical schema in *branch* + differs from the current field definitions in main. + + Drift means at least one of: + - A field exists in main but not in the branch snapshot (needs add_field) + - A field exists in the branch snapshot but not in main (needs remove_field) + - A field exists in both but has a different name, type, or constraint + that affects the DB column (needs alter_field) + + Returns False when netbox-branching is not installed. + """ + try: + from netbox_branching.utilities import activate_branch + except ImportError: + return False + + main_fields = {f.pk: f for f in self.fields.all()} + with activate_branch(branch): + branch_fields = {f.pk: f for f in self.fields.all()} + + main_pks = set(main_fields) + branch_pks = set(branch_fields) + + if main_pks != branch_pks: + return True + + def _schema_key(f): + return (f.name, f.type, f.required, f.default, f.unique, f.related_object_type_id) + + return any( + _schema_key(branch_fields[pk]) != _schema_key(main_fields[pk]) + for pk in main_pks + ) + def save(self, *args, **kwargs): needs_db_create = self._state.adding From 3f5e92502945d85c89832da396e8df1dcccfbeb7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 14:31:12 -0700 Subject: [PATCH 003/115] remove branch warning --- README.md | 14 ---------- netbox_custom_objects/api/views.py | 9 ------ ...omobjecttypefield_related_name_and_more.py | 2 +- netbox_custom_objects/models.py | 28 ++++++++++++------- .../custom_object_bulk_edit.html | 6 +--- .../custom_object_bulk_import.html | 10 ++----- .../customobject_edit.html | 10 ++----- .../inc/branch_warning.html | 12 -------- netbox_custom_objects/utilities.py | 15 ---------- netbox_custom_objects/views.py | 4 --- 10 files changed, 26 insertions(+), 84 deletions(-) delete mode 100644 netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html diff --git a/README.md b/README.md index 1a3cf5f0..68796f72 100644 --- a/README.md +++ b/README.md @@ -32,20 +32,6 @@ $ ./manage.py migrate sudo systemctl restart netbox netbox-rq ``` -> [!NOTE] -> If you are using NetBox Custom Objects with NetBox Branching, you need to insert the following into your `configuration.py`. See the docs for a full description of how NetBox Custom Objects currently works with NetBox Branching. - -``` -PLUGINS_CONFIG = { - 'netbox_branching': { - 'exempt_models': [ - 'netbox_custom_objects.customobjecttype', - 'netbox_custom_objects.customobjecttypefield', - ], - }, -} -``` - ## Known Limitations NetBox Custom Objects is now Generally Available which means you can use it in production and migrations to future versions will work. There are many upcoming features including GraphQL support - the best place to see what's on the way is the [issues](https://github.com/netboxlabs/netbox-custom-objects/issues) list on the GitHub repository. diff --git a/netbox_custom_objects/api/views.py b/netbox_custom_objects/api/views.py index 17293777..274be014 100644 --- a/netbox_custom_objects/api/views.py +++ b/netbox_custom_objects/api/views.py @@ -11,13 +11,8 @@ from netbox_custom_objects.filtersets import get_filterset_class from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField -from netbox_custom_objects.utilities import is_in_branch - from . import serializers -# Constants -BRANCH_ACTIVE_ERROR_MESSAGE = _("Please switch to the main branch to perform this operation.") - class RootView(APIRootView): def get_view_name(self): @@ -71,13 +66,9 @@ def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) def create(self, request, *args, **kwargs): - if is_in_branch(): - raise ValidationError(BRANCH_ACTIVE_ERROR_MESSAGE) return super().create(request, *args, **kwargs) def update(self, request, *args, **kwargs): - if is_in_branch(): - raise ValidationError(BRANCH_ACTIVE_ERROR_MESSAGE) # Replicate DRF's UpdateModelMixin.update() so we can snapshot the instance # before the serializer is constructed. Calling super().update() would invoke # get_object() a second time and return a fresh, un-snapshotted instance. diff --git a/netbox_custom_objects/migrations/0005_customobjecttypefield_related_name_and_more.py b/netbox_custom_objects/migrations/0005_customobjecttypefield_related_name_and_more.py index 52512800..8eb63d6c 100644 --- a/netbox_custom_objects/migrations/0005_customobjecttypefield_related_name_and_more.py +++ b/netbox_custom_objects/migrations/0005_customobjecttypefield_related_name_and_more.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ ('core', '0021_job_queue_name'), ('extras', '0134_owner'), - ('netbox_custom_objects', '0004_customobjecttype_group_name'), + ('netbox_custom_objects', '0005_customobjecttypefield_context'), ] operations = [ diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index f34b9dad..22ceb924 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -714,7 +714,7 @@ def create_model(self): self.object_type.public = True self.object_type.save() - with connection.schema_editor() as schema_editor: + with _get_schema_connection().schema_editor() as schema_editor: schema_editor.create_model(model) get_serializer_class(model) @@ -772,20 +772,31 @@ def delete(self, *args, **kwargs): self.clear_model_cache(self.id) model = self.get_model() + schema_conn = _get_schema_connection() + in_branch = schema_conn is not connection # Delete all CustomObjectTypeFields that reference this CustomObjectType for field in CustomObjectTypeField.objects.filter(related_object_type=self.object_type): field.delete() object_type = ObjectType.objects.get_for_model(model) - ObjectChange.objects.filter(changed_object_type=object_type).delete() + + # ObjectChange and ObjectType records live in the main schema. Only clean + # them up when operating outside a branch; inside a branch they belong to + # main and must not be touched until the deletion is merged. + if not in_branch: + ObjectChange.objects.filter(changed_object_type=object_type).delete() + super().delete(*args, **kwargs) - # Temporarily disconnect the pre_delete handler to skip the ObjectType deletion - # TODO: Remove this disconnect/reconnect after ObjectType has been exempted from handle_deleted_object - pre_delete.disconnect(handle_deleted_object) - object_type.delete() - with connection.schema_editor() as schema_editor: + if not in_branch: + # Temporarily disconnect the pre_delete handler to skip the ObjectType deletion + # TODO: Remove this disconnect/reconnect after ObjectType has been exempted from handle_deleted_object + pre_delete.disconnect(handle_deleted_object) + object_type.delete() + pre_delete.connect(handle_deleted_object) + + with schema_conn.schema_editor() as schema_editor: schema_editor.delete_model(model) # Unregister the model and its through-models from Django's app registry so @@ -811,9 +822,6 @@ def delete(self, *args, **kwargs): # Re-clear the model cache to remove re-cached model from get_model. self.clear_model_cache(self.id) - # Reconnect the pre_delete handler after all cleanup is done. - pre_delete.connect(handle_deleted_object) - @receiver(post_save, sender=CustomObjectType) def custom_object_type_post_save_handler(sender, instance, created, **kwargs): diff --git a/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_edit.html b/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_edit.html index 2ed1a515..7371ebec 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_edit.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_edit.html @@ -9,10 +9,6 @@ {# Edit form #}
- {% if branch_warning %} - {% include 'netbox_custom_objects/inc/branch_warning.html' %} - {% endif %} -
{% csrf_token %} @@ -85,7 +81,7 @@

{% trans "Comments" %}

{% trans "Cancel" %} - +
diff --git a/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_import.html b/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_import.html index 3ae447b5..8bce96e7 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_import.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/custom_object_bulk_import.html @@ -10,10 +10,6 @@
- {% if branch_warning %} - {% include 'netbox_custom_objects/inc/branch_warning.html' %} - {% endif %} -
{% csrf_token %} @@ -36,7 +32,7 @@ {% if return_url %} {% trans "Cancel" %} {% endif %} - +
@@ -68,7 +64,7 @@ {% if return_url %} {% trans "Cancel" %} {% endif %} - +
@@ -101,7 +97,7 @@ {% if return_url %} {% trans "Cancel" %} {% endif %} - + diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html index 6cba146a..fdf82574 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject_edit.html @@ -13,10 +13,6 @@ {% endblock title %} {% block form %} - {% if branch_warning %} - {% include 'netbox_custom_objects/inc/branch_warning.html' %} - {% endif %} - {# Render hidden fields #} {% for field in form.hidden_fields %} {{ field }} @@ -50,15 +46,15 @@

{{ group }}

{% block buttons %} {% trans "Cancel" %} {% if object.pk %} - {% else %}
- -
diff --git a/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html b/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html deleted file mode 100644 index ed7d0274..00000000 --- a/netbox_custom_objects/templates/netbox_custom_objects/inc/branch_warning.html +++ /dev/null @@ -1,12 +0,0 @@ -{% load i18n %} - - diff --git a/netbox_custom_objects/utilities.py b/netbox_custom_objects/utilities.py index d8d901f8..7a16cf4f 100644 --- a/netbox_custom_objects/utilities.py +++ b/netbox_custom_objects/utilities.py @@ -8,7 +8,6 @@ "AppsProxy", "generate_model", "get_viewname", - "is_in_branch", ) @@ -110,17 +109,3 @@ def generate_model(*args, **kwargs): return model - -def is_in_branch(): - """ - Check if currently operating within a branch. - - Returns: - bool: True if currently in a branch, False otherwise. - """ - try: - from netbox_branching.contextvars import active_branch - return active_branch.get() is not None - except ImportError: - # Branching plugin not installed - return False diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index af646a01..5ede42e3 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -29,7 +29,6 @@ from extras.choices import CustomFieldTypeChoices from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.dynamic_forms import build_filterset_form_class -from netbox_custom_objects.utilities import is_in_branch logger = logging.getLogger("netbox_custom_objects.views") @@ -588,7 +587,6 @@ def custom_save(self, commit=True): def get_extra_context(self, request, obj): return { - 'branch_warning': is_in_branch(), } @@ -702,7 +700,6 @@ def get_form(self, queryset): def get_extra_context(self, request): return { - 'branch_warning': is_in_branch(), } @@ -793,7 +790,6 @@ def get_model_form(self, queryset): def get_extra_context(self, request): return { - 'branch_warning': is_in_branch(), } From 45315518972b5fec1e03e0028a1e0899ffc897f7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 15:41:17 -0700 Subject: [PATCH 004/115] get merge working --- netbox_custom_objects/models.py | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 22ceb924..ba54dda1 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -720,6 +720,48 @@ def create_model(self): get_serializer_class(model) self.register_custom_object_search_index(model) + @classmethod + def deserialize_object(cls, data, pk=None): + """ + Custom deserialization hook for netbox-branching's merge/revert engine. + + ``ObjectChange.apply()`` normally uses ``DeserializedObject.save()``, which + calls ``Model.save_base(raw=True)`` — bypassing our ``save()`` override and + all ``post_save`` signals. That means ``create_model()`` never runs and the + physical table is never created in the destination schema. + + By implementing this classmethod the apply engine calls our version instead, + returning a wrapper whose ``save()`` invokes the full ``CustomObjectType.save()`` + lifecycle (signals included) so that the table is created as a side effect of + replaying the ObjectChange. + + ``object_type`` is cleared before saving so the ``custom_object_type_post_save_handler`` + can re-create and link it correctly in the destination schema, avoiding any FK + mismatch between the branch and main ``ObjectType`` pks. + """ + from utilities.serialization import deserialize_object as _deserialize + + inner = _deserialize(cls, data, pk=pk) + + class _SchemaAwareDeserialized: + def __init__(self, deserialized): + self._inner = deserialized + self.object = deserialized.object + + def save(self, using=None, **kwargs): + # Clear the ObjectType FK — it may not exist in main yet. + # custom_object_type_post_save_handler re-sets it after INSERT. + self.object.object_type = None + self.object.object_type_id = None + self.object.save() + # Re-apply any M2M data (tags, etc.) that was stripped during deserialization. + if self._inner.m2m_data: + for accessor_name, object_list in self._inner.m2m_data.items(): + getattr(self.object, accessor_name).set(object_list) + self._inner.m2m_data = None + + return _SchemaAwareDeserialized(inner) + def has_branch_schema_drift(self, branch) -> bool: """ Return True if this custom object type's physical schema in *branch* @@ -1677,6 +1719,36 @@ def through_table_name(self): def through_model_name(self): return f"Through_{self.through_table_name}" + @classmethod + def deserialize_object(cls, data, pk=None): + """ + Custom deserialization hook for netbox-branching's merge/revert engine. + + Same problem as ``CustomObjectType.deserialize_object``: the default + ``DeserializedObject.save(raw=True)`` bypasses ``CustomObjectTypeField.save()``, + so the physical column is never added to the custom object table. + + This wrapper calls the real ``save()`` so that ``add_field`` runs as a side + effect of replaying the CREATE ObjectChange during a merge. + """ + from utilities.serialization import deserialize_object as _deserialize + + inner = _deserialize(cls, data, pk=pk) + + class _SchemaAwareDeserialized: + def __init__(self, deserialized): + self._inner = deserialized + self.object = deserialized.object + + def save(self, using=None, **kwargs): + self.object.save() + if self._inner.m2m_data: + for accessor_name, object_list in self._inner.m2m_data.items(): + getattr(self.object, accessor_name).set(object_list) + self._inner.m2m_data = None + + return _SchemaAwareDeserialized(inner) + def save(self, *args, **kwargs): is_new = self._state.adding From e425d0c67b1007563e263395557b0eea6c29ae44 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 15:42:27 -0700 Subject: [PATCH 005/115] get merge working --- netbox_custom_objects/utilities.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox_custom_objects/utilities.py b/netbox_custom_objects/utilities.py index 7a16cf4f..45fdac3c 100644 --- a/netbox_custom_objects/utilities.py +++ b/netbox_custom_objects/utilities.py @@ -108,4 +108,3 @@ def generate_model(*args, **kwargs): apps.clear_cache = apps.clear_cache return model - From de5840474c0771accc6b6afd29457ba412cc760f Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 16:50:46 -0700 Subject: [PATCH 006/115] test updates --- netbox_custom_objects/models.py | 129 +++++++++++++++++++++++++++- netbox_custom_objects/tests/base.py | 2 + 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index ba54dda1..aa486f25 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -86,6 +86,59 @@ def _get_schema_connection(): return connection +def _apply_deferred_co_field(field_instance): + """ + Apply any deferred CO field values after a column is added to the DB. + + Called by CustomObjectTypeField.save() after schema_editor.add_field() so that + custom object rows inserted before their columns existed (squash merge ordering) + receive their correct values via a raw UPDATE. + + ``CustomObject._deferred_field_data`` has the shape:: + + {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} + + For TYPE_OBJECT fields the postchange_data key is ``{name}`` but the DB column + is ``{name}_id`` — this function maps accordingly. + For TYPE_MULTIOBJECT fields there is no column on the main table, so they are + skipped entirely. + """ + from extras.choices import CustomFieldTypeChoices + + # No deferred data at all — fast path. + if not CustomObject._deferred_field_data: + return + + cot = field_instance.custom_object_type + table_name = cot.get_database_table_name() + per_table = CustomObject._deferred_field_data.get(table_name) + if not per_table: + return + + # M2M has no column on the main table — nothing to UPDATE. + if field_instance.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + return + + # For TYPE_OBJECT the data key is the field name but the DB column ends with _id. + data_key = field_instance.name + if field_instance.type == CustomFieldTypeChoices.TYPE_OBJECT: + col_name = f'{field_instance.name}_id' + else: + col_name = field_instance.name + + schema_conn = _get_schema_connection() + + with schema_conn.cursor() as cursor: + for co_pk, entry in per_table.items(): + value = entry['data'].get(data_key) + if value is None: + continue + cursor.execute( + f'UPDATE "{table_name}" SET "{col_name}" = %s WHERE id = %s', + [value, co_pk], + ) + + class CustomObject( BookmarksMixin, ChangeLoggingMixin, @@ -115,9 +168,75 @@ class CustomObject( objects = RestrictedQuerySet.as_manager() + # Pending custom-field values for CO rows created before their columns existed. + # Populated by deserialize_object(); consumed by _apply_deferred_co_field(). + # Format: {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} + _deferred_field_data: dict = {} + class Meta: abstract = True + @classmethod + def deserialize_object(cls, data, pk=None): + """ + Hook called by ObjectChange.apply() for CREATE actions. + + The squash merge strategy may apply a CO's CREATE before its + CustomObjectTypeField rows are in main (the dependency graph has no FK + from CO to fields). When that happens, the standard Django + deserialization would INSERT the CO with NULL custom-field values + because the columns don't exist yet. + + This hook: + 1. Deserializes the CO using a fresh model (re-queried from DB). + 2. Does save_base(raw=True) as normal. + 3. Stores the full postchange_data in _deferred_field_data so that + CustomObjectTypeField.save() can UPDATE the row after each column is + added (handles the squash ordering case). + """ + from utilities.serialization import deserialize_object as _deserialize + + # Derive the COT primary key from the model class name (e.g. 'Table1Model' → 1) + model_name = cls.__name__ # e.g. 'Table1Model' + try: + cot_id = int(model_name.lower().replace('table', '').replace('model', '')) + except ValueError: + # Not a generated model name — fall back to standard deserialization. + return _deserialize(cls, data, pk=pk) + + # Refresh the model cache so we pick up any fields already applied to main. + # (In the squash case the cache may still point to a zero-field model.) + from netbox_custom_objects.models import CustomObjectType as _COT # noqa: F401 + _COT.clear_model_cache(cot_id) + try: + cot = _COT.objects.get(pk=cot_id) + fresh_model = cot.get_model() + except _COT.DoesNotExist: + fresh_model = cls + + inner = _deserialize(fresh_model, data, pk=pk) + obj = inner.object + obj_pk = pk if pk is not None else obj.pk + table_name = fresh_model._meta.db_table + full_data = dict(data) + + class _Deserialized: + object = obj + + def save(self, using=None, **_kwargs): + from django.db import DEFAULT_DB_ALIAS + _using = using or DEFAULT_DB_ALIAS + models.Model.save_base(obj, using=_using, raw=True) + # Register full data for deferred column updates (squash ordering fix). + if table_name not in cls._deferred_field_data: + cls._deferred_field_data[table_name] = {} + cls._deferred_field_data[table_name][obj_pk] = { + 'using': _using, + 'data': full_data, + } + + return _Deserialized() + def __str__(self): # Find the field with primary=True and return that field's "name" as the name of the object primary_field = self._field_objects.get(self._primary_field_id, None) @@ -664,7 +783,7 @@ def _ensure_field_fk_constraint(self, model, field_name): related_table = related_model._meta.db_table column_name = model_field.column - with connection.cursor() as cursor: + with _get_schema_connection().cursor() as cursor: # Drop existing FK constraint if it exists # Query for existing constraints cursor.execute(""" @@ -832,6 +951,13 @@ def delete(self, *args, **kwargs): super().delete(*args, **kwargs) if not in_branch: + # ChangeDiff has a PROTECT FK to ContentType/ObjectType — delete those + # records first so object_type.delete() is not blocked. + try: + from netbox_branching.models import ChangeDiff + ChangeDiff.objects.filter(object_type=object_type).delete() + except ImportError: + pass # Temporarily disconnect the pre_delete handler to skip the ObjectType deletion # TODO: Remove this disconnect/reconnect after ObjectType has been exempted from handle_deleted_object pre_delete.disconnect(handle_deleted_object) @@ -1764,6 +1890,7 @@ def save(self, *args, **kwargs): with schema_conn.schema_editor() as schema_editor: if self._state.adding: schema_editor.add_field(model, model_field) + _apply_deferred_co_field(self) if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: field_type.create_m2m_table(self, model, self.name, schema_conn=schema_conn) else: diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index ae48fc57..c9295ce3 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -15,6 +15,8 @@ class TransactionCleanupMixin: """ def tearDown(self): + from netbox_custom_objects.models import CustomObject + CustomObject._deferred_field_data.clear() for cot in CustomObjectType.objects.all(): try: cot.delete() From a9c0219fd9251d9193fc60b55dd45340169bc298 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Apr 2026 17:03:01 -0700 Subject: [PATCH 007/115] add branching tests --- netbox_custom_objects/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index d6d5aa1b..4f4f497b 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -29,7 +29,7 @@ from extras.choices import CustomFieldTypeChoices from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.dynamic_forms import build_filterset_form_class -from netbox_custom_objects.utilities import extract_cot_id_from_model_name, is_in_branch +from netbox_custom_objects.utilities import extract_cot_id_from_model_name logger = logging.getLogger("netbox_custom_objects.views") From f24dcaa91cdc178edd2c5aaa801c39aced6a7fdd Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 08:32:39 -0700 Subject: [PATCH 008/115] fix tests --- netbox_custom_objects/models.py | 32 ++++++++++++++++++++++++----- netbox_custom_objects/tests/base.py | 8 ++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 34fd0296..69634b13 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -376,6 +376,9 @@ def __str__(self): return self.display_name def clean(self): + # Guard against None (can arrive via update_object during branch revert) + if self.custom_field_data is None: + self.custom_field_data = {} super().clean() if not self.slug: @@ -807,7 +810,9 @@ def _ensure_field_fk_constraint(self, model, field_name): constraint_name = row[0] cursor.execute(f'ALTER TABLE "{table_name}" DROP CONSTRAINT IF EXISTS "{constraint_name}"') - # Create new FK constraint with ON DELETE CASCADE + # Create new FK constraint with ON DELETE CASCADE. + # Not DEFERRABLE: a deferred constraint leaves pending trigger events that block + # subsequent ALTER TABLE calls (e.g. during branch revert remove_field). constraint_name = f"{table_name}_{column_name}_fk_cascade" cursor.execute(f""" ALTER TABLE "{table_name}" @@ -815,7 +820,6 @@ def _ensure_field_fk_constraint(self, model, field_name): FOREIGN KEY ("{column_name}") REFERENCES "{related_table}" ("id") ON DELETE CASCADE - DEFERRABLE INITIALLY DEFERRED """) def _ensure_all_fk_constraints(self, model): @@ -1012,6 +1016,9 @@ def custom_object_type_post_save_handler(sender, instance, created, **kwargs): app_label=APP_LABEL, model=content_type_name ) + # Snapshot before modifying so change logging records a correct pre-state. + # Without this, diff()['pre'] would set all fields to None during branch revert. + instance.snapshot() instance.object_type = ct instance.save() @@ -2024,7 +2031,10 @@ def save(self, *args, **kwargs): # Clear and refresh the model cache for this CustomObjectType when a field is modified self.custom_object_type.clear_model_cache(self.custom_object_type.id) - # Update parent's cache_timestamp to invalidate cache across all workers + # Update parent's cache_timestamp to invalidate cache across all workers. + # snapshot() must be called first so that change logging has a correct pre-state; + # without it, diff()['pre'] would set ALL fields to None during branch revert. + self.custom_object_type.snapshot() self.custom_object_type.save(update_fields=['cache_timestamp']) super().save(*args, **kwargs) @@ -2062,18 +2072,30 @@ def delete(self, *args, **kwargs): apps = model._meta.apps through_model = apps.get_model(APP_LABEL, self.through_model_name) schema_editor.delete_model(through_model) + # Flush any pending DEFERRABLE FK trigger events before ALTER TABLE; + # otherwise PostgreSQL raises "pending trigger events" when removing a FK field. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') schema_editor.remove_field(model, model_field) # Clear the model cache for this CustomObjectType when a field is deleted self.custom_object_type.clear_model_cache(self.custom_object_type.id) - # Update parent's cache_timestamp to invalidate cache across all workers + # Update parent's cache_timestamp to invalidate cache across all workers. + # snapshot() must be called first so that change logging has a correct pre-state. + self.custom_object_type.snapshot() self.custom_object_type.save(update_fields=['cache_timestamp']) super().delete(*args, **kwargs) + # Regenerate and re-register the model so the app registry no longer includes + # the removed field. During squash revert the squash strategy may try to query + # CO rows (model.objects.get(pk=...)) after undoing this field but before undoing + # the CO itself. If the stale model class is still in the app registry it will + # include the now-absent column in its SELECT, causing ProgrammingError. + updated_model = self.custom_object_type.get_model() + # Reregister SearchIndex with new set of searchable fields - self.custom_object_type.register_custom_object_search_index(model) + self.custom_object_type.register_custom_object_search_index(updated_model) # Reindex all objects of this type since a searchable field was removed if self.search_weight > 0: diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index c9295ce3..55c07188 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -15,13 +15,21 @@ class TransactionCleanupMixin: """ def tearDown(self): + from core.models import ObjectChange from netbox_custom_objects.models import CustomObject + # Flush deferred CO field data so it doesn't bleed into the next test. CustomObject._deferred_field_data.clear() + # Delete COTs and their backing tables before the DB flush. for cot in CustomObjectType.objects.all(): try: cot.delete() except Exception as exc: print(f"WARNING: tearDown could not delete COT {cot.pk}: {exc}") + # Remove any ObjectChange records created during the test (merge/revert creates + # them in main with the test user's ID). If left in place, the serialized_rollback + # snapshot accumulates them and restoring it after the next flush produces FK + # violations (user referenced by ObjectChange no longer exists). + ObjectChange.objects.all().delete() super().tearDown() From ed9ec9e873a85380d82a838f2c8b103531be9f8f Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 08:52:19 -0700 Subject: [PATCH 009/115] refactor --- netbox_custom_objects/branching.py | 90 ++--------- netbox_custom_objects/models.py | 230 +++++++++++++++-------------- 2 files changed, 131 insertions(+), 189 deletions(-) diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 06db2033..651f4ac6 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -21,7 +21,7 @@ import logging -from django.db import connections, models +from django.db import connections logger = logging.getLogger('netbox_custom_objects.branching') @@ -151,12 +151,13 @@ def on_branch_migrated(sender, branch, user, **kwargs): exists in both, with different ``name`` values) rather than being treated as an unrelated delete + add. """ - from extras.choices import CustomFieldTypeChoices from netbox_branching.utilities import activate_branch - from netbox_custom_objects.constants import APP_LABEL - from netbox_custom_objects.field_types import FIELD_TYPE_CLASS - from netbox_custom_objects.models import CustomObjectType - from netbox_custom_objects.utilities import generate_model + from netbox_custom_objects.models import ( + CustomObjectType, + _schema_add_field, + _schema_alter_field, + _schema_remove_field, + ) branch_connection = connections[branch.connection_name] @@ -209,78 +210,17 @@ def on_branch_migrated(sender, branch, user, **kwargs): with branch_connection.schema_editor() as schema_editor: - # ── add_field ──────────────────────────────────────────────────── for fi in to_add: logger.debug('add_field %r on %s', fi.name, cot) - ft = FIELD_TYPE_CLASS[fi.type]() - mf = ft.get_model_field(fi) - mf.contribute_to_class(model, fi.name) - schema_editor.add_field(model, mf) - if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - ft.create_m2m_table(fi, model, fi.name, schema_conn=branch_connection) - - # ── remove_field ───────────────────────────────────────────────── + _schema_add_field(fi, model, schema_editor, branch_connection) + for fi in to_remove: logger.debug('remove_field %r on %s', fi.name, cot) - ft = FIELD_TYPE_CLASS[fi.type]() - mf = ft.get_model_field(fi) - mf.contribute_to_class(model, fi.name) - if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - through_table = f'custom_objects_{cot.pk}_{fi.name}' - if through_table in branch_tables: - ThroughMeta = type( - 'Meta', (), - {'db_table': through_table, 'app_label': APP_LABEL, 'managed': True}, - ) - through_model = type( - f'_TempThrough_{through_table}', - (models.Model,), - {'Meta': ThroughMeta, '__module__': 'netbox_custom_objects.models'}, - ) - schema_editor.delete_model(through_model) - schema_editor.remove_field(model, mf) - - # ── alter_field ────────────────────────────────────────────────── + _schema_remove_field(fi, model, schema_editor, existing_tables=branch_tables) + for old_fi, new_fi in to_alter: - logger.debug( - 'alter_field %r → %r on %s', - old_fi.name, new_fi.name, cot, + logger.debug('alter_field %r → %r on %s', old_fi.name, new_fi.name, cot) + _schema_alter_field( + old_fi, new_fi, model, schema_editor, branch_connection, + existing_tables=branch_tables, ) - old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) - new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) - old_mf.contribute_to_class(model, old_fi.name) - new_mf.contribute_to_class(model, new_fi.name) - - # When a MULTIOBJECT field is renamed, the through table must - # be renamed first (same logic as CustomObjectTypeField.save()). - if ( - new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT - and old_fi.name != new_fi.name - ): - old_through = f'custom_objects_{cot.pk}_{old_fi.name}' - new_through = f'custom_objects_{cot.pk}_{new_fi.name}' - if old_through in branch_tables: - OldThroughMeta = type( - 'Meta', (), - {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, - ) - old_through_model = generate_model( - f'_TempOldThrough_{old_through}', - (models.Model,), - { - '__module__': 'netbox_custom_objects.models', - 'Meta': OldThroughMeta, - 'id': models.AutoField(primary_key=True), - 'source': models.ForeignKey( - model, on_delete=models.CASCADE, - db_column='source_id', related_name='+', - ), - 'target': models.ForeignKey( - model, on_delete=models.CASCADE, - db_column='target_id', related_name='+', - ), - }, - ) - schema_editor.alter_db_table(old_through_model, old_through, new_through) - - schema_editor.alter_field(model, old_mf, new_mf) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 69634b13..00354961 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -139,6 +139,119 @@ def _apply_deferred_co_field(field_instance): ) +def _schema_add_field(fi, model, schema_editor, schema_conn): + """ + Issue ``add_field`` against the physical schema for *fi*. + + Handles through-table creation for MULTIOBJECT fields. Does NOT apply + deferred CO field data — callers that need that (squash merge context) must + call ``_apply_deferred_co_field(fi)`` separately after this returns. + """ + ft = FIELD_TYPE_CLASS[fi.type]() + mf = ft.get_model_field(fi) + mf.contribute_to_class(model, fi.name) + schema_editor.add_field(model, mf) + if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + ft.create_m2m_table(fi, model, fi.name, schema_conn=schema_conn) + + +def _schema_remove_field(fi, model, schema_editor, existing_tables=None): + """ + Issue ``remove_field`` against the physical schema for *fi*. + + For MULTIOBJECT fields the through table is dropped first. When + *existing_tables* is a pre-fetched list only tables present in it are + dropped; when it is ``None`` (main-schema context) the drop is always + attempted. + + Always issues ``SET CONSTRAINTS ALL IMMEDIATE`` before ``remove_field`` to + flush any DEFERRABLE FK trigger events that would otherwise cause PostgreSQL + to reject the subsequent ALTER TABLE. + """ + ft = FIELD_TYPE_CLASS[fi.type]() + mf = ft.get_model_field(fi) + mf.contribute_to_class(model, fi.name) + + if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + through_table = fi.through_table_name + if existing_tables is None or through_table in existing_tables: + through_meta = type( + 'Meta', (), + {'db_table': through_table, 'app_label': APP_LABEL, 'managed': True}, + ) + through_model = type( + f'_TempThrough_{through_table}', + (models.Model,), + {'Meta': through_meta, '__module__': 'netbox_custom_objects.models'}, + ) + schema_editor.delete_model(through_model) + + # Flush any pending DEFERRABLE FK trigger events before ALTER TABLE; + # otherwise PostgreSQL raises "pending trigger events" when removing a FK field. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') + schema_editor.remove_field(model, mf) + + +def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, existing_tables=None): + """ + Issue ``alter_field`` against the physical schema, updating *old_fi* to *new_fi*. + + For MULTIOBJECT fields whose name changes the through table is renamed before + ``alter_field`` is called. When the old through table is absent (e.g. the + branch has never seen this field) the new through table is created from scratch + instead. + + *existing_tables* — optional pre-fetched table name list from the target + connection. When given, through-table operations are guarded by membership + checks. When ``None`` (main-schema context) the schema_conn is introspected + once on demand. + """ + old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) + new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) + old_mf.contribute_to_class(model, old_fi.name) + new_mf.contribute_to_class(model, new_fi.name) + + if ( + new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + and old_fi.name != new_fi.name + ): + old_through = old_fi.through_table_name + new_through = new_fi.through_table_name + + tables = existing_tables + if tables is None: + with schema_conn.cursor() as cursor: + tables = schema_conn.introspection.table_names(cursor) + + if old_through in tables: + old_through_meta = type( + 'Meta', (), + {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, + ) + old_through_model = generate_model( + f'_TempOldThrough_{old_through}', + (models.Model,), + { + '__module__': 'netbox_custom_objects.models', + 'Meta': old_through_meta, + 'id': models.AutoField(primary_key=True), + 'source': models.ForeignKey( + model, on_delete=models.CASCADE, db_column='source_id', related_name='+', + ), + 'target': models.ForeignKey( + model, on_delete=models.CASCADE, db_column='target_id', related_name='+', + ), + }, + ) + schema_editor.alter_db_table(old_through_model, old_through, new_through) + else: + # Old through table absent — create the new one from scratch + ft = FIELD_TYPE_CLASS[new_fi.type]() + ft.create_m2m_table(new_fi, model, new_fi.name, schema_conn=schema_conn) + + schema_editor.alter_field(model, old_mf, new_mf) + + class CustomObject( BookmarksMixin, ChangeLoggingMixin, @@ -1901,115 +2014,14 @@ def save(self, *args, **kwargs): # editor operations target the branch schema rather than main. schema_conn = _get_schema_connection() - field_type = FIELD_TYPE_CLASS[self.type]() - model_field = field_type.get_model_field(self) model = self.custom_object_type.get_model() - model_field.contribute_to_class(model, self.name) with schema_conn.schema_editor() as schema_editor: if self._state.adding: - schema_editor.add_field(model, model_field) + _schema_add_field(self, model, schema_editor, schema_conn) _apply_deferred_co_field(self) - if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - field_type.create_m2m_table(self, model, self.name, schema_conn=schema_conn) else: - old_field = field_type.get_model_field(self.original) - old_field.contribute_to_class(model, self._original_name) - - # Special handling for MultiObject fields when the name changes - if ( - self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT - and self.name != self._original_name - ): - # For renamed MultiObject fields, we just need to rename the through table - old_through_table_name = self.original.through_table_name - new_through_table_name = self.through_table_name - - # Check if old through table exists - with schema_conn.cursor() as cursor: - tables = schema_conn.introspection.table_names(cursor) - old_table_exists = old_through_table_name in tables - - if old_table_exists: - # Create temporary models to represent the old and new through table states - old_through_meta = type( - "Meta", - (), - { - "db_table": old_through_table_name, - "app_label": APP_LABEL, - "managed": True, - }, - ) - old_through_model = generate_model( - f"TempOld{self.original.through_model_name}", - (models.Model,), - { - "__module__": "netbox_custom_objects.models", - "Meta": old_through_meta, - "id": models.AutoField(primary_key=True), - "source": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="source_id", - related_name="+", - ), - "target": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="target_id", - related_name="+", - ), - }, - ) - - new_through_meta = type( - "Meta", - (), - { - "db_table": new_through_table_name, - "app_label": APP_LABEL, - "managed": True, - }, - ) - new_through_model = generate_model( - f"TempNew{self.through_model_name}", - (models.Model,), - { - "__module__": "netbox_custom_objects.models", - "Meta": new_through_meta, - "id": models.AutoField(primary_key=True), - "source": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="source_id", - related_name="+", - ), - "target": models.ForeignKey( - model, - on_delete=models.CASCADE, - db_column="target_id", - related_name="+", - ), - }, - ) - new_through_model # To silence ruff error - - # Rename the table using Django's schema editor - schema_editor.alter_db_table( - old_through_model, - old_through_table_name, - new_through_table_name, - ) - else: - # No old table exists, create the new through table - field_type.create_m2m_table(self, model, self.name, schema_conn=schema_conn) - - # Alter the field normally (this updates the field definition) - schema_editor.alter_field(model, old_field, model_field) - else: - # Normal field alteration - schema_editor.alter_field(model, old_field, model_field) + _schema_alter_field(self.original, self, model, schema_editor, schema_conn) # Ensure FK constraints are properly created for OBJECT fields with CASCADE behavior should_ensure_fk = False @@ -2062,20 +2074,10 @@ def delete(self, *args, **kwargs): # Use the branch connection when operating inside a branch. schema_conn = _get_schema_connection() - field_type = FIELD_TYPE_CLASS[self.type]() - model_field = field_type.get_model_field(self) model = self.custom_object_type.get_model() - model_field.contribute_to_class(model, self.name) with schema_conn.schema_editor() as schema_editor: - if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - apps = model._meta.apps - through_model = apps.get_model(APP_LABEL, self.through_model_name) - schema_editor.delete_model(through_model) - # Flush any pending DEFERRABLE FK trigger events before ALTER TABLE; - # otherwise PostgreSQL raises "pending trigger events" when removing a FK field. - schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') - schema_editor.remove_field(model, model_field) + _schema_remove_field(self, model, schema_editor) # Clear the model cache for this CustomObjectType when a field is deleted self.custom_object_type.clear_model_cache(self.custom_object_type.id) From 8cd04a00906916b502cb43fedd6a6a84e74cc0e1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 09:10:12 -0700 Subject: [PATCH 010/115] test cleanup --- netbox_custom_objects/filtersets.py | 3 +++ netbox_custom_objects/tests/test_field_types.py | 7 ++++--- netbox_custom_objects/tests/test_filtersets.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/netbox_custom_objects/filtersets.py b/netbox_custom_objects/filtersets.py index 740e2075..f5cbced5 100644 --- a/netbox_custom_objects/filtersets.py +++ b/netbox_custom_objects/filtersets.py @@ -3,6 +3,7 @@ from django.contrib.postgres.fields import ArrayField from django.db.models import JSONField, Q from django.utils.dateparse import parse_date, parse_datetime +from django.utils.timezone import make_aware, is_aware from extras.choices import CustomFieldTypeChoices from netbox.filtersets import NetBoxModelFilterSet @@ -89,6 +90,8 @@ def search(self, queryset, name, value): elif field.type == CustomFieldTypeChoices.TYPE_DATETIME: parsed = parse_datetime(value) if parsed is not None: + if not is_aware(parsed): + parsed = make_aware(parsed) q |= Q(**{f"{field.name}__exact": parsed}) if not q: return queryset.none() diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 4dff11da..c23c0f9e 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -356,11 +356,11 @@ def test_datetime_field_creation(self): name="created_datetime", label="Created DateTime", type="datetime", - default="2023-01-01T12:00:00" + default="2023-01-01T12:00:00+00:00" ) self.assertEqual(field.type, "datetime") - self.assertEqual(field.default, "2023-01-01T12:00:00") + self.assertEqual(field.default, "2023-01-01T12:00:00+00:00") def test_datetime_field_validation(self): """Test datetime field validation.""" @@ -390,8 +390,9 @@ def test_datetime_field_model_generation(self): type="datetime" ) + import datetime as dt model = self.custom_object_type.get_model() - test_datetime = datetime(2023, 1, 1, 12, 0, 0) + test_datetime = datetime(2023, 1, 1, 12, 0, 0, tzinfo=dt.timezone.utc) instance = model.objects.create(name="Test", created_datetime=test_datetime) self.assertEqual(instance.created_datetime, test_datetime) diff --git a/netbox_custom_objects/tests/test_filtersets.py b/netbox_custom_objects/tests/test_filtersets.py index 2965c5c9..7d903f54 100644 --- a/netbox_custom_objects/tests/test_filtersets.py +++ b/netbox_custom_objects/tests/test_filtersets.py @@ -573,8 +573,8 @@ def setUpTestData(cls): ) model = cls.cot.get_model() - cls.obj_morning = model.objects.create(ts=datetime.datetime(2025, 3, 10, 9, 0, 0)) - cls.obj_evening = model.objects.create(ts=datetime.datetime(2025, 3, 10, 18, 30, 0)) + cls.obj_morning = model.objects.create(ts=datetime.datetime(2025, 3, 10, 9, 0, 0, tzinfo=datetime.timezone.utc)) + cls.obj_evening = model.objects.create(ts=datetime.datetime(2025, 3, 10, 18, 30, 0, tzinfo=datetime.timezone.utc)) def _search(self, value): model = self.cot.get_model() From 0aed016e232527751c1524dcda83877a39a9879c Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 09:10:48 -0700 Subject: [PATCH 011/115] test cleanup --- netbox_custom_objects/tests/test_filtersets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/netbox_custom_objects/tests/test_filtersets.py b/netbox_custom_objects/tests/test_filtersets.py index 7d903f54..b33ed65f 100644 --- a/netbox_custom_objects/tests/test_filtersets.py +++ b/netbox_custom_objects/tests/test_filtersets.py @@ -573,8 +573,9 @@ def setUpTestData(cls): ) model = cls.cot.get_model() - cls.obj_morning = model.objects.create(ts=datetime.datetime(2025, 3, 10, 9, 0, 0, tzinfo=datetime.timezone.utc)) - cls.obj_evening = model.objects.create(ts=datetime.datetime(2025, 3, 10, 18, 30, 0, tzinfo=datetime.timezone.utc)) + utc = datetime.timezone.utc + cls.obj_morning = model.objects.create(ts=datetime.datetime(2025, 3, 10, 9, 0, 0, tzinfo=utc)) + cls.obj_evening = model.objects.create(ts=datetime.datetime(2025, 3, 10, 18, 30, 0, tzinfo=utc)) def _search(self, value): model = self.cot.get_model() From f47ac66b417b652dde64412be6b31d7a7262e5e7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 10:08:08 -0700 Subject: [PATCH 012/115] test cleanup --- netbox_custom_objects/tests/base.py | 64 +++++++++++++++++++ netbox_custom_objects/tests/test_branching.py | 9 +-- netbox_custom_objects/tests/test_deletion.py | 16 +---- 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 55c07188..73cb3bd5 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -1,12 +1,55 @@ # Test utilities for netbox_custom_objects plugin +from django.apps import apps as django_apps +from django.contrib.contenttypes.management import create_contenttypes from django.test import Client from core.models import ObjectType from extras.models import CustomFieldChoiceSet from utilities.testing import create_test_user +from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField +def _recreate_contenttypes(): + """Recreate ContentType rows for all installed apps using get_or_create. + + Called after a TransactionTestCase flush so that subsequent test classes — + whether TransactionTestCase or regular TestCase — can look up ContentTypes. + Using create_contenttypes (get_or_create) avoids the duplicate-key + violations that serialized_rollback causes in the parallel test runner. + """ + for app_config in django_apps.get_app_configs(): + create_contenttypes(app_config, verbosity=0) + + +def _purge_stale_generated_models(): + """Remove dynamically generated CustomObject models from the app registry. + + Regular TestCase subclasses wrap each test in a transaction that is rolled + back at the end. Rolling back the transaction drops any tables created by + DDL inside the test (e.g. CREATE TABLE custom_objects_1), but Django's + in-memory model registry is NOT rolled back. The stale model entry then + causes problems for subsequent TransactionTestCase tests: + + - netbox-branching's get_tables_to_replicate() iterates over registered + models to build the list of tables to clone into the branch schema. + A stale model entry makes it try to COPY a table that no longer exists. + - Django's cascade-delete collector queries non-existent tables. + + Calling this in setUp() of every TransactionTestCase prevents both failure + modes. + """ + stale = [ + name + for name, model in list(django_apps.all_models.get(APP_LABEL, {}).items()) + if getattr(model, '_generated_table_model', False) + ] + for name in stale: + django_apps.all_models[APP_LABEL].pop(name, None) + if stale: + django_apps.clear_cache() + + class TransactionCleanupMixin: """Mixin for TransactionTestCase subclasses that create CustomObjectType instances. @@ -14,6 +57,14 @@ class TransactionCleanupMixin: database flush that TransactionTestCase performs between tests. """ + def setUp(self): + # Purge stale in-memory model registrations left by earlier TestCase + # classes whose rolled-back transactions dropped the backing tables. + # Must run before any code that iterates the model registry (e.g. + # netbox-branching's get_tables_to_replicate() during provisioning). + _purge_stale_generated_models() + super().setUp() + def tearDown(self): from core.models import ObjectChange from netbox_custom_objects.models import CustomObject @@ -32,6 +83,19 @@ def tearDown(self): ObjectChange.objects.all().delete() super().tearDown() + def _fixture_teardown(self): + """Flush tables and restore ContentTypes for the next test class. + + TransactionTestCase._fixture_teardown() TRUNCATEs all tables after each + test. Any TestCase class that follows on the same worker then finds no + ContentTypes and fails trying to look up ObjectType rows. Recreating + them here (idempotently, via get_or_create) avoids that without the + duplicate-key violations that serialized_rollback=True causes when the + parallel runner tries to INSERT rows that already exist. + """ + super()._fixture_teardown() + _recreate_contenttypes() + class CustomObjectsTestCase: """ diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 85ff3779..364632bd 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -28,7 +28,7 @@ HAS_BRANCHING = False from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField -from netbox_custom_objects.tests.base import TransactionCleanupMixin +from netbox_custom_objects.tests.base import TransactionCleanupMixin, _recreate_contenttypes User = get_user_model() @@ -83,9 +83,10 @@ class IterativeBranchingTestCase(BaseBranchingTests, TransactionTestCase): """ MERGE_STRATEGY = None - serialized_rollback = True def setUp(self): + super().setUp() # → TransactionCleanupMixin.setUp() → _purge_stale_generated_models() + _recreate_contenttypes() self.user = User.objects.create_user(username='testuser') self.request = _make_request(self.user) @@ -327,9 +328,9 @@ class BranchSyncTestCase(TransactionCleanupMixin, TransactionTestCase): available in the branch after sync. """ - serialized_rollback = True - def setUp(self): + super().setUp() # → TransactionCleanupMixin.setUp() → _purge_stale_generated_models() + _recreate_contenttypes() self.user = User.objects.create_user(username='testuser') self.request = _make_request(self.user) diff --git a/netbox_custom_objects/tests/test_deletion.py b/netbox_custom_objects/tests/test_deletion.py index 047aee40..4dcf5f8d 100644 --- a/netbox_custom_objects/tests/test_deletion.py +++ b/netbox_custom_objects/tests/test_deletion.py @@ -21,21 +21,9 @@ class DeletionTestCase(TransactionCleanupMixin, CustomObjectsTestCase, Transacti """Test deletion scenarios with cascading effects.""" def setUp(self): - """Purge stale dynamic models left in the app registry by earlier TestCase - classes whose setUpTestData transaction was rolled back (dropping the backing - tables). Leaving them registered causes Django's cascade-delete collector to - query non-existent tables when a related core object is deleted. - """ + # TransactionCleanupMixin.setUp() purges stale generated models and + # CustomObjectsTestCase.setUp() creates the test user and client. super().setUp() - stale = [ - name - for name, model in list(django_apps.all_models.get(APP_LABEL, {}).items()) - if getattr(model, '_generated_table_model', False) - ] - for name in stale: - django_apps.all_models[APP_LABEL].pop(name, None) - if stale: - django_apps.clear_cache() # ------------------------------------------------------------------ # Helpers From 539416dad19d6a66332aa50cb9cd0008d53e5165 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 10:35:45 -0700 Subject: [PATCH 013/115] update drift check --- netbox_custom_objects/branching.py | 8 ++++++-- netbox_custom_objects/models.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 651f4ac6..b969b466 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -126,12 +126,16 @@ def _fields_schema_differ(branch_f, main_f): Return True if the two ``CustomObjectTypeField`` instances differ in any attribute that affects the physical DB column, meaning an ALTER TABLE is needed to bring the branch schema up to date. + + Excluded (application-level only, no DB impact): + - required: enforced by forms/serializers; all field types use null=True + regardless, so required never maps to a NOT NULL column constraint. + - default: Python-level default applied by the ORM, not a DB DEFAULT + clause; changing it on an existing column needs no ALTER TABLE. """ return ( branch_f.name != main_f.name or branch_f.type != main_f.type - or branch_f.required != main_f.required - or branch_f.default != main_f.default or branch_f.unique != main_f.unique or branch_f.related_object_type_id != main_f.related_object_type_id ) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 00354961..339d4e8d 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1036,7 +1036,12 @@ def has_branch_schema_drift(self, branch) -> bool: return True def _schema_key(f): - return (f.name, f.type, f.required, f.default, f.unique, f.related_object_type_id) + # Only attributes that affect the physical DB column are included. + # - required: enforced at form/serializer level only; all field types + # use null=True regardless, so required never maps to NOT NULL. + # - default: Python-level default, not a DB DEFAULT clause; changing + # it on an existing column requires no ALTER TABLE. + return (f.name, f.type, f.unique, f.related_object_type_id) return any( _schema_key(branch_fields[pk]) != _schema_key(main_fields[pk]) From 549f267234a424a008f766da0192dc13daf35785 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 10:46:20 -0700 Subject: [PATCH 014/115] cleanup --- netbox_custom_objects/__init__.py | 6 +++++- netbox_custom_objects/branching.py | 31 ++++++++++++++++++------------ netbox_custom_objects/models.py | 20 +++++++++---------- netbox_custom_objects/views.py | 9 +++++---- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 4cbf3a02..a661afda 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -1,4 +1,5 @@ import contextvars +import logging import sys import warnings @@ -228,7 +229,10 @@ def ready(self): from netbox_custom_objects.branching import check_pending_branch_migrations check_pending_branch_migrations() except Exception: - pass + logging.getLogger('netbox_custom_objects').exception( + 'check_pending_branch_migrations() failed at startup; ' + 'some branches may not be marked as PENDING_MIGRATIONS' + ) super().ready() diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index b969b466..8d5fb0b6 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -121,24 +121,31 @@ def on_custom_object_field_changed(sender, instance, **kwargs): ) +def _field_schema_key(f): + """ + Return the subset of ``CustomObjectTypeField`` attributes that affect the + physical DB column schema. + + Excluded (application-level only, no physical DB impact): + - required: all field types use null=True regardless; required never maps + to a NOT NULL column constraint. + - default: Python-level default applied by the ORM, not a SQL DEFAULT + clause; changing it on an existing column needs no ALTER TABLE. + + Used by both ``_fields_schema_differ()`` and + ``CustomObjectType.has_branch_schema_drift()`` to ensure the two callers + always agree on what constitutes a schema-affecting change. + """ + return (f.name, f.type, f.unique, f.related_object_type_id) + + def _fields_schema_differ(branch_f, main_f): """ Return True if the two ``CustomObjectTypeField`` instances differ in any attribute that affects the physical DB column, meaning an ALTER TABLE is needed to bring the branch schema up to date. - - Excluded (application-level only, no DB impact): - - required: enforced by forms/serializers; all field types use null=True - regardless, so required never maps to a NOT NULL column constraint. - - default: Python-level default applied by the ORM, not a DB DEFAULT - clause; changing it on an existing column needs no ALTER TABLE. """ - return ( - branch_f.name != main_f.name - or branch_f.type != main_f.type - or branch_f.unique != main_f.unique - or branch_f.related_object_type_id != main_f.related_object_type_id - ) + return _field_schema_key(branch_f) != _field_schema_key(main_f) def on_branch_migrated(sender, branch, user, **kwargs): diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 339d4e8d..b5987697 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -133,6 +133,10 @@ def _apply_deferred_co_field(field_instance): value = entry['data'].get(data_key) if value is None: continue + # table_name comes from get_database_table_name() (controlled by our + # code) and col_name from field.name, which is validated by the + # ^[a-z0-9_]+$ regex — no double-quote characters are possible, so + # the f-string interpolation is safe against SQL injection here. cursor.execute( f'UPDATE "{table_name}" SET "{col_name}" = %s WHERE id = %s', [value, co_pk], @@ -329,7 +333,6 @@ def deserialize_object(cls, data, pk=None): inner = _deserialize(fresh_model, data, pk=pk) obj = inner.object - obj_pk = pk if pk is not None else obj.pk table_name = fresh_model._meta.db_table full_data = dict(data) @@ -340,6 +343,9 @@ def save(self, using=None, **_kwargs): from django.db import DEFAULT_DB_ALIAS _using = using or DEFAULT_DB_ALIAS models.Model.save_base(obj, using=_using, raw=True) + # Read pk after save_base so that auto-assigned PKs are captured. + # (If pk was None before save_base, obj.pk is now the DB-assigned id.) + obj_pk = obj.pk # Register full data for deferred column updates (squash ordering fix). if table_name not in cls._deferred_field_data: cls._deferred_field_data[table_name] = {} @@ -1025,6 +1031,8 @@ def has_branch_schema_drift(self, branch) -> bool: except ImportError: return False + from netbox_custom_objects.branching import _field_schema_key + main_fields = {f.pk: f for f in self.fields.all()} with activate_branch(branch): branch_fields = {f.pk: f for f in self.fields.all()} @@ -1035,16 +1043,8 @@ def has_branch_schema_drift(self, branch) -> bool: if main_pks != branch_pks: return True - def _schema_key(f): - # Only attributes that affect the physical DB column are included. - # - required: enforced at form/serializer level only; all field types - # use null=True regardless, so required never maps to NOT NULL. - # - default: Python-level default, not a DB DEFAULT clause; changing - # it on an existing column requires no ALTER TABLE. - return (f.name, f.type, f.unique, f.related_object_type_id) - return any( - _schema_key(branch_fields[pk]) != _schema_key(main_fields[pk]) + _field_schema_key(branch_fields[pk]) != _field_schema_key(main_fields[pk]) for pk in main_pks ) diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 4f4f497b..15ea6f47 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -529,10 +529,11 @@ def custom_init(self, *args, **kwargs): # Custom object type from netbox_custom_objects.models import CustomObjectType custom_object_type_id = extract_cot_id_from_model_name(content_type.model) - assert custom_object_type_id is not None, ( - f"Expected tablemodel name for {APP_LABEL} content type, " - f"got {content_type.model!r}" - ) + if custom_object_type_id is None: + raise ValueError( + f"Expected tablemodel name for {APP_LABEL} content type, " + f"got {content_type.model!r}" + ) custom_object_type = CustomObjectType.objects.get(pk=custom_object_type_id) model = custom_object_type.get_model(skip_object_fields=True) else: From 493aacbf436d6c86443ba38d86c1c6281020a697 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 11:00:57 -0700 Subject: [PATCH 015/115] cleanup --- netbox_custom_objects/branching.py | 34 ++++++++----------- netbox_custom_objects/tests/test_branching.py | 23 ++++++++++--- netbox_custom_objects/views.py | 12 ------- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 8d5fb0b6..98ca3ac7 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -77,15 +77,20 @@ def check_pending_branch_migrations(): def on_custom_object_field_changed(sender, instance, **kwargs): """ - Mark any READY branches that contain the affected custom object type's table - as PENDING_MIGRATIONS when a ``CustomObjectTypeField`` is created, modified, - or deleted in the main schema. + Mark all READY branches as PENDING_MIGRATIONS when a ``CustomObjectTypeField`` + is created, modified, or deleted in the main schema. This surfaces the pending state in the branching UI exactly like a normal Django-migration, prompting users to click "Migrate Branch". That action calls ``Branch.migrate()``, which fires ``on_branch_migrated`` and reconciles the physical column differences. + Branches provisioned before this COT's table existed will be over-marked as + PENDING_MIGRATIONS but ``on_branch_migrated`` will skip them (no table in + branch) so the extra marking is harmless. The alternative — opening a + connection per branch to check whether the table exists — adds O(branches) + latency to every field save, which is worse at scale. + Skipped when the change happens inside a branch context — the field edit only affects that branch's schema, not main, so no other branches need updating. """ @@ -99,25 +104,14 @@ def on_custom_object_field_changed(sender, instance, **kwargs): from netbox_branching.choices import BranchStatusChoices from netbox_branching.models import Branch - cot = instance.custom_object_type - db_table = cot.get_database_table_name() - - ready_branches = list(Branch.objects.filter(status=BranchStatusChoices.READY)) - to_update = [] - for branch in ready_branches: - branch_connection = connections[branch.connection_name] - with branch_connection.cursor() as cursor: - branch_tables = branch_connection.introspection.table_names(cursor) - if db_table in branch_tables: - branch.status = BranchStatusChoices.PENDING_MIGRATIONS - to_update.append(branch) - - if to_update: - Branch.objects.bulk_update(to_update, ['status']) + updated = Branch.objects.filter(status=BranchStatusChoices.READY).update( + status=BranchStatusChoices.PENDING_MIGRATIONS, + ) + if updated: logger.info( 'Marked %d branch(es) as PENDING_MIGRATIONS due to field changes on %s', - len(to_update), - cot, + updated, + instance.custom_object_type, ) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 364632bd..f5c7e0fe 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -9,6 +9,8 @@ in separate PostgreSQL schemas backed by distinct database connections that cannot be rolled back inside a single SAVEPOINT-based transaction. """ +import datetime +import decimal import time import unittest import uuid @@ -186,11 +188,13 @@ def test_comprehensive_merge_and_revert(self): Field types ----------- - text — plain VARCHAR column - integer — INTEGER column - boolean — BOOLEAN column - select — VARCHAR column with a ChoiceSet - object — ForeignKey column (to dcim.Site) + text — plain VARCHAR column + integer — INTEGER column + decimal — DECIMAL column (exercises numeric precision handling) + boolean — BOOLEAN column + datetime — TIMESTAMPTZ column (exercises timezone-aware handling) + select — VARCHAR column with a ChoiceSet + object — ForeignKey column (to dcim.Site) multiobject — M2M through-table (to dcim.Site) This exercises every distinct schema-editor operation @@ -216,6 +220,9 @@ def test_comprehensive_merge_and_revert(self): field_pks = {} co_pk = None + test_dt = datetime.datetime(2024, 6, 15, 12, 0, 0, tzinfo=datetime.timezone.utc) + test_decimal = decimal.Decimal('3.14') + # ── create inside branch ────────────────────────────────────────── with activate_branch(branch), event_tracking(request): choice_set = CustomFieldChoiceSet.objects.create( @@ -228,7 +235,9 @@ def test_comprehensive_merge_and_revert(self): field_specs = [ ('text_field', {'type': 'text'}), ('int_field', {'type': 'integer'}), + ('dec_field', {'type': 'decimal'}), ('bool_field', {'type': 'boolean'}), + ('dt_field', {'type': 'datetime'}), ('select_field', {'type': 'select', 'choice_set': choice_set}), ('obj_field', {'type': 'object', 'related_object_type': site_ot}), ('multi_field', {'type': 'multiobject', 'related_object_type': site_ot}), @@ -246,7 +255,9 @@ def test_comprehensive_merge_and_revert(self): co = Model.objects.create( text_field='hello', int_field=42, + dec_field=test_decimal, bool_field=True, + dt_field=test_dt, select_field='active', obj_field_id=site.pk, # multi_field left empty — through-table creation is still exercised @@ -284,7 +295,9 @@ def test_comprehensive_merge_and_revert(self): co_main = cot_main.get_model().objects.get(pk=co_pk) self.assertEqual(co_main.text_field, 'hello') self.assertEqual(co_main.int_field, 42) + self.assertEqual(co_main.dec_field, test_decimal) self.assertTrue(co_main.bool_field) + self.assertEqual(co_main.dt_field, test_dt) self.assertEqual(co_main.select_field, 'active') self.assertEqual(co_main.obj_field_id, site.pk) diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 15ea6f47..4af7494e 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -591,10 +591,6 @@ def custom_save(self, commit=True): return form_class - def get_extra_context(self, request, obj): - return { - } - @register_model_view(CustomObject, "delete") class CustomObjectDeleteView(generic.ObjectDeleteView): @@ -704,10 +700,6 @@ def get_form(self, queryset): return form - def get_extra_context(self, request): - return { - } - @register_model_view(CustomObject, "bulk_delete", path="delete", detail=False) class CustomObjectBulkDeleteView(CustomObjectTableMixin, generic.BulkDeleteView): @@ -794,10 +786,6 @@ def get_model_form(self, queryset): return form - def get_extra_context(self, request): - return { - } - class CustomObjectJournalView(ConditionalLoginRequiredMixin, View): """ From 0480531a338cb3883dbacc431f2db66d7917250c Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 11:01:08 -0700 Subject: [PATCH 016/115] cleanup --- .../0007_fix_object_field_fk_deferrable.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py diff --git a/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py b/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py new file mode 100644 index 00000000..c03b7cad --- /dev/null +++ b/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py @@ -0,0 +1,74 @@ +""" +Drop DEFERRABLE INITIALLY DEFERRED from FK constraints on custom object tables. + +Prior to this migration, _ensure_field_fk_constraint() created FK constraints +with DEFERRABLE INITIALLY DEFERRED. That attribute causes PostgreSQL to queue +trigger events that block subsequent ALTER TABLE calls (e.g. remove_field during +a branch revert), raising "cannot ALTER TABLE because it has pending trigger +events". + +This migration finds all DEFERRABLE FK constraints on tables whose names start +with "custom_objects_" and recreates them as non-DEFERRABLE with ON DELETE +CASCADE, matching the behaviour of the updated _ensure_field_fk_constraint(). +""" + +from django.db import migrations + + +def fix_deferrable_fk_constraints(apps, schema_editor): + """ + Re-create any DEFERRABLE FK constraints on custom object tables as + non-DEFERRABLE. Uses information_schema so no Django model loading + is required — safe to run during the migration pass even though dynamic + models are not yet registered. + """ + with schema_editor.connection.cursor() as cursor: + # Find all DEFERRABLE FK constraints on custom_objects_* tables. + cursor.execute(""" + SELECT + tc.table_name, + tc.constraint_name, + kcu.column_name, + ccu.table_name AS foreign_table_name + FROM information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + JOIN information_schema.referential_constraints AS rc + ON tc.constraint_name = rc.constraint_name + AND tc.table_schema = rc.constraint_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_name LIKE 'custom_objects\\_%%' + AND rc.is_deferrable = 'YES' + """) + rows = cursor.fetchall() + + for table_name, constraint_name, column_name, foreign_table in rows: + new_constraint_name = f'{table_name}_{column_name}_fk_cascade' + cursor.execute( + f'ALTER TABLE "{table_name}" DROP CONSTRAINT "{constraint_name}"' + ) + cursor.execute(f""" + ALTER TABLE "{table_name}" + ADD CONSTRAINT "{new_constraint_name}" + FOREIGN KEY ("{column_name}") + REFERENCES "{foreign_table}" ("id") + ON DELETE CASCADE + """) + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_custom_objects', '0006_customobjecttypefield_related_name_and_more'), + ] + + operations = [ + migrations.RunPython( + fix_deferrable_fk_constraints, + migrations.RunPython.noop, + ), + ] From a3b0d8a9b953cae02434e0d2988c0ffd2e4a61fa Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 11:04:01 -0700 Subject: [PATCH 017/115] fix squash merge --- netbox_custom_objects/models.py | 37 ++++++++++++++++++----------- netbox_custom_objects/tests/base.py | 6 ++--- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index b5987697..09315b6e 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1,3 +1,4 @@ +import contextvars import decimal import re import threading @@ -66,6 +67,14 @@ class UniquenessConstraintTestError(Exception): USER_TABLE_DATABASE_NAME_PREFIX = "custom_objects_" +# Per-context storage for CO field values deferred during squash merge. +# Using ContextVar instead of a class-level dict so that concurrent merges +# (different threads or coroutines) each get an isolated copy. +# Shape: {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} +_deferred_co_field_data: contextvars.ContextVar[dict | None] = contextvars.ContextVar( + '_deferred_co_field_data', default=None +) + def _get_schema_connection(): """ @@ -94,7 +103,7 @@ def _apply_deferred_co_field(field_instance): custom object rows inserted before their columns existed (squash merge ordering) receive their correct values via a raw UPDATE. - ``CustomObject._deferred_field_data`` has the shape:: + ``_deferred_co_field_data`` (ContextVar) has the shape:: {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} @@ -106,12 +115,13 @@ def _apply_deferred_co_field(field_instance): from extras.choices import CustomFieldTypeChoices # No deferred data at all — fast path. - if not CustomObject._deferred_field_data: + deferred = _deferred_co_field_data.get() + if not deferred: return cot = field_instance.custom_object_type table_name = cot.get_database_table_name() - per_table = CustomObject._deferred_field_data.get(table_name) + per_table = deferred.get(table_name) if not per_table: return @@ -285,11 +295,6 @@ class CustomObject( objects = RestrictedQuerySet.as_manager() - # Pending custom-field values for CO rows created before their columns existed. - # Populated by deserialize_object(); consumed by _apply_deferred_co_field(). - # Format: {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} - _deferred_field_data: dict = {} - class Meta: abstract = True @@ -307,9 +312,9 @@ def deserialize_object(cls, data, pk=None): This hook: 1. Deserializes the CO using a fresh model (re-queried from DB). 2. Does save_base(raw=True) as normal. - 3. Stores the full postchange_data in _deferred_field_data so that - CustomObjectTypeField.save() can UPDATE the row after each column is - added (handles the squash ordering case). + 3. Stores the full postchange_data in _deferred_co_field_data (ContextVar) + so that CustomObjectTypeField.save() can UPDATE the row after each + column is added (handles the squash ordering case). """ from utilities.serialization import deserialize_object as _deserialize from netbox_custom_objects.utilities import extract_cot_id_from_model_name @@ -347,9 +352,13 @@ def save(self, using=None, **_kwargs): # (If pk was None before save_base, obj.pk is now the DB-assigned id.) obj_pk = obj.pk # Register full data for deferred column updates (squash ordering fix). - if table_name not in cls._deferred_field_data: - cls._deferred_field_data[table_name] = {} - cls._deferred_field_data[table_name][obj_pk] = { + deferred = _deferred_co_field_data.get() + if deferred is None: + deferred = {} + _deferred_co_field_data.set(deferred) + if table_name not in deferred: + deferred[table_name] = {} + deferred[table_name][obj_pk] = { 'using': _using, 'data': full_data, } diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 73cb3bd5..e5c052d2 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -67,9 +67,9 @@ def setUp(self): def tearDown(self): from core.models import ObjectChange - from netbox_custom_objects.models import CustomObject - # Flush deferred CO field data so it doesn't bleed into the next test. - CustomObject._deferred_field_data.clear() + from netbox_custom_objects.models import _deferred_co_field_data + # Reset deferred CO field data so it doesn't bleed into the next test. + _deferred_co_field_data.set(None) # Delete COTs and their backing tables before the DB flush. for cot in CustomObjectType.objects.all(): try: From 38366696792a4559e8ee98f9d1482167ec6a4413 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Apr 2026 13:37:10 -0700 Subject: [PATCH 018/115] add tests --- .../0007_fix_object_field_fk_deferrable.py | 2 +- netbox_custom_objects/models.py | 6 +- netbox_custom_objects/tests/test_branching.py | 558 ++++++++++++++++++ 3 files changed, 563 insertions(+), 3 deletions(-) diff --git a/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py b/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py index c03b7cad..6e57b53b 100644 --- a/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py +++ b/netbox_custom_objects/migrations/0007_fix_object_field_fk_deferrable.py @@ -42,7 +42,7 @@ def fix_deferrable_fk_constraints(apps, schema_editor): AND tc.table_schema = rc.constraint_schema WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name LIKE 'custom_objects\\_%%' - AND rc.is_deferrable = 'YES' + AND tc.is_deferrable = 'YES' """) rows = cursor.fetchall() diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 09315b6e..8078507f 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1508,12 +1508,14 @@ def clean(self): {"unique": _("Uniqueness cannot be enforced for boolean or multiobject fields")} ) - # Check if uniqueness constraint can be applied when changing from non-unique to unique + # Check if uniqueness constraint can be applied when changing from non-unique to unique. + # Skip when _original is absent (e.g. during deserialization in branch merge/revert). if ( self.pk and self.unique - and not self.original.unique and not self._state.adding + and hasattr(self, '_original') + and not self.original.unique ): field_type = FIELD_TYPE_CLASS[self.type]() model_field = field_type.get_model_field(self) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index f5c7e0fe..2f869e08 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -316,6 +316,281 @@ def test_comprehensive_merge_and_revert(self): f'Field {name!r} must not be in main after revert', ) + # ── object modified inside branch ───────────────────────────────────── + + def test_object_modified_merge_and_revert(self): + """ + CO that exists in main is modified inside a branch. Merge brings the + new value to main; revert restores the original. + + The COT and field are created in main before provisioning so the branch + has a full schema copy. Only the CO data changes inside the branch. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='modify_cot', slug='modify-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='notes', + label='Notes', + type='text', + ) + Model = cot.get_model() + co = Model.objects.create(notes='original value') + + co_pk = co.pk + branch = _provision_branch('Modify Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # Modify CO inside the branch. + with activate_branch(branch), event_tracking(branch_request): + branch_co = cot.get_model().objects.get(pk=co_pk) + branch_co.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + branch_co.notes = 'modified in branch' + branch_co.save() + + # Main must not see the modification yet. + co.refresh_from_db() + self.assertEqual(co.notes, 'original value', 'Main must not see branch modification before merge') + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + co.refresh_from_db() + self.assertEqual(co.notes, 'modified in branch', 'Main must see modified value after merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + co.refresh_from_db() + self.assertEqual(co.notes, 'original value', 'Main must have original value after revert') + + # ── object deleted inside branch ────────────────────────────────────── + + def test_object_deleted_merge_and_revert(self): + """ + CO that exists in main is deleted inside a branch. Merge removes it + from main; revert restores it. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='delete_cot', slug='delete-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='notes', + label='Notes', + type='text', + ) + Model = cot.get_model() + co = Model.objects.create(notes='will be deleted') + + co_pk = co.pk + branch = _provision_branch('Delete Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot.get_model().objects.get(pk=co_pk).delete() + + # CO must still exist in main before merge. + self.assertTrue( + cot.get_model().objects.filter(pk=co_pk).exists(), + 'CO must still exist in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertFalse( + cot.get_model().objects.filter(pk=co_pk).exists(), + 'CO must be deleted from main after merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertTrue( + cot.get_model().objects.filter(pk=co_pk).exists(), + 'CO must be restored in main after revert', + ) + + # ── field renamed inside branch → merge ─────────────────────────────── + + def test_field_rename_merge_and_revert(self): + """ + Field created and then renamed inside a branch. Merge brings the COT + with the renamed column to main; revert removes it. + + Exercises _schema_alter_field via the merge deserialization path using + PK-based rename detection (same PK, different name values). + """ + branch = _provision_branch('Rename Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='rename_cot', slug='rename-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='old_name', + label='Old Name', + type='text', + ) + # Load from DB so _original is set, then rename. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + field.name = 'new_name' + field.label = 'New Name' + field.save() + Model = cot.get_model() + co = Model.objects.create(new_name='value after rename') + + field_pk, cot_pk, co_pk = field.pk, cot.pk, co.pk + + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must not exist in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertTrue(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertEqual(field_main.name, 'new_name', 'Field must have new name in main after merge') + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.new_name, 'value after rename') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + + # ── unique constraint toggled inside branch → merge ─────────────────── + + def test_field_unique_toggle_merge_and_revert(self): + """ + Field created without a unique constraint inside a branch, then + toggled to unique=True. Merge brings the COT with the UNIQUE + constraint to main; revert removes it. + + Exercises alter_field for constraint-only changes via the merge path. + """ + from django.db import connection as main_conn + + branch = _provision_branch('Unique Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='unique_cot', slug='unique-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='code', + label='Code', + type='text', + unique=False, + ) + # Load from DB so _original is set, then enable unique. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + field.unique = True + field.save() + Model = cot.get_model() + co_pk = Model.objects.create(code='ABC').pk + + field_pk, cot_pk = field.pk, cot.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertTrue(field_main.unique, 'Field must have unique=True in main after merge') + + # Verify the UNIQUE constraint exists in main's physical schema. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + table_name = cot_main.get_database_table_name() + with main_conn.cursor() as cursor: + constraints = main_conn.introspection.get_constraints(cursor, table_name) + self.assertTrue( + any(c['unique'] and c.get('columns') == ['code'] for c in constraints.values()), + 'UNIQUE constraint on "code" must exist in main schema after merge', + ) + + # CO with code='ABC' must have survived the merge. + cot_main2 = CustomObjectType.objects.get(pk=cot_pk) + self.assertTrue(cot_main2.get_model().objects.filter(pk=co_pk).exists()) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + # ── non-schema field attributes inside branch → merge ───────────────── + + def test_field_non_schema_attrs_merge_and_revert(self): + """ + Field attributes that do not affect the physical schema (label, + primary, required, description) survive a branch merge and revert + without causing schema errors or spurious ALTER TABLE calls. + + These attributes are excluded from _field_schema_key and exist only + at the application layer. This test confirms that altering them in + a branch and merging correctly updates the ORM-level field record in + main without touching the DB column definition. + """ + branch = _provision_branch('Attrs Branch', self.MERGE_STRATEGY, self.user) + request = _make_request(self.user) + + with activate_branch(branch), event_tracking(request): + cot = CustomObjectType.objects.create(name='attrs_cot', slug='attrs-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='title', + label='Title', + type='text', + primary=False, + required=False, + description='original description', + ) + # Load from DB so _original is set, then mutate non-schema attrs. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.snapshot() # captures pre-change state for ObjectChange.diff()['pre'] during revert + field.label = 'Updated Title' + field.primary = True + field.required = True + field.description = 'updated description' + field.save() + + field_pk, cot_pk = field.pk, cot.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertEqual(field_main.label, 'Updated Title') + self.assertTrue(field_main.primary) + self.assertTrue(field_main.required) + self.assertEqual(field_main.description, 'updated description') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + # ── Concrete test classes (one per merge strategy) ──────────────────────────── @@ -411,3 +686,286 @@ def test_main_changes_synced_to_branch(self): BranchModel = cot_branch.get_model() co_branch = BranchModel.objects.get(pk=co_pk) self.assertEqual(co_branch.title, 'main object') + + +# ── Drift detection and Branch.migrate() tests ──────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class BranchMigrateTestCase(TransactionCleanupMixin, TransactionTestCase): + """ + Tests for the drift-detection and schema-reconciliation path. + + Scenario: + 1. A COT and field(s) are created in main and a branch is provisioned + (branch has a full schema copy at provision time). + 2. A field is then modified in main (added, removed, renamed, or + unique-toggled). on_custom_object_field_changed fires and marks the + branch PENDING_MIGRATIONS. + 3. branch.migrate(user) runs the normal Django migration pass and then + emits post_migrate, which fires on_branch_migrated. That handler + reconciles the branch's physical schema against main's current field + definitions. + 4. The branch is back to READY and its physical column layout matches main. + + These tests do NOT use merge/revert — they isolate the separate + on_branch_migrated reconciliation path for branches that pre-date a main + schema change. They use the iterative strategy only because the strategy + affects merge order, not schema reconciliation. + """ + + def setUp(self): + super().setUp() + _recreate_contenttypes() + self.user = User.objects.create_user(username='testuser') + self.request = _make_request(self.user) + + def tearDown(self): + _close_branch_connections() + super().tearDown() + + def _get_branch_columns(self, branch, table_name): + """Return the set of column names on table_name in the branch schema.""" + conn = connections[branch.connection_name] + with conn.cursor() as cursor: + return {col.name for col in conn.introspection.get_table_description(cursor, table_name)} + + def _get_branch_tables(self, branch): + """Return the set of table names in the branch schema.""" + conn = connections[branch.connection_name] + with conn.cursor() as cursor: + return set(conn.introspection.table_names(cursor)) + + def _get_branch_constraints(self, branch, table_name): + """Return the constraints dict for table_name in the branch schema.""" + conn = connections[branch.connection_name] + with conn.cursor() as cursor: + return conn.introspection.get_constraints(cursor, table_name) + + # ── field added to main ─────────────────────────────────────────────── + + def test_field_added_to_main_triggers_branch_migrate(self): + """ + Field added to a COT in main marks the branch PENDING_MIGRATIONS. + branch.migrate() fires on_branch_migrated which calls add_field to + create the new column in the branch schema. + """ + cot = CustomObjectType.objects.create(name='drift_add_cot', slug='drift-add-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='existing_field', + label='Existing', + type='text', + ) + + branch = _provision_branch('Drift Add Branch', 'iterative', self.user) + table_name = cot.get_database_table_name() + + self.assertEqual(branch.status, BranchStatusChoices.READY) + self.assertIn('existing_field', self._get_branch_columns(branch, table_name)) + + # Add a new field to main — on_custom_object_field_changed marks branch PENDING. + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='new_field', + label='New Field', + type='integer', + ) + + branch.refresh_from_db() + self.assertEqual( + branch.status, BranchStatusChoices.PENDING_MIGRATIONS, + 'Branch must be PENDING_MIGRATIONS after field added to main', + ) + self.assertNotIn( + 'new_field', self._get_branch_columns(branch, table_name), + 'New column must not exist in branch before migrate', + ) + + # migrate() fires on_branch_migrated → add_field runs on the branch schema. + branch.migrate(user=self.user) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.READY) + + self.assertIn( + 'new_field', self._get_branch_columns(branch, table_name), + 'New column must be present in branch schema after migrate', + ) + + # ── field deleted from main ─────────────────────────────────────────── + + def test_field_deleted_from_main_triggers_branch_migrate(self): + """ + Field deleted from a COT in main marks the branch PENDING_MIGRATIONS. + branch.migrate() fires on_branch_migrated which calls remove_field to + drop the column from the branch schema. + """ + cot = CustomObjectType.objects.create(name='drift_del_cot', slug='drift-del-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='keep_field', + label='Keep', + type='text', + ) + drop_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='drop_field', + label='Drop', + type='text', + ) + + branch = _provision_branch('Drift Del Branch', 'iterative', self.user) + table_name = cot.get_database_table_name() + + self.assertIn('drop_field', self._get_branch_columns(branch, table_name)) + + # Delete field from main. + drop_field.delete() + + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) + + # Column still present in branch before migrate. + self.assertIn('drop_field', self._get_branch_columns(branch, table_name)) + + branch.migrate(user=self.user) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.READY) + + cols = self._get_branch_columns(branch, table_name) + self.assertNotIn( + 'drop_field', cols, + 'Deleted column must be absent from branch schema after migrate', + ) + self.assertIn('keep_field', cols, 'Unmodified column must remain in branch schema') + + # ── field renamed in main ───────────────────────────────────────────── + + def test_field_renamed_in_main_triggers_branch_migrate(self): + """ + Field renamed in main marks the branch PENDING_MIGRATIONS. + branch.migrate() fires on_branch_migrated which calls alter_field to + rename the column in the branch schema using PK-based matching (same + PK in both main and branch, different name values). + """ + cot = CustomObjectType.objects.create(name='drift_ren_cot', slug='drift-ren-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='old_col', + label='Old', + type='text', + ) + + branch = _provision_branch('Drift Ren Branch', 'iterative', self.user) + table_name = cot.get_database_table_name() + + self.assertIn('old_col', self._get_branch_columns(branch, table_name)) + + # Rename in main — load from DB so _original is set before modifying. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.name = 'new_col' + field.label = 'New' + field.save() + + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) + + branch.migrate(user=self.user) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.READY) + + cols = self._get_branch_columns(branch, table_name) + self.assertIn('new_col', cols, 'Renamed column must exist in branch schema after migrate') + self.assertNotIn('old_col', cols, 'Old column name must be absent from branch schema after migrate') + + # ── unique constraint toggled in main ───────────────────────────────── + + def test_unique_toggled_in_main_triggers_branch_migrate(self): + """ + Field's unique constraint toggled in main marks the branch + PENDING_MIGRATIONS. branch.migrate() reconciles the constraint in + the branch's physical schema. + """ + cot = CustomObjectType.objects.create(name='drift_uniq_cot', slug='drift-uniq-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='code', + label='Code', + type='text', + unique=False, + ) + + branch = _provision_branch('Drift Uniq Branch', 'iterative', self.user) + table_name = cot.get_database_table_name() + + # Branch must not have a unique constraint on 'code' initially. + constraints_before = self._get_branch_constraints(branch, table_name) + self.assertFalse( + any(c['unique'] and c.get('columns') == ['code'] for c in constraints_before.values()), + 'Branch must not have UNIQUE on "code" before toggle', + ) + + # Toggle unique=True in main. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.unique = True + field.save() + + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) + + branch.migrate(user=self.user) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.READY) + + constraints_after = self._get_branch_constraints(branch, table_name) + self.assertTrue( + any(c['unique'] and c.get('columns') == ['code'] for c in constraints_after.values()), + 'Branch must have UNIQUE constraint on "code" after migrate', + ) + + # ── multiobject field renamed in main (through-table rename) ───────── + + def test_multiobject_field_renamed_in_main_triggers_branch_migrate(self): + """ + MULTIOBJECT field renamed in main marks the branch PENDING_MIGRATIONS. + branch.migrate() fires on_branch_migrated which renames the through + table in the branch schema in addition to the column alter_field. + + Exercises the MULTIOBJECT rename branch of _schema_alter_field. + """ + from core.models import ObjectType + + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + cot = CustomObjectType.objects.create(name='drift_m2m_cot', slug='drift-m2m-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='related_sites', + label='Related Sites', + type='multiobject', + related_object_type=site_ot, + ) + + old_through = field.through_table_name + + branch = _provision_branch('Drift M2M Branch', 'iterative', self.user) + + self.assertIn(old_through, self._get_branch_tables(branch)) + + # Rename in main. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.name = 'linked_sites' + field.label = 'Linked Sites' + field.save() + + new_through = field.through_table_name + + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) + + branch.migrate(user=self.user) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.READY) + + branch_tables = self._get_branch_tables(branch) + self.assertIn(new_through, branch_tables, 'Renamed through table must exist in branch after migrate') + self.assertNotIn(old_through, branch_tables, 'Old through table must be absent from branch after migrate') From 1b1584ca9bf1492a9d59f1cb5fa39b7a50a6fdf9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 11:53:35 -0700 Subject: [PATCH 019/115] update changelog entries on rename, remove branch migrate --- netbox_custom_objects/__init__.py | 6 +-- netbox_custom_objects/branching.py | 61 ++++++--------------- netbox_custom_objects/models.py | 85 ++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 51 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index a661afda..0e7e04a5 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -184,13 +184,9 @@ def ready(self): # Wire into the netbox-branching lifecycle when that plugin is installed. # Import is lazy so the plugin remains functional without branching. try: - from django.db.models.signals import post_delete, post_save from netbox_branching.signals import post_migrate as branching_post_migrate - from netbox_custom_objects.branching import on_branch_migrated, on_custom_object_field_changed - from netbox_custom_objects.models import CustomObjectTypeField + from netbox_custom_objects.branching import on_branch_migrated branching_post_migrate.connect(on_branch_migrated) - post_save.connect(on_custom_object_field_changed, sender=CustomObjectTypeField) - post_delete.connect(on_custom_object_field_changed, sender=CustomObjectTypeField) except ImportError: pass diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 98ca3ac7..488f699d 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -8,15 +8,24 @@ To close that gap, ``on_branch_migrated`` fires after the migration pass and reconciles each custom object type's physical schema in the branch against the -current field definitions in main. The reconciliation mirrors what -``CustomObjectTypeField.save()`` already does when detecting old-vs-new field -state: it compares the branch's ``CustomObjectTypeField`` rows (snapshot from -provision time) with main's current rows, matched by primary key, and calls -``add_field`` / ``alter_field`` / ``remove_field`` for any differences. +current field definitions in main. It compares the branch's +``CustomObjectTypeField`` rows (snapshot from provision time) with main's +current rows, matched by primary key, and calls ``add_field`` / ``alter_field`` +/ ``remove_field`` for any differences. + +Schema ordering during sync and merge is handled naturally by the chronological +ObjectChange replay in ``Branch.sync()`` / ``Branch.merge()``: +``CustomObjectTypeField`` changes always precede ``CustomObject`` data changes +in the log. The schema helpers (``_schema_add_field``, ``_schema_alter_field``) +are idempotent, so replaying an ObjectChange after ``on_branch_migrated`` has +already applied the same schema change is safe. ``CustomObjectTypeField`` has ``ChangeLoggingMixin``, so its rows are replicated into the branch schema at provision time — the same source-of-truth mechanism -used for all other branchable models. +used for all other branchable models. Field-rename conflicts (same field PK +renamed differently in main and branch) surface as ``ChangeDiff`` conflicts and +require user acknowledgment before merge/sync proceeds; the existing branching +conflict mechanism handles this without any special casing. """ import logging @@ -75,46 +84,6 @@ def check_pending_branch_migrations(): ) -def on_custom_object_field_changed(sender, instance, **kwargs): - """ - Mark all READY branches as PENDING_MIGRATIONS when a ``CustomObjectTypeField`` - is created, modified, or deleted in the main schema. - - This surfaces the pending state in the branching UI exactly like a normal - Django-migration, prompting users to click "Migrate Branch". That action - calls ``Branch.migrate()``, which fires ``on_branch_migrated`` and reconciles - the physical column differences. - - Branches provisioned before this COT's table existed will be over-marked as - PENDING_MIGRATIONS but ``on_branch_migrated`` will skip them (no table in - branch) so the extra marking is harmless. The alternative — opening a - connection per branch to check whether the table exists — adds O(branches) - latency to every field save, which is worse at scale. - - Skipped when the change happens inside a branch context — the field edit only - affects that branch's schema, not main, so no other branches need updating. - """ - try: - from netbox_branching.contextvars import active_branch - if active_branch.get() is not None: - return - except ImportError: - return - - from netbox_branching.choices import BranchStatusChoices - from netbox_branching.models import Branch - - updated = Branch.objects.filter(status=BranchStatusChoices.READY).update( - status=BranchStatusChoices.PENDING_MIGRATIONS, - ) - if updated: - logger.info( - 'Marked %d branch(es) as PENDING_MIGRATIONS due to field changes on %s', - updated, - instance.custom_object_type, - ) - - def _field_schema_key(f): """ Return the subset of ``CustomObjectTypeField`` attributes that affect the diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 8078507f..5f7deea4 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1,5 +1,6 @@ import contextvars import decimal +import logging import re import threading from datetime import date, datetime @@ -59,6 +60,9 @@ from netbox_custom_objects.utilities import _suppress_clear_cache, generate_model +logger = logging.getLogger('netbox_custom_objects.models') + + class UniquenessConstraintTestError(Exception): """Custom exception used to signal successful uniqueness constraint test.""" @@ -160,10 +164,23 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): Handles through-table creation for MULTIOBJECT fields. Does NOT apply deferred CO field data — callers that need that (squash merge context) must call ``_apply_deferred_co_field(fi)`` separately after this returns. + + Idempotent: skips the ALTER TABLE if the column already exists (e.g. when + ``on_branch_migrated`` pre-added it and sync later replays the ObjectChange). """ ft = FIELD_TYPE_CLASS[fi.type]() mf = ft.get_model_field(fi) mf.contribute_to_class(model, fi.name) + + with schema_conn.cursor() as cursor: + existing_cols = { + col.name + for col in schema_conn.introspection.get_table_description(cursor, model._meta.db_table) + } + if mf.column in existing_cols: + logger.debug('_schema_add_field: %r already exists on %s, skipping', mf.column, model._meta.db_table) + return + schema_editor.add_field(model, mf) if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: ft.create_m2m_table(fi, model, fi.name, schema_conn=schema_conn) @@ -219,12 +236,28 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist connection. When given, through-table operations are guarded by membership checks. When ``None`` (main-schema context) the schema_conn is introspected once on demand. + + Idempotent: skips the ALTER TABLE if the old column is already gone and the + new column already exists (e.g. when ``on_branch_migrated`` pre-applied the + rename and sync later replays the ObjectChange). """ old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) old_mf.contribute_to_class(model, old_fi.name) new_mf.contribute_to_class(model, new_fi.name) + with schema_conn.cursor() as cursor: + existing_cols = { + col.name + for col in schema_conn.introspection.get_table_description(cursor, model._meta.db_table) + } + if old_mf.column not in existing_cols and new_mf.column in existing_cols: + logger.debug( + '_schema_alter_field: %r already renamed to %r on %s, skipping', + old_mf.column, new_mf.column, model._meta.db_table, + ) + return + if ( new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT and old_fi.name != new_fi.name @@ -1150,6 +1183,53 @@ def custom_object_type_post_save_handler(sender, instance, created, **kwargs): instance.save() +def _rename_objectchange_field_key(fi, old_name, new_name): + """ + Rename a JSON key in all ObjectChange records for CustomObject instances of + this field's type, reflecting a field rename from *old_name* to *new_name*. + + Updates both ``prechange_data`` and ``postchange_data`` in the ObjectChange + table, and ``original``/``modified``/``current`` in netbox-branching's + ChangeDiff table when that plugin is installed. + + Field names are validated with ``^[a-z0-9_]+$`` so string formatting of the + column names here is safe against SQL injection. + + This runs inside the same ``transaction.atomic()`` block as + ``CustomObjectTypeField.save()``, so it rolls back cleanly if the enclosing + transaction is aborted. + """ + from django.db import connections + + cot = fi.custom_object_type + model = cot.get_model() + ct = ContentType.objects.get_for_model(model) + conn = _get_schema_connection() + + oc_sql = ( + 'UPDATE core_objectchange ' + 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' + 'WHERE changed_object_type_id = %s AND {col} ? %s' + ) + with connections[conn.alias].cursor() as cursor: + for json_col in ('prechange_data', 'postchange_data'): + cursor.execute(oc_sql.format(col=json_col), [old_name, new_name, old_name, ct.id, old_name]) + + logger.debug('_rename_objectchange_field_key: %r → %r for %s', old_name, new_name, ct) + + try: + cd_sql = ( + 'UPDATE netbox_branching_changediff ' + 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' + 'WHERE object_type_id = %s AND {col} IS NOT NULL AND {col} ? %s' + ) + with connections[conn.alias].cursor() as cursor: + for json_col in ('original', 'modified', 'current'): + cursor.execute(cd_sql.format(col=json_col), [old_name, new_name, old_name, ct.id, old_name]) + except Exception: + pass # netbox-branching not installed or table absent + + class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedModel): custom_object_type = models.ForeignKey( CustomObjectType, on_delete=models.CASCADE, related_name="fields" @@ -2039,6 +2119,11 @@ def save(self, *args, **kwargs): else: _schema_alter_field(self.original, self, model, schema_editor, schema_conn) + # When the field is renamed, update ObjectChange / ChangeDiff JSON keys so + # historical audit records and branch diffs stay consistent with the new name. + if not self._state.adding and self._original_name != self.name: + _rename_objectchange_field_key(self, self._original_name, self.name) + # Ensure FK constraints are properly created for OBJECT fields with CASCADE behavior should_ensure_fk = False if self.type == CustomFieldTypeChoices.TYPE_OBJECT: From ba8eda3b850d238d9ca761fa1e3a0e25862ea025 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 13:09:13 -0700 Subject: [PATCH 020/115] cleanup --- netbox_custom_objects/models.py | 44 ++++++++++++++++++- netbox_custom_objects/tests/test_branching.py | 10 +++-- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 5f7deea4..90f16a6f 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -156,6 +156,21 @@ def _apply_deferred_co_field(field_instance): [value, co_pk], ) + # Remove the consumed key from each entry so that processed field data does + # not persist in the ContextVar beyond its useful lifetime (e.g. on a retry + # after a partial failure, stale data from a previous attempt is avoided). + for entry in per_table.values(): + entry['data'].pop(data_key, None) + + # Prune entries whose data dict is now exhausted. + exhausted = [pk for pk, entry in per_table.items() if not entry['data']] + for pk in exhausted: + del per_table[pk] + if not per_table: + del deferred[table_name] + if not deferred: + _deferred_co_field_data.set(None) + def _schema_add_field(fi, model, schema_editor, schema_conn): """ @@ -241,6 +256,22 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist new column already exists (e.g. when ``on_branch_migrated`` pre-applied the rename and sync later replays the ObjectChange). """ + old_is_m2m = old_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + new_is_m2m = new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + + # A type change between MULTIOBJECT and a scalar type (or vice versa) is not + # a simple column rename/alter — the storage representation is fundamentally + # different (through-table vs column). Attempting alter_field in this case + # would fail at the DB level. Log and skip; the caller is expected to handle + # such changes as remove + add rather than alter. + if old_is_m2m != new_is_m2m: + logger.warning( + '_schema_alter_field: skipping unsupported type change %r→%r on %s ' + '(MULTIOBJECT ↔ scalar changes require remove+add, not alter)', + old_fi.type, new_fi.type, model._meta.db_table, + ) + return + old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) old_mf.contribute_to_class(model, old_fi.name) @@ -259,7 +290,7 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist return if ( - new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT + new_is_m2m and old_fi.name != new_fi.name ): old_through = old_fi.through_table_name @@ -1042,6 +1073,9 @@ def __init__(self, deserialized): self.object = deserialized.object def save(self, using=None, **kwargs): + # Snapshot before modifying so that diff()['pre'] records the + # current state rather than showing all fields as None on revert. + self.object.snapshot() # Clear the ObjectType FK — it may not exist in main yet. # custom_object_type_post_save_handler re-sets it after INSERT. self.object.object_type = None @@ -1218,6 +1252,7 @@ def _rename_objectchange_field_key(fi, old_name, new_name): logger.debug('_rename_objectchange_field_key: %r → %r for %s', old_name, new_name, ct) try: + from netbox_branching.models import ChangeDiff # noqa: F401 — presence check only cd_sql = ( 'UPDATE netbox_branching_changediff ' 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' @@ -1226,8 +1261,13 @@ def _rename_objectchange_field_key(fi, old_name, new_name): with connections[conn.alias].cursor() as cursor: for json_col in ('original', 'modified', 'current'): cursor.execute(cd_sql.format(col=json_col), [old_name, new_name, old_name, ct.id, old_name]) + except ImportError: + pass # netbox-branching not installed except Exception: - pass # netbox-branching not installed or table absent + logger.debug( + '_rename_objectchange_field_key: ChangeDiff rename failed for %r → %r', + old_name, new_name, exc_info=True, + ) class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedModel): diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 2f869e08..fdc3ff47 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -63,8 +63,10 @@ def _provision_branch(name, merge_strategy, user): def _close_branch_connections(): """Close any open branch database connections.""" for branch in Branch.objects.all(): - if hasattr(connections, branch.connection_name): + try: connections[branch.connection_name].close() + except Exception: + pass # ── Shared merge/revert tests (strategy-agnostic) ──────────────────────────── @@ -699,8 +701,8 @@ class BranchMigrateTestCase(TransactionCleanupMixin, TransactionTestCase): 1. A COT and field(s) are created in main and a branch is provisioned (branch has a full schema copy at provision time). 2. A field is then modified in main (added, removed, renamed, or - unique-toggled). on_custom_object_field_changed fires and marks the - branch PENDING_MIGRATIONS. + unique-toggled). netbox-branching detects the change to the non-exempt + CustomObjectTypeField model and marks the branch PENDING_MIGRATIONS. 3. branch.migrate(user) runs the normal Django migration pass and then emits post_migrate, which fires on_branch_migrated. That handler reconciles the branch's physical schema against main's current field @@ -763,7 +765,7 @@ def test_field_added_to_main_triggers_branch_migrate(self): self.assertEqual(branch.status, BranchStatusChoices.READY) self.assertIn('existing_field', self._get_branch_columns(branch, table_name)) - # Add a new field to main — on_custom_object_field_changed marks branch PENDING. + # Add a new field to main — netbox-branching detects the change and marks branch PENDING. CustomObjectTypeField.objects.create( custom_object_type=cot, name='new_field', From 993fae7e05fc63d3bc6898ad49e02126ca9f5826 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 13:46:33 -0700 Subject: [PATCH 021/115] refactor --- netbox_custom_objects/__init__.py | 21 - netbox_custom_objects/branching.py | 200 ----- netbox_custom_objects/models.py | 87 +- netbox_custom_objects/tests/test_branching.py | 843 +++++++++++++----- 4 files changed, 679 insertions(+), 472 deletions(-) delete mode 100644 netbox_custom_objects/branching.py diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index df17acff..af41097c 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -1,5 +1,4 @@ import contextvars -import logging import sys import warnings @@ -181,15 +180,6 @@ def ready(self): pre_migrate.connect(_migration_started) post_migrate.connect(_migration_finished) - # Wire into the netbox-branching lifecycle when that plugin is installed. - # Import is lazy so the plugin remains functional without branching. - try: - from netbox_branching.signals import post_migrate as branching_post_migrate - from netbox_custom_objects.branching import on_branch_migrated - branching_post_migrate.connect(on_branch_migrated) - except ImportError: - pass - # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() @@ -219,17 +209,6 @@ def ready(self): super().ready() return - # Scan for branches with pre-existing custom object schema drift - # (covers upgrades and any changes made while the app was offline). - try: - from netbox_custom_objects.branching import check_pending_branch_migrations - check_pending_branch_migrations() - except Exception: - logging.getLogger('netbox_custom_objects').exception( - 'check_pending_branch_migrations() failed at startup; ' - 'some branches may not be marked as PENDING_MIGRATIONS' - ) - super().ready() def get_model(self, model_name, require_ready=True): diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py deleted file mode 100644 index 488f699d..00000000 --- a/netbox_custom_objects/branching.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -Branching support for NetBox Custom Objects. - -When netbox-branching runs ``Branch.migrate()``, the Django migration pass only -handles normal app migrations. Custom object field changes (add/alter/remove -column) are applied directly via the schema editor and are therefore invisible -to Django's migration framework. - -To close that gap, ``on_branch_migrated`` fires after the migration pass and -reconciles each custom object type's physical schema in the branch against the -current field definitions in main. It compares the branch's -``CustomObjectTypeField`` rows (snapshot from provision time) with main's -current rows, matched by primary key, and calls ``add_field`` / ``alter_field`` -/ ``remove_field`` for any differences. - -Schema ordering during sync and merge is handled naturally by the chronological -ObjectChange replay in ``Branch.sync()`` / ``Branch.merge()``: -``CustomObjectTypeField`` changes always precede ``CustomObject`` data changes -in the log. The schema helpers (``_schema_add_field``, ``_schema_alter_field``) -are idempotent, so replaying an ObjectChange after ``on_branch_migrated`` has -already applied the same schema change is safe. - -``CustomObjectTypeField`` has ``ChangeLoggingMixin``, so its rows are replicated -into the branch schema at provision time — the same source-of-truth mechanism -used for all other branchable models. Field-rename conflicts (same field PK -renamed differently in main and branch) surface as ``ChangeDiff`` conflicts and -require user acknowledgment before merge/sync proceeds; the existing branching -conflict mechanism handles this without any special casing. -""" - -import logging - -from django.db import connections - -logger = logging.getLogger('netbox_custom_objects.branching') - - -def check_pending_branch_migrations(): - """ - Scan all READY branches at startup and mark any that have custom object - schema drift as PENDING_MIGRATIONS. - - This catches drift that pre-dates the signal handler — for example, after - upgrading the plugin on an instance that already has branches, or if field - changes were applied while the application was not running. - - Called once from ``CustomObjectsPluginConfig.ready()`` after the DB is - confirmed ready. - """ - try: - from netbox_branching.choices import BranchStatusChoices - from netbox_branching.models import Branch - from netbox_custom_objects.models import CustomObjectType - except ImportError: - return - - ready_branches = list(Branch.objects.filter(status=BranchStatusChoices.READY)) - if not ready_branches: - return - - cots = list(CustomObjectType.objects.all()) - if not cots: - return - - to_update = [] - for branch in ready_branches: - branch_connection = connections[branch.connection_name] - with branch_connection.cursor() as cursor: - branch_tables = branch_connection.introspection.table_names(cursor) - - for cot in cots: - if cot.get_database_table_name() not in branch_tables: - continue - if cot.has_branch_schema_drift(branch): - branch.status = BranchStatusChoices.PENDING_MIGRATIONS - to_update.append(branch) - break # One drifted COT is enough — no need to check the rest - - if to_update: - Branch.objects.bulk_update(to_update, ['status']) - logger.info( - 'Marked %d branch(es) as PENDING_MIGRATIONS at startup due to custom object schema drift', - len(to_update), - ) - - -def _field_schema_key(f): - """ - Return the subset of ``CustomObjectTypeField`` attributes that affect the - physical DB column schema. - - Excluded (application-level only, no physical DB impact): - - required: all field types use null=True regardless; required never maps - to a NOT NULL column constraint. - - default: Python-level default applied by the ORM, not a SQL DEFAULT - clause; changing it on an existing column needs no ALTER TABLE. - - Used by both ``_fields_schema_differ()`` and - ``CustomObjectType.has_branch_schema_drift()`` to ensure the two callers - always agree on what constitutes a schema-affecting change. - """ - return (f.name, f.type, f.unique, f.related_object_type_id) - - -def _fields_schema_differ(branch_f, main_f): - """ - Return True if the two ``CustomObjectTypeField`` instances differ in any - attribute that affects the physical DB column, meaning an ALTER TABLE is - needed to bring the branch schema up to date. - """ - return _field_schema_key(branch_f) != _field_schema_key(main_f) - - -def on_branch_migrated(sender, branch, user, **kwargs): - """ - Reconcile each custom object type's physical schema in the branch against - the current field definitions in main. - - For each ``CustomObjectType`` whose table exists in the branch schema: - - Fields present in main but absent from the branch → ``add_field`` - - Fields absent from main but present in the branch → ``remove_field`` - - Fields present in both with differing definitions → ``alter_field`` - - Matching is done by primary key so renames are detected correctly (the pk - exists in both, with different ``name`` values) rather than being treated - as an unrelated delete + add. - """ - from netbox_branching.utilities import activate_branch - from netbox_custom_objects.models import ( - CustomObjectType, - _schema_add_field, - _schema_alter_field, - _schema_remove_field, - ) - - branch_connection = connections[branch.connection_name] - - with branch_connection.cursor() as cursor: - branch_tables = branch_connection.introspection.table_names(cursor) - - for cot in CustomObjectType.objects.all(): - db_table = cot.get_database_table_name() - if db_table not in branch_tables: - # Table absent — COT was created after this branch was provisioned - # and the branch hasn't been synced yet. Skip; the user needs to - # sync first to pull in the new table. - logger.debug('Skipping %s — table %r not in branch schema', cot, db_table) - continue - - # Main's current field definitions (queried from the public schema since - # active_branch is not set here). - main_fields = { - f.pk: f - for f in cot.fields.select_related('related_object_type', 'choice_set').all() - } - - # Branch's field snapshot (as of provision time, or last sync). - with activate_branch(branch): - branch_fields = { - f.pk: f - for f in cot.fields.select_related('related_object_type', 'choice_set').all() - } - - main_pks = set(main_fields) - branch_pks = set(branch_fields) - - to_add = [main_fields[pk] for pk in main_pks - branch_pks] - to_remove = [branch_fields[pk] for pk in branch_pks - main_pks] - to_alter = [ - (branch_fields[pk], main_fields[pk]) # (old, new) - for pk in main_pks & branch_pks - if _fields_schema_differ(branch_fields[pk], main_fields[pk]) - ] - - if not (to_add or to_remove or to_alter): - continue - - logger.info( - 'Migrating branch schema for %s: %d add, %d remove, %d alter', - cot, len(to_add), len(to_remove), len(to_alter), - ) - - model = cot.get_model() - - with branch_connection.schema_editor() as schema_editor: - - for fi in to_add: - logger.debug('add_field %r on %s', fi.name, cot) - _schema_add_field(fi, model, schema_editor, branch_connection) - - for fi in to_remove: - logger.debug('remove_field %r on %s', fi.name, cot) - _schema_remove_field(fi, model, schema_editor, existing_tables=branch_tables) - - for old_fi, new_fi in to_alter: - logger.debug('alter_field %r → %r on %s', old_fi.name, new_fi.name, cot) - _schema_alter_field( - old_fi, new_fi, model, schema_editor, branch_connection, - existing_tables=branch_tables, - ) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 90f16a6f..733c2804 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -181,7 +181,7 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): call ``_apply_deferred_co_field(fi)`` separately after this returns. Idempotent: skips the ALTER TABLE if the column already exists (e.g. when - ``on_branch_migrated`` pre-added it and sync later replays the ObjectChange). + sync/merge replays an ObjectChange that was already applied). """ ft = FIELD_TYPE_CLASS[fi.type]() mf = ft.get_model_field(fi) @@ -253,8 +253,14 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist once on demand. Idempotent: skips the ALTER TABLE if the old column is already gone and the - new column already exists (e.g. when ``on_branch_migrated`` pre-applied the - rename and sync later replays the ObjectChange). + new column already exists (e.g. when sync/merge replays an ObjectChange that + was already applied). + + Conflict resolution: when neither the old nor the new column exists (the field + was independently renamed in the target schema — e.g. branch renamed A→X while + main renamed A→Y), the live field record is looked up from the target schema to + find the actual current column name, which is then renamed to the new target. + A warning is logged to flag the conflict. """ old_is_m2m = old_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT new_is_m2m = new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT @@ -282,11 +288,43 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist col.name for col in schema_conn.introspection.get_table_description(cursor, model._meta.db_table) } - if old_mf.column not in existing_cols and new_mf.column in existing_cols: - logger.debug( - '_schema_alter_field: %r already renamed to %r on %s, skipping', - old_mf.column, new_mf.column, model._meta.db_table, + if old_mf.column not in existing_cols: + if new_mf.column in existing_cols: + logger.debug( + '_schema_alter_field: %r already renamed to %r on %s, skipping', + old_mf.column, new_mf.column, model._meta.db_table, + ) + return + if old_is_m2m: + # M2M fields have no physical column; the old through table is absent. + return + # Scalar field: neither the old nor the new column exists. The field was + # independently renamed in this schema (e.g. branch renamed A→X while main + # renamed A→Y; now applying main's rename to the branch). Look up the live + # field record in the target schema to find the actual column and rename it. + logger.warning( + '_schema_alter_field: rename conflict on %s — source column %r and ' + 'target column %r are both absent; field pk=%d was independently renamed ' + 'in this schema; resolving by looking up live column', + model._meta.db_table, old_mf.column, new_mf.column, new_fi.pk, ) + try: + live_fi = CustomObjectTypeField.objects.using(schema_conn.alias).get(pk=new_fi.pk) + except CustomObjectTypeField.DoesNotExist: + logger.debug( + '_schema_alter_field: field pk=%d not found in %s; skipping', + new_fi.pk, schema_conn.alias, + ) + return + live_mf = FIELD_TYPE_CLASS[live_fi.type]().get_model_field(live_fi) + live_mf.contribute_to_class(model, live_fi.name) + if live_mf.column not in existing_cols: + logger.debug( + '_schema_alter_field: live column %r also absent on %s; skipping', + live_mf.column, model._meta.db_table, + ) + return + schema_editor.alter_field(model, live_mf, new_mf) return if ( @@ -1089,41 +1127,6 @@ def save(self, using=None, **kwargs): return _SchemaAwareDeserialized(inner) - def has_branch_schema_drift(self, branch) -> bool: - """ - Return True if this custom object type's physical schema in *branch* - differs from the current field definitions in main. - - Drift means at least one of: - - A field exists in main but not in the branch snapshot (needs add_field) - - A field exists in the branch snapshot but not in main (needs remove_field) - - A field exists in both but has a different name, type, or constraint - that affects the DB column (needs alter_field) - - Returns False when netbox-branching is not installed. - """ - try: - from netbox_branching.utilities import activate_branch - except ImportError: - return False - - from netbox_custom_objects.branching import _field_schema_key - - main_fields = {f.pk: f for f in self.fields.all()} - with activate_branch(branch): - branch_fields = {f.pk: f for f in self.fields.all()} - - main_pks = set(main_fields) - branch_pks = set(branch_fields) - - if main_pks != branch_pks: - return True - - return any( - _field_schema_key(branch_fields[pk]) != _field_schema_key(main_fields[pk]) - for pk in main_pks - ) - def save(self, *args, **kwargs): needs_db_create = self._state.adding diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index fdc3ff47..88309f8c 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -690,29 +690,16 @@ def test_main_changes_synced_to_branch(self): self.assertEqual(co_branch.title, 'main object') -# ── Drift detection and Branch.migrate() tests ──────────────────────────────── +# ── Concurrent-edit tests (both main and branch modified before sync/merge) ─── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class BranchMigrateTestCase(TransactionCleanupMixin, TransactionTestCase): +class ConcurrentEditSyncTestCase(TransactionCleanupMixin, TransactionTestCase): """ - Tests for the drift-detection and schema-reconciliation path. - - Scenario: - 1. A COT and field(s) are created in main and a branch is provisioned - (branch has a full schema copy at provision time). - 2. A field is then modified in main (added, removed, renamed, or - unique-toggled). netbox-branching detects the change to the non-exempt - CustomObjectTypeField model and marks the branch PENDING_MIGRATIONS. - 3. branch.migrate(user) runs the normal Django migration pass and then - emits post_migrate, which fires on_branch_migrated. That handler - reconciles the branch's physical schema against main's current field - definitions. - 4. The branch is back to READY and its physical column layout matches main. - - These tests do NOT use merge/revert — they isolate the separate - on_branch_migrated reconciliation path for branches that pre-date a main - schema change. They use the iterative strategy only because the strategy - affects merge order, not schema reconciliation. + Sync scenarios where both main and branch accumulate changes before sync(). + + Mirrors netbox-branching's test_sync_m2m_tags_concurrent_changes pattern: + after sync(), main's ObjectChanges are applied on top of whatever the branch + did, so main's post-change state takes precedence for any conflicting record. """ def setUp(self): @@ -725,249 +712,687 @@ def tearDown(self): _close_branch_connections() super().tearDown() - def _get_branch_columns(self, branch, table_name): - """Return the set of column names on table_name in the branch schema.""" - conn = connections[branch.connection_name] - with conn.cursor() as cursor: - return {col.name for col in conn.introspection.get_table_description(cursor, table_name)} - - def _get_branch_tables(self, branch): - """Return the set of table names in the branch schema.""" - conn = connections[branch.connection_name] - with conn.cursor() as cursor: - return set(conn.introspection.table_names(cursor)) - - def _get_branch_constraints(self, branch, table_name): - """Return the constraints dict for table_name in the branch schema.""" - conn = connections[branch.connection_name] - with conn.cursor() as cursor: - return conn.introspection.get_constraints(cursor, table_name) - - # ── field added to main ─────────────────────────────────────────────── - - def test_field_added_to_main_triggers_branch_migrate(self): - """ - Field added to a COT in main marks the branch PENDING_MIGRATIONS. - branch.migrate() fires on_branch_migrated which calls add_field to - create the new column in the branch schema. - """ - cot = CustomObjectType.objects.create(name='drift_add_cot', slug='drift-add-cot') - CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='existing_field', - label='Existing', - type='text', - ) + def test_co_values_modified_in_both_sync(self): + """ + CO field values modified in both main and branch before sync. - branch = _provision_branch('Drift Add Branch', 'iterative', self.user) - table_name = cot.get_database_table_name() + Scenario + -------- + 1. Create COT + field 'notes' + two COs in main. + 2. Provision branch (branch sees same COs). + 3. In branch: update shared CO to 'modified in branch'. + 4. In main: update a different CO; create a brand-new CO. + 5. sync() applies main's ObjectChanges to the branch. + + Expected after sync + ------------------- + - Main's new CO is visible in the branch. + - CO modified in main has main's value in the branch. + - Branch's own CO modification is overwritten by main's replay for + the same PK (main wins on sync, same as tag-conflict behaviour). + """ + request = _make_request(self.user) - self.assertEqual(branch.status, BranchStatusChoices.READY) - self.assertIn('existing_field', self._get_branch_columns(branch, table_name)) + with event_tracking(request): + cot = CustomObjectType.objects.create(name='sync_co_cot', slug='sync-co-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='notes', label='Notes', type='text', + ) + Model = cot.get_model() + co_shared = Model.objects.create(notes='original shared') + co_main_only = Model.objects.create(notes='main only original') - # Add a new field to main — netbox-branching detects the change and marks branch PENDING. - CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='new_field', - label='New Field', - type='integer', - ) + co_shared_pk = co_shared.pk + co_main_only_pk = co_main_only.pk + branch = _provision_branch('Sync CO Both', 'iterative', self.user) + branch_request = _make_request(self.user) - branch.refresh_from_db() - self.assertEqual( - branch.status, BranchStatusChoices.PENDING_MIGRATIONS, - 'Branch must be PENDING_MIGRATIONS after field added to main', - ) - self.assertNotIn( - 'new_field', self._get_branch_columns(branch, table_name), - 'New column must not exist in branch before migrate', - ) + # ── branch: update the shared CO ───────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + BM = cot.get_model() + co = BM.objects.get(pk=co_shared_pk) + co.snapshot() + co.notes = 'modified in branch' + co.save() + branch_new = BM.objects.create(notes='new in branch') + branch_new_pk = branch_new.pk + + # ── main: update the other CO; add a new CO ─────────────────────── + with event_tracking(request): + MM = cot.get_model() + co = MM.objects.get(pk=co_main_only_pk) + co.snapshot() + co.notes = 'modified in main' + co.save() + main_new = MM.objects.create(notes='new in main') + main_new_pk = main_new.pk - # migrate() fires on_branch_migrated → add_field runs on the branch schema. - branch.migrate(user=self.user) - branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.READY) + # ── sync ────────────────────────────────────────────────────────── + branch.sync(user=self.user, commit=True) - self.assertIn( - 'new_field', self._get_branch_columns(branch, table_name), - 'New column must be present in branch schema after migrate', - ) + with activate_branch(branch): + SyncedCOT = CustomObjectType.objects.get(pk=cot.pk) + SM = SyncedCOT.get_model() + + # CO modified in main must reflect main's value. + self.assertEqual(SM.objects.get(pk=co_main_only_pk).notes, 'modified in main') + # CO created in main must be visible in branch after sync. + self.assertTrue(SM.objects.filter(pk=main_new_pk).exists(), + 'CO created in main must appear in branch after sync') + # CO created in branch was not deleted by sync. + self.assertTrue(SM.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must still exist after sync') + + def test_field_rename_in_branch_co_add_in_main_sync(self): + """ + Field renamed inside a branch; new CO added in main (no rename in main). + + Scenario + -------- + 1. Create COT + field 'alpha' + CO in main. + 2. Provision branch. + 3. In branch: rename 'alpha' → 'branch_alpha'; create a CO. + 4. In main: create a new CO using the original field name 'alpha'. + 5. sync() replays main's CO-create on top of the branch. + + Expected after sync + ------------------- + - Branch retains the renamed field ('branch_alpha'). + - CO created in main is present in branch. + - CO created in branch still exists. + + This exercises the schema-mismatch path: main's CO ObjectChange carries + the original field key 'alpha' while the branch schema uses 'branch_alpha'. + The CO data from main is applied by the branching engine regardless of the + column name divergence (branching replays at the data layer). + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='sync_rename_cot', slug='sync-rename-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + cot.get_model().objects.create(alpha='original') + + branch = _provision_branch('Sync Rename Branch', 'iterative', self.user) + branch_request = _make_request(self.user) - # ── field deleted from main ─────────────────────────────────────────── + # ── branch: rename alpha → branch_alpha; create CO ──────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + field.snapshot() + field.name = 'branch_alpha' + field.label = 'Branch Alpha' + field.save() + BM = cot.get_model() + branch_new = BM.objects.create(branch_alpha='new in branch') + branch_new_pk = branch_new.pk - def test_field_deleted_from_main_triggers_branch_migrate(self): + # ── main: add a CO (no field rename) ────────────────────────────── + with event_tracking(request): + cot.get_model().objects.create(alpha='new in main') + + # ── sync ────────────────────────────────────────────────────────── + branch.sync(user=self.user, commit=True) + + with activate_branch(branch): + cot_b = CustomObjectType.objects.get(pk=cot.pk) + field_b = CustomObjectTypeField.objects.get(custom_object_type=cot_b) + # Branch field rename must be preserved (main did not rename). + self.assertEqual(field_b.name, 'branch_alpha', + 'Branch field rename must be preserved after sync') + BM = cot_b.get_model() + self.assertTrue(BM.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must survive sync') + + def test_concurrent_field_rename_sync_no_crash(self): """ - Field deleted from a COT in main marks the branch PENDING_MIGRATIONS. - branch.migrate() fires on_branch_migrated which calls remove_field to - drop the column from the branch schema. + Field renamed to different names in both main and branch before sync. + + The same CustomObjectTypeField PK was modified in both schemas. + _schema_alter_field detects the conflict (neither the original 'alpha' + column nor the target 'main_alpha' column exists in the branch), looks up + the live column name in the branch ('branch_alpha'), and renames it to + 'main_alpha' to converge the branch schema on main's post-sync state. + + Scenario + -------- + 1. Create COT + field 'alpha' in main. + 2. Provision branch. + 3. Branch renames 'alpha' → 'branch_alpha'. + 4. Main renames 'alpha' → 'main_alpha'. + 5. sync() applies main's rename ObjectChange to the branch. + _schema_alter_field resolves the conflict: 'branch_alpha' → 'main_alpha'. + + Expected + -------- + - sync() completes without raising a DB error. + - Branch physical column is 'main_alpha' (converged to main's rename). + - 'branch_alpha' and 'alpha' columns are absent from the branch table. """ - cot = CustomObjectType.objects.create(name='drift_del_cot', slug='drift-del-cot') - CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='keep_field', - label='Keep', - type='text', - ) - drop_field = CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='drop_field', - label='Drop', - type='text', - ) + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='confl_sync_cot', slug='confl-sync-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) - branch = _provision_branch('Drift Del Branch', 'iterative', self.user) - table_name = cot.get_database_table_name() + branch = _provision_branch('Conflict Sync Branch', 'iterative', self.user) + branch_request = _make_request(self.user) - self.assertIn('drop_field', self._get_branch_columns(branch, table_name)) + # ── branch: rename alpha → branch_alpha ─────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.snapshot() + f.name = 'branch_alpha' + f.label = 'Branch Alpha' + f.save() - # Delete field from main. - drop_field.delete() + # ── main: rename alpha → main_alpha ─────────────────────────────── + with event_tracking(request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.name = 'main_alpha' + f.label = 'Main Alpha' + f.save() + + # ── sync — must not raise ────────────────────────────────────────── + try: + branch.sync(user=self.user, commit=True) + except Exception as exc: + self.fail(f'sync() raised an unexpected exception: {exc!r}') branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) - # Column still present in branch before migrate. - self.assertIn('drop_field', self._get_branch_columns(branch, table_name)) + # Branch column should now be 'main_alpha' — the conflict was resolved by + # renaming the branch's live 'branch_alpha' column to main's target name. + branch_conn = connections[branch.connection_name] + with branch_conn.cursor() as cursor: + branch_cols = { + col.name + for col in branch_conn.introspection.get_table_description( + cursor, cot.get_database_table_name(), + ) + } + self.assertIn('main_alpha', branch_cols, 'Branch column must converge to main_alpha after sync') + self.assertNotIn('branch_alpha', branch_cols, 'branch_alpha column must be gone after sync') + self.assertNotIn('alpha', branch_cols, 'alpha column must be gone after sync') - branch.migrate(user=self.user) - branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.READY) - cols = self._get_branch_columns(branch, table_name) - self.assertNotIn( - 'drop_field', cols, - 'Deleted column must be absent from branch schema after migrate', - ) - self.assertIn('keep_field', cols, 'Unmodified column must remain in branch schema') +# ── Concurrent-edit merge tests ─────────────────────────────────────────────── - # ── field renamed in main ───────────────────────────────────────────── +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class ConcurrentEditMergeTestCase(TransactionCleanupMixin, TransactionTestCase): + """ + Merge scenarios where both main and branch accumulate changes before merge(). + """ - def test_field_renamed_in_main_triggers_branch_migrate(self): + def setUp(self): + super().setUp() + _recreate_contenttypes() + self.user = User.objects.create_user(username='testuser') + self.request = _make_request(self.user) + + def tearDown(self): + _close_branch_connections() + super().tearDown() + + def test_field_rename_in_branch_co_changes_merge_iterative(self): """ - Field renamed in main marks the branch PENDING_MIGRATIONS. - branch.migrate() fires on_branch_migrated which calls alter_field to - rename the column in the branch schema using PK-based matching (same - PK in both main and branch, different name values). + Field renamed inside a branch; COs added/updated in branch; main adds a CO. + Iterative merge brings the branch rename and CO changes into main. + + Scenario + -------- + 1. Create COT + field 'alpha' + CO in main. + 2. Provision branch. + 3. In branch: rename 'alpha' → 'beta'; update the existing CO; create a CO. + 4. In main: add a CO (field name 'alpha', no rename). + 5. merge() → revert(). + + Expected after merge + -------------------- + - Field name in main is 'beta'. + - Existing CO has the branch-updated value. + - CO created in branch is present in main. + + Expected after revert + --------------------- + - Field name is 'alpha' again. + - CO created in branch is gone. + - Existing CO has its original value. """ - cot = CustomObjectType.objects.create(name='drift_ren_cot', slug='drift-ren-cot') - field = CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='old_col', - label='Old', - type='text', - ) + request = _make_request(self.user) - branch = _provision_branch('Drift Ren Branch', 'iterative', self.user) - table_name = cot.get_database_table_name() + with event_tracking(request): + cot = CustomObjectType.objects.create(name='merge_rename_cot', slug='merge-rename-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + Model = cot.get_model() + co_existing = Model.objects.create(alpha='original') - self.assertIn('old_col', self._get_branch_columns(branch, table_name)) + co_existing_pk = co_existing.pk + branch = _provision_branch('Merge Rename Branch', 'iterative', self.user) + branch_request = _make_request(self.user) - # Rename in main — load from DB so _original is set before modifying. - field = CustomObjectTypeField.objects.get(pk=field.pk) - field.name = 'new_col' - field.label = 'New' - field.save() + # ── branch: rename; update CO; create CO ────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + field.snapshot() + field.name = 'beta' + field.label = 'Beta' + field.save() + BM = cot.get_model() + co = BM.objects.get(pk=co_existing_pk) + co.snapshot() + co.beta = 'updated in branch' + co.save() + branch_new = BM.objects.create(beta='new in branch') + branch_new_pk = branch_new.pk + + # ── main: add a CO (no schema change) ───────────────────────────── + with event_tracking(request): + main_new = cot.get_model().objects.create(alpha='new in main') + main_new_pk = main_new.pk + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) + self.assertEqual(branch.status, BranchStatusChoices.MERGED) - branch.migrate(user=self.user) - branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.READY) + MergedModel = cot.get_model() + field_main = CustomObjectTypeField.objects.get(custom_object_type=cot) + self.assertEqual(field_main.name, 'beta', 'Field must be "beta" in main after merge') + self.assertEqual(MergedModel.objects.get(pk=co_existing_pk).beta, 'updated in branch') + self.assertTrue(MergedModel.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must be in main after merge') + self.assertTrue(MergedModel.objects.filter(pk=main_new_pk).exists(), + 'CO added in main must still be present after merge') - cols = self._get_branch_columns(branch, table_name) - self.assertIn('new_col', cols, 'Renamed column must exist in branch schema after migrate') - self.assertNotIn('old_col', cols, 'Old column name must be absent from branch schema after migrate') + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) - # ── unique constraint toggled in main ───────────────────────────────── + field_reverted = CustomObjectTypeField.objects.get(custom_object_type=cot) + self.assertEqual(field_reverted.name, 'alpha', 'Field must be "alpha" after revert') + RevertedModel = cot.get_model() + self.assertEqual(RevertedModel.objects.get(pk=co_existing_pk).alpha, 'original') + self.assertFalse(RevertedModel.objects.filter(pk=branch_new_pk).exists(), + 'CO created in branch must be gone after revert') - def test_unique_toggled_in_main_triggers_branch_migrate(self): + def test_co_values_modified_in_both_merge_iterative(self): """ - Field's unique constraint toggled in main marks the branch - PENDING_MIGRATIONS. branch.migrate() reconciles the constraint in - the branch's physical schema. + CO values modified in both main and branch before merge. + Branch changes win because merge applies branch ObjectChanges to main. + + Scenario + -------- + 1. Create COT + field 'notes' + shared CO in main. + 2. Provision branch. + 3. Branch updates shared CO to 'modified in branch'; creates a CO. + 4. Main creates a separate CO. + 5. merge() → revert(). + + Expected after merge: branch CO changes are in main, main CO preserved. + Expected after revert: shared CO back to original, branch CO gone. """ - cot = CustomObjectType.objects.create(name='drift_uniq_cot', slug='drift-uniq-cot') - field = CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='code', - label='Code', - type='text', - unique=False, - ) + request = _make_request(self.user) - branch = _provision_branch('Drift Uniq Branch', 'iterative', self.user) - table_name = cot.get_database_table_name() + with event_tracking(request): + cot = CustomObjectType.objects.create(name='merge_co_cot', slug='merge-co-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='notes', label='Notes', type='text', + ) + Model = cot.get_model() + co_shared = Model.objects.create(notes='original') - # Branch must not have a unique constraint on 'code' initially. - constraints_before = self._get_branch_constraints(branch, table_name) - self.assertFalse( - any(c['unique'] and c.get('columns') == ['code'] for c in constraints_before.values()), - 'Branch must not have UNIQUE on "code" before toggle', - ) + co_shared_pk = co_shared.pk + branch = _provision_branch('Merge CO Both', 'iterative', self.user) + branch_request = _make_request(self.user) - # Toggle unique=True in main. - field = CustomObjectTypeField.objects.get(pk=field.pk) - field.unique = True - field.save() + # ── branch ──────────────────────────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + BM = cot.get_model() + co = BM.objects.get(pk=co_shared_pk) + co.snapshot() + co.notes = 'modified in branch' + co.save() + branch_new = BM.objects.create(notes='new in branch') + branch_new_pk = branch_new.pk + + # ── main ────────────────────────────────────────────────────────── + with event_tracking(request): + main_new = cot.get_model().objects.create(notes='new in main') + main_new_pk = main_new.pk + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) + self.assertEqual(branch.status, BranchStatusChoices.MERGED) - branch.migrate(user=self.user) - branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.READY) + MM = cot.get_model() + self.assertEqual(MM.objects.get(pk=co_shared_pk).notes, 'modified in branch') + self.assertTrue(MM.objects.filter(pk=branch_new_pk).exists()) + self.assertTrue(MM.objects.filter(pk=main_new_pk).exists(), + 'CO added to main must survive the merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + RM = cot.get_model() + self.assertEqual(RM.objects.get(pk=co_shared_pk).notes, 'original') + self.assertFalse(RM.objects.filter(pk=branch_new_pk).exists()) - constraints_after = self._get_branch_constraints(branch, table_name) - self.assertTrue( - any(c['unique'] and c.get('columns') == ['code'] for c in constraints_after.values()), - 'Branch must have UNIQUE constraint on "code" after migrate', - ) - # ── multiobject field renamed in main (through-table rename) ───────── +# ── Sequential multi-rename tests ───────────────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SequentialRenameTestCase(TransactionCleanupMixin, TransactionTestCase): + """ + Tests for sequential field renames (A→B→C) in a branch with CO changes at + each step, plus independent changes in main. + + Exercises the iterative ObjectChange replay order: the rename chain must be + applied in the right sequence so that each CO update sees the correct column + name at merge time. + + Run with both iterative and squash strategies to verify that squash correctly + collapses the A→B→C chain to a single A→C alter. + """ + + MERGE_STRATEGY = 'iterative' + + def setUp(self): + super().setUp() + _recreate_contenttypes() + self.user = User.objects.create_user(username='testuser') + self.request = _make_request(self.user) + + def tearDown(self): + _close_branch_connections() + super().tearDown() - def test_multiobject_field_renamed_in_main_triggers_branch_migrate(self): + def _run_sequential_rename_merge(self, cot_name, cot_slug): """ - MULTIOBJECT field renamed in main marks the branch PENDING_MIGRATIONS. - branch.migrate() fires on_branch_migrated which renames the through - table in the branch schema in addition to the column alter_field. + Shared implementation for the sequential rename merge test. - Exercises the MULTIOBJECT rename branch of _schema_alter_field. + Branch: rename alpha→beta (update+create CO), rename beta→gamma (update+create CO). + Main: add a new independent field + CO (no rename of alpha). + merge() then revert(). """ - from core.models import ObjectType + request = _make_request(self.user) - site_ot = ObjectType.objects.get(app_label='dcim', model='site') + with event_tracking(request): + cot = CustomObjectType.objects.create(name=cot_name, slug=cot_slug) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + Model = cot.get_model() + co_original = Model.objects.create(alpha='original value') - cot = CustomObjectType.objects.create(name='drift_m2m_cot', slug='drift-m2m-cot') - field = CustomObjectTypeField.objects.create( - custom_object_type=cot, - name='related_sites', - label='Related Sites', - type='multiobject', - related_object_type=site_ot, - ) + co_original_pk = co_original.pk + branch = _provision_branch(f'{cot_name} branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) - old_through = field.through_table_name + # ── branch: alpha → beta; update CO; create CO ──────────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + field.snapshot() + field.name = 'beta' + field.label = 'Beta' + field.save() + BM = cot.get_model() + co = BM.objects.get(pk=co_original_pk) + co.snapshot() + co.beta = 'after rename to beta' + co.save() + co_at_beta = BM.objects.create(beta='created at beta') + co_at_beta_pk = co_at_beta.pk + + # ── branch: beta → gamma; update CO; create CO ──────────────────── + with activate_branch(branch), event_tracking(branch_request): + field = CustomObjectTypeField.objects.get(custom_object_type=cot, name='beta') + field.snapshot() + field.name = 'gamma' + field.label = 'Gamma' + field.save() + BM = cot.get_model() + co = BM.objects.get(pk=co_original_pk) + co.snapshot() + co.gamma = 'after rename to gamma' + co.save() + co_at_gamma = BM.objects.create(gamma='created at gamma') + co_at_gamma_pk = co_at_gamma.pk + + # ── main: add a new independent field + CO ──────────────────────── + with event_tracking(request): + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='extra', label='Extra', type='text', + ) + co_main = cot.get_model().objects.create(alpha='main added', extra='extra val') + co_main_pk = co_main.pk - branch = _provision_branch('Drift M2M Branch', 'iterative', self.user) + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_names = {f.name for f in CustomObjectTypeField.objects.filter(custom_object_type=cot)} + self.assertIn('gamma', field_names, 'Final field name must be "gamma" after merge') + self.assertNotIn('alpha', field_names, '"alpha" must be absent after merge') + self.assertNotIn('beta', field_names, '"beta" must be absent after merge') + self.assertIn('extra', field_names, '"extra" field from main must be present after merge') + + MergedModel = cot.get_model() + self.assertEqual(MergedModel.objects.get(pk=co_original_pk).gamma, 'after rename to gamma') + self.assertTrue(MergedModel.objects.filter(pk=co_at_beta_pk).exists(), + 'CO created at beta step must survive merge') + self.assertTrue(MergedModel.objects.filter(pk=co_at_gamma_pk).exists(), + 'CO created at gamma step must survive merge') + self.assertTrue(MergedModel.objects.filter(pk=co_main_pk).exists(), + 'CO added in main must survive merge') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) - self.assertIn(old_through, self._get_branch_tables(branch)) + field_names_r = {f.name for f in CustomObjectTypeField.objects.filter(custom_object_type=cot)} + self.assertIn('alpha', field_names_r, '"alpha" must be restored after revert') + self.assertNotIn('gamma', field_names_r, '"gamma" must be gone after revert') - # Rename in main. - field = CustomObjectTypeField.objects.get(pk=field.pk) - field.name = 'linked_sites' - field.label = 'Linked Sites' - field.save() + RevertedModel = cot.get_model() + self.assertEqual(RevertedModel.objects.get(pk=co_original_pk).alpha, 'original value', + 'Original CO value must be restored after revert') + self.assertFalse(RevertedModel.objects.filter(pk=co_at_beta_pk).exists()) + self.assertFalse(RevertedModel.objects.filter(pk=co_at_gamma_pk).exists()) - new_through = field.through_table_name + def test_sequential_renames_alpha_beta_gamma_merge(self): + """Field renamed A→B→C in branch with CO changes at each step; merge + revert.""" + self._run_sequential_rename_merge('seq_iter_cot', 'seq-iter-cot') + + def test_sequential_renames_both_sides_sync(self): + """ + Branch renames A→B→C while main renames A→D. + Both schemas independently rename the same field to different names. + + After sync(), main's rename (A→D) is applied on top of the branch's + state. Because 'alpha' no longer exists in the branch (it was renamed + to 'gamma' via beta), _schema_alter_field detects the conflict, looks up + the live column name in the branch ('gamma'), and renames it to 'delta' + to converge the branch schema on main's post-sync state. + + Scenario + -------- + 1. Create COT + field 'alpha' + CO in main. + 2. Provision branch. + 3. Branch: alpha→beta (update CO), beta→gamma (update CO, add CO). + 4. Main: alpha→delta (update CO, add CO). + 5. sync(): apply main's changes to branch — column converges to 'delta'. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='seq_sync_cot', slug='seq-sync-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + co = cot.get_model().objects.create(alpha='original') + + co_pk = co.pk + branch = _provision_branch('Seq Sync Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + # ── branch: alpha → beta → gamma ────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.snapshot() + f.name = 'beta' + f.label = 'Beta' + f.save() + BM = cot.get_model() + co_b = BM.objects.get(pk=co_pk) + co_b.snapshot() + co_b.beta = 'at beta' + co_b.save() + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='beta') + f.snapshot() + f.name = 'gamma' + f.label = 'Gamma' + f.save() + BM = cot.get_model() + co_g = BM.objects.get(pk=co_pk) + co_g.snapshot() + co_g.gamma = 'at gamma' + co_g.save() + BM.objects.create(gamma='new in branch') + + # ── main: alpha → delta ─────────────────────────────────────────── + with event_tracking(request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.name = 'delta' + f.label = 'Delta' + f.save() + MM = cot.get_model() + co_m = MM.objects.get(pk=co_pk) + co_m.snapshot() + co_m.delta = 'updated in main' + co_m.save() + MM.objects.create(delta='main new') + + # ── sync ────────────────────────────────────────────────────────── + try: + branch.sync(user=self.user, commit=True) + except Exception as exc: + self.fail(f'sync() must not raise when schemas have conflicting renames: {exc!r}') branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.PENDING_MIGRATIONS) - branch.migrate(user=self.user) + # _schema_alter_field resolved the conflict by looking up the live column + # ('gamma') in the branch and renaming it to main's target name ('delta'). + branch_conn = connections[branch.connection_name] + with branch_conn.cursor() as cursor: + branch_cols = { + col.name + for col in branch_conn.introspection.get_table_description( + cursor, cot.get_database_table_name(), + ) + } + self.assertIn('delta', branch_cols, 'Branch column must converge to delta after sync') + self.assertNotIn('gamma', branch_cols, 'gamma column must be gone after sync') + self.assertNotIn('beta', branch_cols, 'beta column must be gone after sync') + self.assertNotIn('alpha', branch_cols, 'alpha column must be gone after sync') + + def test_sequential_renames_both_sides_merge(self): + """ + Branch renames A→B→C; main renames A→D independently. + merge() applies branch's rename chain to main. + + When merging the first branch rename (alpha→beta) into main, 'alpha' no + longer exists in main (it was renamed to 'delta') and 'beta' doesn't exist + either. _schema_alter_field detects the conflict, looks up the live column + in main ('delta'), and renames it to 'beta'. The second branch rename + (beta→gamma) then finds 'beta' in main and renames it normally to 'gamma'. + + Expected after merge: main's physical column is 'gamma'. + """ + request = _make_request(self.user) + + with event_tracking(request): + cot = CustomObjectType.objects.create(name='seq_merge_cot', slug='seq-merge-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='alpha', label='Alpha', type='text', + ) + co = cot.get_model().objects.create(alpha='original') + + co_pk = co.pk + branch = _provision_branch('Seq Merge Conflict Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + # ── branch: alpha → beta → gamma ────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.snapshot() + f.name = 'beta' + f.label = 'Beta' + f.save() + BM = cot.get_model() + co_b = BM.objects.get(pk=co_pk) + co_b.snapshot() + co_b.beta = 'at beta' + co_b.save() + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='beta') + f.snapshot() + f.name = 'gamma' + f.label = 'Gamma' + f.save() + BM = cot.get_model() + co_g = BM.objects.get(pk=co_pk) + co_g.snapshot() + co_g.gamma = 'at gamma' + co_g.save() + BM.objects.create(gamma='new in branch') + + # ── main: alpha → delta ─────────────────────────────────────────── + with event_tracking(request): + f = CustomObjectTypeField.objects.get(custom_object_type=cot, name='alpha') + f.name = 'delta' + f.label = 'Delta' + f.save() + + # ── merge ───────────────────────────────────────────────────────── + try: + branch.merge(user=self.user, commit=True) + except Exception as exc: + self.fail(f'merge() must not raise when schemas have conflicting renames: {exc!r}') + branch.refresh_from_db() - self.assertEqual(branch.status, BranchStatusChoices.READY) - branch_tables = self._get_branch_tables(branch) - self.assertIn(new_through, branch_tables, 'Renamed through table must exist in branch after migrate') - self.assertNotIn(old_through, branch_tables, 'Old through table must be absent from branch after migrate') + # _schema_alter_field resolved the conflict for the first branch rename + # (alpha→beta): it found 'delta' (main's live column) and renamed it to + # 'beta'. The second rename (beta→gamma) then proceeded normally. + # Main's final physical column should be 'gamma'. + from django.db import connection as main_conn + with main_conn.cursor() as cursor: + main_cols = { + col.name + for col in main_conn.introspection.get_table_description( + cursor, cot.get_database_table_name(), + ) + } + self.assertIn('gamma', main_cols, 'Main column must be gamma after merge') + self.assertNotIn('delta', main_cols, 'delta column must be gone after merge') + self.assertNotIn('beta', main_cols, 'beta column must be gone after merge') + self.assertNotIn('alpha', main_cols, 'alpha column must be gone after merge') + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SequentialRenameSquashTestCase(SequentialRenameTestCase, TransactionTestCase): + """Run SequentialRenameTestCase with the squash merge strategy.""" + MERGE_STRATEGY = 'squash' + + def test_sequential_renames_alpha_beta_gamma_merge(self): + self._run_sequential_rename_merge('seq_squash_cot', 'seq-squash-cot') From 5431a3870c11ed89255d425dbfc042f9549a9dcc Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 14:30:55 -0700 Subject: [PATCH 022/115] freeze db column name --- .../0008_customobjecttypefield_db_column.py | 46 +++++++++++++++++++ netbox_custom_objects/models.py | 37 ++++++++++++--- 2 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py diff --git a/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py b/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py new file mode 100644 index 00000000..1a40504c --- /dev/null +++ b/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py @@ -0,0 +1,46 @@ +""" +Add ``db_column`` to CustomObjectTypeField and back-fill it from ``name``. + +``db_column`` is frozen at field creation time so that subsequent renames are +pure metadata operations — the physical database column name never changes. +This prevents cross-schema column-name mismatches when a field is renamed in +one schema (e.g. a branch) and the model is then used to query a different +schema (e.g. main) that still has the original column name. + +The data migration sets ``db_column = name`` for all existing fields so that +``effective_db_column`` returns the same value as before the migration. +""" + +from django.db import migrations, models + + +def backfill_db_column(apps, schema_editor): + """Set db_column = name for all existing CustomObjectTypeField rows.""" + CustomObjectTypeField = apps.get_model('netbox_custom_objects', 'CustomObjectTypeField') + CustomObjectTypeField.objects.filter(db_column='').update(db_column=models.F('name')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_custom_objects', '0007_fix_object_field_fk_deferrable'), + ] + + operations = [ + migrations.AddField( + model_name='customobjecttypefield', + name='db_column', + field=models.CharField( + blank=True, + default='', + help_text='Physical database column name. Set once at creation and never changed, so renames are pure metadata changes that do not require DDL.', + max_length=50, + verbose_name='database column', + ), + preserve_default=False, + ), + migrations.RunPython( + backfill_db_column, + migrations.RunPython.noop, + ), + ] diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 733c2804..3a8bc8b4 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -184,7 +184,7 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): sync/merge replays an ObjectChange that was already applied). """ ft = FIELD_TYPE_CLASS[fi.type]() - mf = ft.get_model_field(fi) + mf = ft.get_model_field(fi, db_column=fi.effective_db_column) mf.contribute_to_class(model, fi.name) with schema_conn.cursor() as cursor: @@ -215,7 +215,7 @@ def _schema_remove_field(fi, model, schema_editor, existing_tables=None): to reject the subsequent ALTER TABLE. """ ft = FIELD_TYPE_CLASS[fi.type]() - mf = ft.get_model_field(fi) + mf = ft.get_model_field(fi, db_column=fi.effective_db_column) mf.contribute_to_class(model, fi.name) if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: @@ -278,8 +278,8 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist ) return - old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) - new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) + old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi, db_column=old_fi.effective_db_column) + new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi, db_column=new_fi.effective_db_column) old_mf.contribute_to_class(model, old_fi.name) new_mf.contribute_to_class(model, new_fi.name) @@ -316,7 +316,7 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist new_fi.pk, schema_conn.alias, ) return - live_mf = FIELD_TYPE_CLASS[live_fi.type]().get_model_field(live_fi) + live_mf = FIELD_TYPE_CLASS[live_fi.type]().get_model_field(live_fi, db_column=live_fi.effective_db_column) live_mf.contribute_to_class(model, live_fi.name) if live_mf.column not in existing_cols: logger.debug( @@ -750,7 +750,7 @@ def _fetch_and_generate_field_attrs( field_name = field.name field_attrs[field.name] = field_type.get_model_field( - field, + field, db_column=field.effective_db_column, ) # Add to field objects only if the field was successfully generated @@ -1324,6 +1324,15 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode ), ), ) + db_column = models.CharField( + verbose_name=_("database column"), + max_length=50, + blank=True, + help_text=_( + "Physical database column name. Set once at creation and never changed, " + "so renames are pure metadata changes that do not require DDL." + ), + ) label = models.CharField( verbose_name=_("label"), max_length=50, @@ -1517,6 +1526,16 @@ def is_single_value(self): def many(self): return self.type in ["multiobject"] + @property + def effective_db_column(self): + """ + Return the physical database column name for this field. + + ``db_column`` is frozen at creation time so that renames are pure + metadata operations — the physical column name never changes. + """ + return self.db_column + def get_child_relations(self, instance): return instance.get_field_value(self) @@ -2149,6 +2168,12 @@ def save(self, using=None, **kwargs): def save(self, *args, **kwargs): is_new = self._state.adding + # Freeze the physical column name at creation. db_column is set once + # here and never updated, so subsequent renames only update the ORM + # field name — no DDL is required for renames. + if self._state.adding and not self.db_column: + self.db_column = self.name + # Use the branch connection when operating inside a branch so that schema # editor operations target the branch schema rather than main. schema_conn = _get_schema_connection() From 44f834a8287e0ea24d46cf99fcb7810109ef707a Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 16:14:52 -0700 Subject: [PATCH 023/115] freeze db column name --- netbox_custom_objects/__init__.py | 49 +++++++++++++++++++++++++++++++ netbox_custom_objects/forms.py | 1 + netbox_custom_objects/models.py | 4 +-- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index af41097c..0b2d104c 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -32,6 +32,51 @@ def _migration_finished(sender, **kwargs): _migrations_checked = None +def _patch_check_object_accessible_in_branch(): + """ + Patch check_object_accessible_in_branch to use an existence check instead of + a full SELECT for custom object models. + + The original implementation does model.objects.get(pk=object_id) which issues + SELECT * including every custom field column. If a field was renamed in the + branch but the stable db_column is not yet reflected in the model (e.g. due to + a stale cache), this can raise ProgrammingError. For custom objects we only + need to know whether the row exists, so filter(pk=...).exists() is sufficient + and avoids referencing any column other than the primary key. + """ + try: + import netbox_branching.signal_receivers as _sr + from netbox_branching.utilities import deactivate_branch + from netbox_branching.models import ChangeDiff + from core.choices import ObjectChangeActionChoices + from django.contrib.contenttypes.models import ContentType + + _original = _sr.check_object_accessible_in_branch + + def _patched(branch, model, object_id): + if model._meta.app_label != APP_LABEL: + return _original(branch, model, object_id) + + # Check existence in main using only the pk — avoids SELECT on + # renamed columns that may not yet exist in main. + with deactivate_branch(): + if model.objects.filter(pk=object_id).exists(): + return True + + # Not in main — was it created in this branch? + content_type = ContentType.objects.get_for_model(model) + return ChangeDiff.objects.filter( + branch=branch, + object_type=content_type, + object_id=object_id, + action=ObjectChangeActionChoices.ACTION_CREATE, + ).exists() + + _sr.check_object_accessible_in_branch = _patched + except (ImportError, AttributeError): + pass + + def _patch_object_selector_view(): """ Patch ObjectSelectorView to support dynamically-generated custom object models. @@ -183,6 +228,10 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() + # Patch check_object_accessible_in_branch to use pk-only existence check, + # avoiding SELECT * which references renamed columns that may not exist in main. + _patch_check_object_accessible_in_branch() + # Suppress warnings about database calls during app initialization with warnings.catch_warnings(): warnings.filterwarnings( diff --git a/netbox_custom_objects/forms.py b/netbox_custom_objects/forms.py index 0642ca34..1157c231 100644 --- a/netbox_custom_objects/forms.py +++ b/netbox_custom_objects/forms.py @@ -176,6 +176,7 @@ class CustomObjectTypeFieldForm(CustomFieldForm): class Meta: model = CustomObjectTypeField fields = "__all__" + exclude = ('db_column',) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 3a8bc8b4..ba1ac9bf 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1660,11 +1660,11 @@ def clean(self): and not self.original.unique ): field_type = FIELD_TYPE_CLASS[self.type]() - model_field = field_type.get_model_field(self) + model_field = field_type.get_model_field(self, db_column=self.effective_db_column) model = self.custom_object_type.get_model() model_field.contribute_to_class(model, self.name) - old_field = field_type.get_model_field(self.original) + old_field = field_type.get_model_field(self.original, db_column=self.original.effective_db_column) old_field.contribute_to_class(model, self._original_name) try: From fe534f25edf77f89a06c69e3525094f196b0d14b Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 17:32:30 -0700 Subject: [PATCH 024/115] freeze db column name --- netbox_custom_objects/__init__.py | 43 +++++++++++++++++++++++++++++++ netbox_custom_objects/models.py | 7 +++++ 2 files changed, 50 insertions(+) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 0b2d104c..91f86e41 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -32,6 +32,45 @@ def _migration_finished(sender, **kwargs): _migrations_checked = None +def _patch_get_serializer_for_model(): + """ + Patch utilities.api.get_serializer_for_model to handle dynamically-generated + custom object models. + + The default implementation resolves serializers by import path convention + (e.g. netbox_custom_objects.api.serializers.Table1ModelSerializer). Dynamic + models have no importable serializer at that path, so the call raises + SerializerNotFound. This patch intercepts the lookup for APP_LABEL models and + delegates to get_serializer_class(), which generates the serializer on the fly. + """ + import utilities.api as _api_utils + from netbox.api.exceptions import SerializerNotFound + + _original = _api_utils.get_serializer_for_model + + def _patched(model, prefix=''): + # Only intercept dynamically-generated custom object models (Table1Model, + # Table2Model, …) identified by their Table{n}Model name pattern. + # CustomObjectType and CustomObjectTypeField live in the same app but + # have importable serializers and must go through the normal path. + if getattr(model, '_meta', None) and model._meta.app_label == APP_LABEL \ + and extract_cot_id_from_model_name(model.__name__.lower()) is not None: + from netbox_custom_objects.api.serializers import get_serializer_class + return get_serializer_class(model) + return _original(model, prefix=prefix) + + _api_utils.get_serializer_for_model = _patched + + # Also patch the reference already imported into extras.events (and anywhere + # else that did `from utilities.api import get_serializer_for_model` before + # our patch ran). + try: + import extras.events as _extras_events + _extras_events.get_serializer_for_model = _patched + except (ImportError, AttributeError): + pass + + def _patch_check_object_accessible_in_branch(): """ Patch check_object_accessible_in_branch to use an existence check instead of @@ -228,6 +267,10 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() + # Patch get_serializer_for_model so event rules, job serializers, etc. can + # resolve serializers for dynamically-generated custom object models. + _patch_get_serializer_for_model() + # Patch check_object_accessible_in_branch to use pk-only existence check, # avoiding SELECT * which references renamed columns that may not exist in main. _patch_check_object_accessible_in_branch() diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index ba1ac9bf..e5dba2fd 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -708,6 +708,13 @@ def get_cached_through_models(cls, custom_object_type_id): """ return cls._through_model_cache.get(custom_object_type_id, {}) + def serialize_object(self, exclude=None): + # cache_timestamp is an internal cache-invalidation field; exclude it + # from ObjectChange records so it doesn't appear as a tracked change. + extra = ['cache_timestamp'] + combined = list(exclude or []) + extra + return super().serialize_object(exclude=combined) + def get_absolute_url(self): return reverse("plugins:netbox_custom_objects:customobjecttype", args=[self.pk]) From df61efbbc49086d4ed623b86f3a6f7758ec05750 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 23 Apr 2026 17:44:58 -0700 Subject: [PATCH 025/115] freeze db column name --- netbox_custom_objects/__init__.py | 1 - .../0008_customobjecttypefield_db_column.py | 5 ++++- netbox_custom_objects/models.py | 12 +++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 91f86e41..b1ec966e 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -44,7 +44,6 @@ def _patch_get_serializer_for_model(): delegates to get_serializer_class(), which generates the serializer on the fly. """ import utilities.api as _api_utils - from netbox.api.exceptions import SerializerNotFound _original = _api_utils.get_serializer_for_model diff --git a/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py b/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py index 1a40504c..70ff4917 100644 --- a/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py +++ b/netbox_custom_objects/migrations/0008_customobjecttypefield_db_column.py @@ -33,7 +33,10 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, default='', - help_text='Physical database column name. Set once at creation and never changed, so renames are pure metadata changes that do not require DDL.', + help_text=( + 'Physical database column name. Set once at creation and never changed, ' + 'so renames are pure metadata changes that do not require DDL.' + ), max_length=50, verbose_name='database column', ), diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index e5dba2fd..ba0c27b5 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -2234,8 +2234,18 @@ def ensure_constraint(): transaction.on_commit(ensure_constraint) + # Regenerate the model from DB now that the field change is persisted. + # _schema_alter_field adds both old and new field names to the model + # class via contribute_to_class, and something between clear_model_cache + # and super().save() (e.g. a post_save signal on the COT) can call + # get_model() while the DB still has the old name, caching a stale model + # without the new name. Forcing a no_cache regeneration here (after + # super().save() committed the new name) ensures apps.all_models holds a + # clean model with exactly the current DB field list. + updated_model = self.custom_object_type.get_model(no_cache=True) + # Reregister SearchIndex with new set of searchable fields - self.custom_object_type.register_custom_object_search_index(model) + self.custom_object_type.register_custom_object_search_index(updated_model) # Reindex all objects of this type if search indexing was affected if is_new: From f12e4a8a47f409b36ae55af6b8802dc85d55f0ee Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 11:40:33 -0700 Subject: [PATCH 026/115] cleanup --- netbox_custom_objects/__init__.py | 29 ++++++++++++++----- .../migrations/0011_branching_support.py | 2 +- netbox_custom_objects/models.py | 3 -- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 76646204..58ab9e6e 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -50,7 +50,18 @@ def _patch_get_serializer_for_model(): models have no importable serializer at that path, so the call raises SerializerNotFound. This patch intercepts the lookup for APP_LABEL models and delegates to get_serializer_class(), which generates the serializer on the fly. + + Patching the source module (utilities.api) is not enough: any module that + did ``from utilities.api import get_serializer_for_model`` before ``ready()`` + ran holds its own bound reference and would call the unpatched version. + Known callers in NetBox core include ``extras.events``, ``extras.api.customfields``, + ``extras.api.serializers_.tags``, ``core.api.serializers_.jobs``, + ``netbox.api.gfk_fields``, ``netbox.api.serializers.generic``, ``ipam.api.views`` + and several ``dcim.api`` modules. Rather than enumerate them (and break when + NetBox adds new ones), we sweep ``sys.modules`` and rebind every module-level + attribute that points at the original function. """ + import sys import utilities.api as _api_utils _original = _api_utils.get_serializer_for_model @@ -68,14 +79,16 @@ def _patched(model, prefix=''): _api_utils.get_serializer_for_model = _patched - # Also patch the reference already imported into extras.events (and anywhere - # else that did `from utilities.api import get_serializer_for_model` before - # our patch ran). - try: - import extras.events as _extras_events - _extras_events.get_serializer_for_model = _patched - except (ImportError, AttributeError): - pass + # Rebind every module that imported the original symbol by name. + for _mod in list(sys.modules.values()): + if _mod is None or _mod is _api_utils: + continue + if getattr(_mod, 'get_serializer_for_model', None) is _original: + try: + _mod.get_serializer_for_model = _patched + except (AttributeError, TypeError): + # Some module objects (e.g. namespace packages) may reject attribute writes. + pass def _patch_check_object_accessible_in_branch(): diff --git a/netbox_custom_objects/migrations/0011_branching_support.py b/netbox_custom_objects/migrations/0011_branching_support.py index a9bb5a64..36d9af70 100644 --- a/netbox_custom_objects/migrations/0011_branching_support.py +++ b/netbox_custom_objects/migrations/0011_branching_support.py @@ -52,7 +52,7 @@ def fix_deferrable_fk_constraints(apps, schema_editor): ON tc.constraint_name = rc.constraint_name AND tc.table_schema = rc.constraint_schema WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_name LIKE 'custom_objects\\_%%' + AND tc.table_name LIKE 'custom_objects\\_%' AND tc.is_deferrable = 'YES' """) rows = cursor.fetchall() diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index f8af3de8..dd3664da 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -63,9 +63,6 @@ logger = logging.getLogger(__name__) -logger = logging.getLogger('netbox_custom_objects.models') - - class UniquenessConstraintTestError(Exception): """Custom exception used to signal successful uniqueness constraint test.""" From 7c1f4e0f1af70091a6a123a3da55e121343ed818 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 11:40:50 -0700 Subject: [PATCH 027/115] cleanup --- netbox_custom_objects/api/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox_custom_objects/api/views.py b/netbox_custom_objects/api/views.py index 1bf4cdb4..dfb71959 100644 --- a/netbox_custom_objects/api/views.py +++ b/netbox_custom_objects/api/views.py @@ -94,6 +94,7 @@ def _serialize_diff(diff) -> dict: # Constants BRANCH_ACTIVE_ERROR_MESSAGE = _("Please switch to the main branch to perform this operation.") + class RootView(APIRootView): def get_view_name(self): return "CustomObjects" From 754875e0d753216e3ba2ef1fad37c2e0a0e1ba6d Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 11:55:00 -0700 Subject: [PATCH 028/115] add test for squash merge --- netbox_custom_objects/tests/test_branching.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 88309f8c..362a469b 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -923,11 +923,17 @@ def test_concurrent_field_rename_sync_no_crash(self): # ── Concurrent-edit merge tests ─────────────────────────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class ConcurrentEditMergeTestCase(TransactionCleanupMixin, TransactionTestCase): +class BaseConcurrentEditMergeTests(TransactionCleanupMixin): """ Merge scenarios where both main and branch accumulate changes before merge(). + + Subclasses must: + - set ``MERGE_STRATEGY`` to an iterative or squash strategy string + - also inherit from ``TransactionTestCase`` """ + MERGE_STRATEGY = None + def setUp(self): super().setUp() _recreate_contenttypes() @@ -938,10 +944,10 @@ def tearDown(self): _close_branch_connections() super().tearDown() - def test_field_rename_in_branch_co_changes_merge_iterative(self): + def test_field_rename_in_branch_co_changes_merge(self): """ Field renamed inside a branch; COs added/updated in branch; main adds a CO. - Iterative merge brings the branch rename and CO changes into main. + Merge brings the branch rename and CO changes into main. Scenario -------- @@ -974,7 +980,7 @@ def test_field_rename_in_branch_co_changes_merge_iterative(self): co_existing = Model.objects.create(alpha='original') co_existing_pk = co_existing.pk - branch = _provision_branch('Merge Rename Branch', 'iterative', self.user) + branch = _provision_branch('Merge Rename Branch', self.MERGE_STRATEGY, self.user) branch_request = _make_request(self.user) # ── branch: rename; update CO; create CO ────────────────────────── @@ -1021,7 +1027,7 @@ def test_field_rename_in_branch_co_changes_merge_iterative(self): self.assertFalse(RevertedModel.objects.filter(pk=branch_new_pk).exists(), 'CO created in branch must be gone after revert') - def test_co_values_modified_in_both_merge_iterative(self): + def test_co_values_modified_in_both_merge(self): """ CO values modified in both main and branch before merge. Branch changes win because merge applies branch ObjectChanges to main. @@ -1048,7 +1054,7 @@ def test_co_values_modified_in_both_merge_iterative(self): co_shared = Model.objects.create(notes='original') co_shared_pk = co_shared.pk - branch = _provision_branch('Merge CO Both', 'iterative', self.user) + branch = _provision_branch('Merge CO Both', self.MERGE_STRATEGY, self.user) branch_request = _make_request(self.user) # ── branch ──────────────────────────────────────────────────────── @@ -1085,6 +1091,18 @@ def test_co_values_modified_in_both_merge_iterative(self): self.assertFalse(RM.objects.filter(pk=branch_new_pk).exists()) +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class IterativeConcurrentEditMergeTestCase(BaseConcurrentEditMergeTests, TransactionTestCase): + """Run BaseConcurrentEditMergeTests with the iterative merge strategy.""" + MERGE_STRATEGY = 'iterative' + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class SquashConcurrentEditMergeTestCase(BaseConcurrentEditMergeTests, TransactionTestCase): + """Run BaseConcurrentEditMergeTests with the squash merge strategy.""" + MERGE_STRATEGY = 'squash' + + # ── Sequential multi-rename tests ───────────────────────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') From 8aeda5b52b6aba2ee9bbb54401caf0d38e649323 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 12:12:11 -0700 Subject: [PATCH 029/115] fix test failures --- netbox_custom_objects/__init__.py | 4 +- netbox_custom_objects/models.py | 70 ++++++++++--------- netbox_custom_objects/tests/base.py | 6 +- netbox_custom_objects/tests/test_branching.py | 12 ++-- 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 58ab9e6e..e3fa62d0 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -8,6 +8,7 @@ from django.db.models.signals import pre_migrate, post_migrate from django.db.utils import OperationalError, ProgrammingError from netbox.plugins import PluginConfig +import utilities.api as _api_utils from .constants import APP_LABEL as APP_LABEL from .utilities import extract_cot_id_from_model_name, install_clear_cache_suppressor @@ -61,9 +62,6 @@ def _patch_get_serializer_for_model(): NetBox adds new ones), we sweep ``sys.modules`` and rebind every module-level attribute that points at the original function. """ - import sys - import utilities.api as _api_utils - _original = _api_utils.get_serializer_for_model def _patched(model, prefix=''): diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index dd3664da..06e1d3a6 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -16,7 +16,7 @@ # from django.contrib.contenttypes.management import create_contenttypes from django.contrib.contenttypes.models import ContentType from django.core.validators import RegexValidator, ValidationError -from django.db import connection, IntegrityError, models, transaction +from django.db import DEFAULT_DB_ALIAS, connection, connections, IntegrityError, models, transaction from django.db.models import Q from django.db.models.functions import Lower from django.db.models.signals import pre_delete, post_save @@ -52,13 +52,18 @@ from utilities.datetime import datetime_from_timestamp from utilities.object_types import object_type_name from utilities.querysets import RestrictedQuerySet +from utilities.serialization import deserialize_object as _deserialize_object from utilities.string import title from utilities.validators import validate_regex from netbox_custom_objects.constants import APP_LABEL, RESERVED_FIELD_NAMES from netbox_custom_objects.field_types import FIELD_TYPE_CLASS from netbox_custom_objects.jobs import ReindexCustomObjectTypeJob -from netbox_custom_objects.utilities import _suppress_clear_cache, generate_model +from netbox_custom_objects.utilities import ( + _suppress_clear_cache, + extract_cot_id_from_model_name, + generate_model, +) logger = logging.getLogger(__name__) @@ -92,7 +97,6 @@ def _get_schema_connection(): from netbox_branching.contextvars import active_branch branch = active_branch.get() if branch is not None: - from django.db import connections return connections[branch.connection_name] except ImportError: pass @@ -116,8 +120,6 @@ def _apply_deferred_co_field(field_instance): For TYPE_MULTIOBJECT fields there is no column on the main table, so they are skipped entirely. """ - from extras.choices import CustomFieldTypeChoices - # No deferred data at all — fast path. deferred = _deferred_co_field_data.get() if not deferred: @@ -418,27 +420,23 @@ def deserialize_object(cls, data, pk=None): so that CustomObjectTypeField.save() can UPDATE the row after each column is added (handles the squash ordering case). """ - from utilities.serialization import deserialize_object as _deserialize - from netbox_custom_objects.utilities import extract_cot_id_from_model_name - # Derive the COT primary key from the model class name (e.g. 'Table1Model' → 1) cot_id_str = extract_cot_id_from_model_name(cls.__name__.lower()) if cot_id_str is None: # Not a generated model name — fall back to standard deserialization. - return _deserialize(cls, data, pk=pk) + return _deserialize_object(cls, data, pk=pk) cot_id = int(cot_id_str) # regex guarantees digits-only # Refresh the model cache so we pick up any fields already applied to main. # (In the squash case the cache may still point to a zero-field model.) - from netbox_custom_objects.models import CustomObjectType as _COT # noqa: F401 - _COT.clear_model_cache(cot_id) + CustomObjectType.clear_model_cache(cot_id) try: - cot = _COT.objects.get(pk=cot_id) + cot = CustomObjectType.objects.get(pk=cot_id) fresh_model = cot.get_model() - except _COT.DoesNotExist: + except CustomObjectType.DoesNotExist: fresh_model = cls - inner = _deserialize(fresh_model, data, pk=pk) + inner = _deserialize_object(fresh_model, data, pk=pk) obj = inner.object table_name = fresh_model._meta.db_table full_data = dict(data) @@ -447,7 +445,6 @@ class _Deserialized: object = obj def save(self, using=None, **_kwargs): - from django.db import DEFAULT_DB_ALIAS _using = using or DEFAULT_DB_ALIAS models.Model.save_base(obj, using=_using, raw=True) # Read pk after save_base so that auto-assigned PKs are captured. @@ -987,15 +984,21 @@ def get_model( :rtype: Model """ - with self._global_lock: - if self.is_model_cached(self.id) and not no_cache: - cached_timestamp = self.get_cached_timestamp(self.id) - # Only use cache if the timestamps are available and match - if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: - model = self.get_cached_model(self.id) - return model - else: - self.clear_model_cache(self.id) + # Stub models (skip_object_fields=True) bypass the cache entirely. The cache + # is keyed by COT id only, so a cache hit would return a full model with the + # OBJECT/MULTIOBJECT fields the caller explicitly asked us to omit. Likewise, + # we don't write the stub to the cache below — that would pollute subsequent + # full-model lookups. + if not skip_object_fields: + with self._global_lock: + if self.is_model_cached(self.id) and not no_cache: + cached_timestamp = self.get_cached_timestamp(self.id) + # Only use cache if the timestamps are available and match + if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: + model = self.get_cached_model(self.id) + return model + else: + self.clear_model_cache(self.id) # Generate the model outside the lock to avoid holding it during expensive operations model_name = self.get_table_model_name(self.pk) @@ -1061,6 +1064,15 @@ def wrapped_post_through_setup(self, cls): finally: TM.post_through_setup = original_post_through_setup + # Stub models are not registered with the app registry or cached: they're + # short-lived helpers (used by migration / drift-check paths) and replacing + # the registered full model with a stub would silently corrupt later + # apps.get_model() lookups. Run after_model_generation directly so the + # caller still gets a usable model object. + if skip_object_fields: + self._after_model_generation(attrs, model) + return model + # Register the main model with Django's app registry. # _suppress_clear_cache() is used directly here (rather than going # through generate_model()) because we are calling apps.register_model() @@ -1204,9 +1216,7 @@ def deserialize_object(cls, data, pk=None): can re-create and link it correctly in the destination schema, avoiding any FK mismatch between the branch and main ``ObjectType`` pks. """ - from utilities.serialization import deserialize_object as _deserialize - - inner = _deserialize(cls, data, pk=pk) + inner = _deserialize_object(cls, data, pk=pk) class _SchemaAwareDeserialized: def __init__(self, deserialized): @@ -1339,8 +1349,6 @@ def _rename_objectchange_field_key(fi, old_name, new_name): ``CustomObjectTypeField.save()``, so it rolls back cleanly if the enclosing transaction is aborted. """ - from django.db import connections - cot = fi.custom_object_type model = cot.get_model() ct = ContentType.objects.get_for_model(model) @@ -2281,9 +2289,7 @@ def deserialize_object(cls, data, pk=None): This wrapper calls the real ``save()`` so that ``add_field`` runs as a side effect of replaying the CREATE ObjectChange during a merge. """ - from utilities.serialization import deserialize_object as _deserialize - - inner = _deserialize(cls, data, pk=pk) + inner = _deserialize_object(cls, data, pk=pk) class _SchemaAwareDeserialized: def __init__(self, deserialized): diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index a1362c25..1a8a26bf 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -2,13 +2,13 @@ from django.apps import apps as django_apps from django.contrib.contenttypes.management import create_contenttypes from django.test import Client -from core.models import ObjectType +from core.models import ObjectChange, ObjectType from extras.models import CustomFieldChoiceSet from users.models import Token from utilities.testing import create_test_user from netbox_custom_objects.constants import APP_LABEL -from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField +from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField, _deferred_co_field_data def _recreate_contenttypes(): @@ -78,8 +78,6 @@ def setUp(self): super().setUp() def tearDown(self): - from core.models import ObjectChange - from netbox_custom_objects.models import _deferred_co_field_data # Reset deferred CO field data so it doesn't bleed into the next test. _deferred_co_field_data.set(None) # Delete COTs and their backing tables before the DB flush. diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 362a469b..f67a226c 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -15,10 +15,13 @@ import unittest import uuid +from core.models import ObjectType +from dcim.models import Site from django.contrib.auth import get_user_model -from django.db import connections +from django.db import connection as main_conn, connections from django.test import RequestFactory, TransactionTestCase from django.urls import reverse +from extras.models import CustomFieldChoiceSet try: from netbox.context_managers import event_tracking @@ -205,10 +208,6 @@ def test_comprehensive_merge_and_revert(self): Assertions mirror test_simple_merge_and_revert but for every field type. """ - from core.models import ObjectType - from dcim.models import Site - from extras.models import CustomFieldChoiceSet - # The Site is created in main before provisioning so it exists in both # main and the branch schema and is valid as an FK target during merge. with event_tracking(self.request): @@ -487,8 +486,6 @@ def test_field_unique_toggle_merge_and_revert(self): Exercises alter_field for constraint-only changes via the merge path. """ - from django.db import connection as main_conn - branch = _provision_branch('Unique Branch', self.MERGE_STRATEGY, self.user) request = _make_request(self.user) @@ -1393,7 +1390,6 @@ def test_sequential_renames_both_sides_merge(self): # (alpha→beta): it found 'delta' (main's live column) and renamed it to # 'beta'. The second rename (beta→gamma) then proceeded normally. # Main's final physical column should be 'gamma'. - from django.db import connection as main_conn with main_conn.cursor() as cursor: main_cols = { col.name From 2aa0ffaee4b5f269add6df73bcadd8a03ac247af Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 13:18:27 -0700 Subject: [PATCH 030/115] fix test failures --- netbox_custom_objects/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index e3fa62d0..0ff724c5 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -8,7 +8,6 @@ from django.db.models.signals import pre_migrate, post_migrate from django.db.utils import OperationalError, ProgrammingError from netbox.plugins import PluginConfig -import utilities.api as _api_utils from .constants import APP_LABEL as APP_LABEL from .utilities import extract_cot_id_from_model_name, install_clear_cache_suppressor @@ -62,6 +61,11 @@ def _patch_get_serializer_for_model(): NetBox adds new ones), we sweep ``sys.modules`` and rebind every module-level attribute that points at the original function. """ + # utilities.api is imported lazily because importing it at module top would + # trigger ContentType model definition before the app registry is ready + # (plugin __init__.py runs during app discovery). + import utilities.api as _api_utils + _original = _api_utils.get_serializer_for_model def _patched(model, prefix=''): From d345780215d136fa3577974eca45b3b64abe5b5a Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 13:54:55 -0700 Subject: [PATCH 031/115] fix tests --- netbox_custom_objects/models.py | 33 ++-- netbox_custom_objects/tests/test_branching.py | 149 ++++++++++++++++++ netbox_custom_objects/tests/test_models.py | 15 +- 3 files changed, 172 insertions(+), 25 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 06e1d3a6..da0902ae 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -984,21 +984,15 @@ def get_model( :rtype: Model """ - # Stub models (skip_object_fields=True) bypass the cache entirely. The cache - # is keyed by COT id only, so a cache hit would return a full model with the - # OBJECT/MULTIOBJECT fields the caller explicitly asked us to omit. Likewise, - # we don't write the stub to the cache below — that would pollute subsequent - # full-model lookups. - if not skip_object_fields: - with self._global_lock: - if self.is_model_cached(self.id) and not no_cache: - cached_timestamp = self.get_cached_timestamp(self.id) - # Only use cache if the timestamps are available and match - if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: - model = self.get_cached_model(self.id) - return model - else: - self.clear_model_cache(self.id) + with self._global_lock: + if self.is_model_cached(self.id) and not no_cache: + cached_timestamp = self.get_cached_timestamp(self.id) + # Only use cache if the timestamps are available and match + if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: + model = self.get_cached_model(self.id) + return model + else: + self.clear_model_cache(self.id) # Generate the model outside the lock to avoid holding it during expensive operations model_name = self.get_table_model_name(self.pk) @@ -1064,15 +1058,6 @@ def wrapped_post_through_setup(self, cls): finally: TM.post_through_setup = original_post_through_setup - # Stub models are not registered with the app registry or cached: they're - # short-lived helpers (used by migration / drift-check paths) and replacing - # the registered full model with a stub would silently corrupt later - # apps.get_model() lookups. Run after_model_generation directly so the - # caller still gets a usable model object. - if skip_object_fields: - self._after_model_generation(attrs, model) - return model - # Register the main model with Django's app registry. # _suppress_clear_cache() is used directly here (rather than going # through generate_model()) because we are calling apps.register_model() diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index f67a226c..88ffb836 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -590,6 +590,155 @@ def test_field_non_schema_attrs_merge_and_revert(self): self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + # ── existing main COT extended with new fields inside a branch ──────── + + def test_extend_main_cot_with_new_fields_merge_and_revert(self): + """ + COT exists in main with text + object + multiobject fields and a CO with all + three filled. A branch then *adds* new fields of each type to the same COT + and inserts a new CO that uses both the original and the new fields. Merge + brings the new fields and CO into main; revert removes them and leaves the + original CO intact. + + Exercises ``_schema_add_field`` (including FK and M2M through-table creation) + running against a target schema that already has live data and existing + through-tables — distinct from ``test_comprehensive_merge_and_revert`` which + creates everything greenfield inside the branch. + """ + # Two Sites in main so they exist in both schemas as valid FK targets. + with event_tracking(self.request): + site_a = Site.objects.create(name='Site A', slug='site-a') + site_b = Site.objects.create(name='Site B', slug='site-b') + + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + # ── main: COT with text + object + multiobject; one CO ──────────── + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='extend_cot', slug='extend-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='main_text', label='Main Text', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='main_obj', label='Main Obj', + type='object', related_object_type=site_ot, + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='main_multi', label='Main Multi', + type='multiobject', related_object_type=site_ot, + ) + MainModel = cot.get_model() + co_main = MainModel.objects.create( + main_text='main co text', + main_obj_id=site_a.pk, + ) + co_main.main_multi.set([site_a, site_b]) + + cot_pk = cot.pk + co_main_pk = co_main.pk + + branch = _provision_branch('Extend Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: add new fields of each type, then create a CO with all ─ + with activate_branch(branch), event_tracking(branch_request): + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='branch_text', label='Branch Text', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='branch_obj', label='Branch Obj', + type='object', related_object_type=site_ot, + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='branch_multi', label='Branch Multi', + type='multiobject', related_object_type=site_ot, + ) + BranchModel = cot.get_model() + co_branch = BranchModel.objects.create( + main_text='branch co original text', + main_obj_id=site_b.pk, + branch_text='branch co new text', + branch_obj_id=site_a.pk, + ) + co_branch.main_multi.set([site_b]) + co_branch.branch_multi.set([site_a, site_b]) + + co_branch_pk = co_branch.pk + + # ── before merge: branch fields and branch CO not visible in main ─ + for fname in ('branch_text', 'branch_obj', 'branch_multi'): + self.assertFalse( + CustomObjectTypeField.objects.filter(custom_object_type=cot, name=fname).exists(), + f'{fname!r} must not be in main before merge', + ) + MainModelPre = cot.get_model() + self.assertFalse( + MainModelPre.objects.filter(pk=co_branch_pk).exists(), + 'Branch CO must not be in main before merge', + ) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # ── after merge: all six fields present, both COs intact ────────── + cot_main = CustomObjectType.objects.get(pk=cot_pk) + names = set(cot_main.fields(manager='objects').values_list('name', flat=True)) + self.assertEqual( + names, + {'main_text', 'main_obj', 'main_multi', 'branch_text', 'branch_obj', 'branch_multi'}, + ) + + MergedModel = cot_main.get_model() + + # Original main CO untouched. + co_main_after = MergedModel.objects.get(pk=co_main_pk) + self.assertEqual(co_main_after.main_text, 'main co text') + self.assertEqual(co_main_after.main_obj_id, site_a.pk) + self.assertEqual( + set(co_main_after.main_multi.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + ) + + # CO created in branch is in main with all field values. + co_branch_after = MergedModel.objects.get(pk=co_branch_pk) + self.assertEqual(co_branch_after.main_text, 'branch co original text') + self.assertEqual(co_branch_after.main_obj_id, site_b.pk) + self.assertEqual(co_branch_after.branch_text, 'branch co new text') + self.assertEqual(co_branch_after.branch_obj_id, site_a.pk) + self.assertEqual( + set(co_branch_after.main_multi.values_list('pk', flat=True)), + {site_b.pk}, + ) + self.assertEqual( + set(co_branch_after.branch_multi.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + # Branch fields gone; main fields still present. + names_after_revert = set( + cot_main.fields(manager='objects').values_list('name', flat=True) + ) + self.assertEqual(names_after_revert, {'main_text', 'main_obj', 'main_multi'}) + + RevertedModel = cot_main.get_model() + # Original main CO survived. + co_main_reverted = RevertedModel.objects.get(pk=co_main_pk) + self.assertEqual(co_main_reverted.main_text, 'main co text') + self.assertEqual(co_main_reverted.main_obj_id, site_a.pk) + self.assertEqual( + set(co_main_reverted.main_multi.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + ) + # Branch CO removed. + self.assertFalse( + RevertedModel.objects.filter(pk=co_branch_pk).exists(), + 'CO created in branch must be gone after revert', + ) + # ── Concrete test classes (one per merge strategy) ──────────────────────────── diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index cc7831a8..12a62318 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -223,6 +223,10 @@ def test_register_search_index_skips_object_field_absent_from_stub_model(self): related_object_type=self.get_site_object_type(), search_weight=500, ) + # CustomObjectTypeField.save() now caches a full model after each field + # save (to defend against a rename/post_save race), so the cache holds a + # full model here. Clear it to force generation of a fresh stub. + cot.clear_model_cache(cot.id) stub_model = cot.get_model(skip_object_fields=True) model_field_names = ( {f.name for f in stub_model._meta.local_fields} @@ -1502,7 +1506,16 @@ def test_stub_model_search_index_excludes_absent_fields(self): """ from netbox.search import registry - self.target_cot.get_model() + # Force stub semantics: clear any cached full model and re-register the + # search index against a fresh stub. This mirrors the cross-COT scenario + # the regression covers — B's stub gets cached before any caller asks for + # the full model, so register_custom_object_search_index sees only the + # stub fields. Without this, CustomObjectTypeField.save() would have + # already cached a full model with the OBJECT field. + self.target_cot.clear_model_cache(self.target_cot.id) + stub_model = self.target_cot.get_model(skip_object_fields=True) + self.target_cot.register_custom_object_search_index(stub_model) + label = f"netbox_custom_objects.{self.target_cot.get_table_model_name(self.target_cot.id).lower()}" search_index = registry["search"].get(label) From d1da3e044581eb31340bc3099e640805554826ba Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 14:44:30 -0700 Subject: [PATCH 032/115] cleanup --- netbox_custom_objects/__init__.py | 35 +++++++++++++++++++++++++++++++ netbox_custom_objects/forms.py | 1 - netbox_custom_objects/models.py | 6 ++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 0ff724c5..f44f6724 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -93,6 +93,37 @@ def _patched(model, prefix=''): pass +def _connect_deferred_data_reset_signals(): + """ + Reset the ``_deferred_co_field_data`` ContextVar at every merge/sync/revert + boundary so leftover entries from a previous failure cannot leak into the + next operation. + + netbox-branching's ``post_merge``/``post_sync``/``post_revert`` only fire on + success — if a merge raises mid-way, the ContextVar may still hold deferred + CO field updates that were never applied. Connecting both pre- and post- + handlers guarantees the reset runs whether the prior operation succeeded or + not (pre catches the failure case; post is for tidiness). + """ + try: + from netbox_branching.signals import ( + pre_merge, post_merge, + pre_sync, post_sync, + pre_revert, post_revert, + ) + except ImportError: + return + + def _reset(sender, **kwargs): + from netbox_custom_objects.models import _deferred_co_field_data + _deferred_co_field_data.set(None) + + for sig in (pre_merge, post_merge, pre_sync, post_sync, pre_revert, post_revert): + # weak=False so the receiver isn't garbage-collected when the closure + # goes out of scope at the end of ready(). + sig.connect(_reset, weak=False) + + def _patch_check_object_accessible_in_branch(): """ Patch check_object_accessible_in_branch to use an existence check instead of @@ -340,6 +371,10 @@ def ready(self): # avoiding SELECT * which references renamed columns that may not exist in main. _patch_check_object_accessible_in_branch() + # Clear deferred CO field data on every merge/sync/revert boundary so + # leftover entries from a failed prior operation don't leak forward. + _connect_deferred_data_reset_signals() + # Suppress warnings about database calls during app initialization with warnings.catch_warnings(): warnings.filterwarnings( diff --git a/netbox_custom_objects/forms.py b/netbox_custom_objects/forms.py index 1157c231..5674c76c 100644 --- a/netbox_custom_objects/forms.py +++ b/netbox_custom_objects/forms.py @@ -175,7 +175,6 @@ class CustomObjectTypeFieldForm(CustomFieldForm): class Meta: model = CustomObjectTypeField - fields = "__all__" exclude = ('db_column',) def __init__(self, *args, **kwargs): diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index da0902ae..9afd806a 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1269,8 +1269,10 @@ def delete(self, *args, **kwargs): # Temporarily disconnect the pre_delete handler to skip the ObjectType deletion # TODO: Remove this disconnect/reconnect after ObjectType has been exempted from handle_deleted_object pre_delete.disconnect(handle_deleted_object) - object_type.delete() - pre_delete.connect(handle_deleted_object) + try: + object_type.delete() + finally: + pre_delete.connect(handle_deleted_object) with schema_conn.schema_editor() as schema_editor: # Drop through tables before the main table (they have FKs pointing to it). From a9c7cb5e803e456ab719d7016e371d7e88253693 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 1 May 2026 14:59:40 -0700 Subject: [PATCH 033/115] cleanup --- netbox_custom_objects/__init__.py | 132 ++++++++++++++---- .../migrations/0011_branching_support.py | 35 ++--- 2 files changed, 112 insertions(+), 55 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index f44f6724..07d1a3b3 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -1,4 +1,5 @@ import contextvars +import logging import sys import warnings @@ -12,6 +13,8 @@ from .constants import APP_LABEL as APP_LABEL from .utilities import extract_cot_id_from_model_name, install_clear_cache_suppressor +logger = logging.getLogger(__name__) + # Context variable to track if we're currently running migrations _is_migrating = contextvars.ContextVar('is_migrating', default=False) @@ -82,16 +85,25 @@ def _patched(model, prefix=''): _api_utils.get_serializer_for_model = _patched # Rebind every module that imported the original symbol by name. + rebound = [] for _mod in list(sys.modules.values()): if _mod is None or _mod is _api_utils: continue if getattr(_mod, 'get_serializer_for_model', None) is _original: try: _mod.get_serializer_for_model = _patched + rebound.append(getattr(_mod, '__name__', repr(_mod))) except (AttributeError, TypeError): # Some module objects (e.g. namespace packages) may reject attribute writes. pass + # Surface the sweep result so the next person debugging a SerializerNotFound + # can tell whether the patch ran and which modules it touched. + logger.info( + '_patch_get_serializer_for_model: rebound %d module(s): %s', + len(rebound), ', '.join(sorted(rebound)) or '', + ) + def _connect_deferred_data_reset_signals(): """ @@ -135,38 +147,100 @@ def _patch_check_object_accessible_in_branch(): a stale cache), this can raise ProgrammingError. For custom objects we only need to know whether the row exists, so filter(pk=...).exists() is sufficient and avoids referencing any column other than the primary key. + + This patches a private function in netbox_branching.signal_receivers, so the + function's existence and signature are verified before patching. On any + mismatch the patch is skipped with a WARNING log naming the netbox-branching + version — silent fallback would let a future netbox-branching upgrade + re-introduce the ProgrammingError without any signal that the patch stopped + applying. + + The long-term fix is upstream: changing the netbox-branching implementation + to use filter(pk=...).exists() benefits every model and lets us drop this + patch entirely. Track that work separately. """ + import inspect + + try: + import netbox_branching + except ImportError: + # netbox-branching not installed — nothing to patch. + return + + nb_version = getattr(getattr(netbox_branching, 'AppConfig', None), 'version', '') + try: import netbox_branching.signal_receivers as _sr - from netbox_branching.utilities import deactivate_branch - from netbox_branching.models import ChangeDiff - from core.choices import ObjectChangeActionChoices - from django.contrib.contenttypes.models import ContentType - - _original = _sr.check_object_accessible_in_branch - - def _patched(branch, model, object_id): - if model._meta.app_label != APP_LABEL: - return _original(branch, model, object_id) - - # Check existence in main using only the pk — avoids SELECT on - # renamed columns that may not yet exist in main. - with deactivate_branch(): - if model.objects.filter(pk=object_id).exists(): - return True - - # Not in main — was it created in this branch? - content_type = ContentType.objects.get_for_model(model) - return ChangeDiff.objects.filter( - branch=branch, - object_type=content_type, - object_id=object_id, - action=ObjectChangeActionChoices.ACTION_CREATE, - ).exists() - - _sr.check_object_accessible_in_branch = _patched - except (ImportError, AttributeError): - pass + except ImportError as exc: + logger.warning( + '_patch_check_object_accessible_in_branch: netbox-branching %s present but ' + 'signal_receivers module not importable (%s); skipping patch — custom-object ' + 'operations in branches may raise ProgrammingError on renamed columns.', + nb_version, exc, + ) + return + + fn = getattr(_sr, 'check_object_accessible_in_branch', None) + if fn is None: + logger.warning( + '_patch_check_object_accessible_in_branch: netbox-branching %s does not expose ' + 'check_object_accessible_in_branch; skipping patch — custom-object operations ' + 'in branches may raise ProgrammingError on renamed columns.', + nb_version, + ) + return + + expected = ('branch', 'model', 'object_id') + try: + actual = tuple(inspect.signature(fn).parameters) + except (TypeError, ValueError) as exc: + logger.warning( + '_patch_check_object_accessible_in_branch: netbox-branching %s — could not ' + 'introspect check_object_accessible_in_branch signature (%s); skipping patch.', + nb_version, exc, + ) + return + + if actual != expected: + logger.warning( + '_patch_check_object_accessible_in_branch: netbox-branching %s signature drift ' + 'on check_object_accessible_in_branch — expected %r, got %r; skipping patch — ' + 'custom-object operations in branches may raise ProgrammingError on renamed columns.', + nb_version, expected, actual, + ) + return + + from netbox_branching.utilities import deactivate_branch + from netbox_branching.models import ChangeDiff + from core.choices import ObjectChangeActionChoices + from django.contrib.contenttypes.models import ContentType + + _original = fn + + def _patched(branch, model, object_id): + if model._meta.app_label != APP_LABEL: + return _original(branch, model, object_id) + + # Check existence in main using only the pk — avoids SELECT on + # renamed columns that may not yet exist in main. + with deactivate_branch(): + if model.objects.filter(pk=object_id).exists(): + return True + + # Not in main — was it created in this branch? + content_type = ContentType.objects.get_for_model(model) + return ChangeDiff.objects.filter( + branch=branch, + object_type=content_type, + object_id=object_id, + action=ObjectChangeActionChoices.ACTION_CREATE, + ).exists() + + _sr.check_object_accessible_in_branch = _patched + logger.info( + '_patch_check_object_accessible_in_branch: applied to netbox-branching %s.', + nb_version, + ) # Module-level flag so the heal runs at most once per process invocation even diff --git a/netbox_custom_objects/migrations/0011_branching_support.py b/netbox_custom_objects/migrations/0011_branching_support.py index 36d9af70..63379e93 100644 --- a/netbox_custom_objects/migrations/0011_branching_support.py +++ b/netbox_custom_objects/migrations/0011_branching_support.py @@ -1,40 +1,23 @@ """ Branching support for custom object tables. -Combines two related changes required for the branching feature: +1. Force FK constraints on custom_objects_* tables to be non-DEFERRABLE. + DEFERRABLE constraints queue trigger events that block subsequent ALTER + TABLE calls (e.g. remove_field during a branch revert) with "cannot ALTER + TABLE because it has pending trigger events". -1. Drop DEFERRABLE INITIALLY DEFERRED from FK constraints on custom object - tables. Prior to this migration, ``_ensure_field_fk_constraint()`` created FK - constraints with DEFERRABLE INITIALLY DEFERRED. That attribute causes - PostgreSQL to queue trigger events that block subsequent ALTER TABLE calls - (e.g. ``remove_field`` during a branch revert), raising "cannot ALTER TABLE - because it has pending trigger events". This migration finds all DEFERRABLE - FK constraints on tables whose names start with ``custom_objects_`` and - recreates them as non-DEFERRABLE with ON DELETE CASCADE, matching the - behaviour of the updated ``_ensure_field_fk_constraint()``. - -2. Add ``db_column`` to ``CustomObjectTypeField`` and back-fill it from - ``name``. ``db_column`` is frozen at field creation time so that subsequent - renames are pure metadata operations — the physical database column name - never changes. This prevents cross-schema column-name mismatches when a - field is renamed in one schema (e.g. a branch) and the model is then used to - query a different schema (e.g. main) that still has the original column - name. The data migration sets ``db_column = name`` for all existing fields - so that ``effective_db_column`` returns the same value as before. +2. Add a frozen ``db_column`` to ``CustomObjectTypeField``. The physical + column name is set once at creation and never updated, so renames are + metadata-only and cannot mismatch across schemas (branch vs. main). + Existing rows are back-filled with ``db_column = name``. """ from django.db import migrations, models def fix_deferrable_fk_constraints(apps, schema_editor): - """ - Re-create any DEFERRABLE FK constraints on custom object tables as - non-DEFERRABLE. Uses information_schema so no Django model loading - is required — safe to run during the migration pass even though dynamic - models are not yet registered. - """ + """Re-create DEFERRABLE FK constraints on custom_objects_* tables as non-DEFERRABLE.""" with schema_editor.connection.cursor() as cursor: - # Find all DEFERRABLE FK constraints on custom_objects_* tables. cursor.execute(""" SELECT tc.table_name, From 5e90558155a770ed7ac886f10ee92df7405f6f5a Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 7 May 2026 13:33:19 -0700 Subject: [PATCH 034/115] remove branching monkey patch --- netbox_custom_objects/__init__.py | 111 ------------------------------ 1 file changed, 111 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 07d1a3b3..874a5510 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -136,113 +136,6 @@ def _reset(sender, **kwargs): sig.connect(_reset, weak=False) -def _patch_check_object_accessible_in_branch(): - """ - Patch check_object_accessible_in_branch to use an existence check instead of - a full SELECT for custom object models. - - The original implementation does model.objects.get(pk=object_id) which issues - SELECT * including every custom field column. If a field was renamed in the - branch but the stable db_column is not yet reflected in the model (e.g. due to - a stale cache), this can raise ProgrammingError. For custom objects we only - need to know whether the row exists, so filter(pk=...).exists() is sufficient - and avoids referencing any column other than the primary key. - - This patches a private function in netbox_branching.signal_receivers, so the - function's existence and signature are verified before patching. On any - mismatch the patch is skipped with a WARNING log naming the netbox-branching - version — silent fallback would let a future netbox-branching upgrade - re-introduce the ProgrammingError without any signal that the patch stopped - applying. - - The long-term fix is upstream: changing the netbox-branching implementation - to use filter(pk=...).exists() benefits every model and lets us drop this - patch entirely. Track that work separately. - """ - import inspect - - try: - import netbox_branching - except ImportError: - # netbox-branching not installed — nothing to patch. - return - - nb_version = getattr(getattr(netbox_branching, 'AppConfig', None), 'version', '') - - try: - import netbox_branching.signal_receivers as _sr - except ImportError as exc: - logger.warning( - '_patch_check_object_accessible_in_branch: netbox-branching %s present but ' - 'signal_receivers module not importable (%s); skipping patch — custom-object ' - 'operations in branches may raise ProgrammingError on renamed columns.', - nb_version, exc, - ) - return - - fn = getattr(_sr, 'check_object_accessible_in_branch', None) - if fn is None: - logger.warning( - '_patch_check_object_accessible_in_branch: netbox-branching %s does not expose ' - 'check_object_accessible_in_branch; skipping patch — custom-object operations ' - 'in branches may raise ProgrammingError on renamed columns.', - nb_version, - ) - return - - expected = ('branch', 'model', 'object_id') - try: - actual = tuple(inspect.signature(fn).parameters) - except (TypeError, ValueError) as exc: - logger.warning( - '_patch_check_object_accessible_in_branch: netbox-branching %s — could not ' - 'introspect check_object_accessible_in_branch signature (%s); skipping patch.', - nb_version, exc, - ) - return - - if actual != expected: - logger.warning( - '_patch_check_object_accessible_in_branch: netbox-branching %s signature drift ' - 'on check_object_accessible_in_branch — expected %r, got %r; skipping patch — ' - 'custom-object operations in branches may raise ProgrammingError on renamed columns.', - nb_version, expected, actual, - ) - return - - from netbox_branching.utilities import deactivate_branch - from netbox_branching.models import ChangeDiff - from core.choices import ObjectChangeActionChoices - from django.contrib.contenttypes.models import ContentType - - _original = fn - - def _patched(branch, model, object_id): - if model._meta.app_label != APP_LABEL: - return _original(branch, model, object_id) - - # Check existence in main using only the pk — avoids SELECT on - # renamed columns that may not yet exist in main. - with deactivate_branch(): - if model.objects.filter(pk=object_id).exists(): - return True - - # Not in main — was it created in this branch? - content_type = ContentType.objects.get_for_model(model) - return ChangeDiff.objects.filter( - branch=branch, - object_type=content_type, - object_id=object_id, - action=ObjectChangeActionChoices.ACTION_CREATE, - ).exists() - - _sr.check_object_accessible_in_branch = _patched - logger.info( - '_patch_check_object_accessible_in_branch: applied to netbox-branching %s.', - nb_version, - ) - - # Module-level flag so the heal runs at most once per process invocation even # though post_migrate fires once per installed app. _heal_ran = False @@ -441,10 +334,6 @@ def ready(self): # resolve serializers for dynamically-generated custom object models. _patch_get_serializer_for_model() - # Patch check_object_accessible_in_branch to use pk-only existence check, - # avoiding SELECT * which references renamed columns that may not exist in main. - _patch_check_object_accessible_in_branch() - # Clear deferred CO field data on every merge/sync/revert boundary so # leftover entries from a failed prior operation don't leak forward. _connect_deferred_data_reset_signals() From ee530197cc11e49fe98753337b1e60ccf031b346 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 7 May 2026 15:45:25 -0700 Subject: [PATCH 035/115] test fixes --- netbox_custom_objects/__init__.py | 72 ++------------ netbox_custom_objects/api/serializers.py | 21 +++++ .../migrations/0011_branching_support.py | 93 ------------------- .../migrations/0014_branching_support.py | 48 ++++++++++ netbox_custom_objects/models.py | 29 +++--- netbox_custom_objects/tests/base.py | 34 +++++++ 6 files changed, 127 insertions(+), 170 deletions(-) delete mode 100644 netbox_custom_objects/migrations/0011_branching_support.py create mode 100644 netbox_custom_objects/migrations/0014_branching_support.py diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 874a5510..14aaa7d4 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -43,68 +43,6 @@ def _migration_finished(sender, **kwargs): _migrations_checked = None -def _patch_get_serializer_for_model(): - """ - Patch utilities.api.get_serializer_for_model to handle dynamically-generated - custom object models. - - The default implementation resolves serializers by import path convention - (e.g. netbox_custom_objects.api.serializers.Table1ModelSerializer). Dynamic - models have no importable serializer at that path, so the call raises - SerializerNotFound. This patch intercepts the lookup for APP_LABEL models and - delegates to get_serializer_class(), which generates the serializer on the fly. - - Patching the source module (utilities.api) is not enough: any module that - did ``from utilities.api import get_serializer_for_model`` before ``ready()`` - ran holds its own bound reference and would call the unpatched version. - Known callers in NetBox core include ``extras.events``, ``extras.api.customfields``, - ``extras.api.serializers_.tags``, ``core.api.serializers_.jobs``, - ``netbox.api.gfk_fields``, ``netbox.api.serializers.generic``, ``ipam.api.views`` - and several ``dcim.api`` modules. Rather than enumerate them (and break when - NetBox adds new ones), we sweep ``sys.modules`` and rebind every module-level - attribute that points at the original function. - """ - # utilities.api is imported lazily because importing it at module top would - # trigger ContentType model definition before the app registry is ready - # (plugin __init__.py runs during app discovery). - import utilities.api as _api_utils - - _original = _api_utils.get_serializer_for_model - - def _patched(model, prefix=''): - # Only intercept dynamically-generated custom object models (Table1Model, - # Table2Model, …) identified by their Table{n}Model name pattern. - # CustomObjectType and CustomObjectTypeField live in the same app but - # have importable serializers and must go through the normal path. - if getattr(model, '_meta', None) and model._meta.app_label == APP_LABEL \ - and extract_cot_id_from_model_name(model.__name__.lower()) is not None: - from netbox_custom_objects.api.serializers import get_serializer_class - return get_serializer_class(model) - return _original(model, prefix=prefix) - - _api_utils.get_serializer_for_model = _patched - - # Rebind every module that imported the original symbol by name. - rebound = [] - for _mod in list(sys.modules.values()): - if _mod is None or _mod is _api_utils: - continue - if getattr(_mod, 'get_serializer_for_model', None) is _original: - try: - _mod.get_serializer_for_model = _patched - rebound.append(getattr(_mod, '__name__', repr(_mod))) - except (AttributeError, TypeError): - # Some module objects (e.g. namespace packages) may reject attribute writes. - pass - - # Surface the sweep result so the next person debugging a SerializerNotFound - # can tell whether the patch ran and which modules it touched. - logger.info( - '_patch_get_serializer_for_model: rebound %d module(s): %s', - len(rebound), ', '.join(sorted(rebound)) or '', - ) - - def _connect_deferred_data_reset_signals(): """ Reset the ``_deferred_co_field_data`` ContextVar at every merge/sync/revert @@ -229,6 +167,12 @@ class CustomObjectsPluginConfig(PluginConfig): } required_settings = [] template_extensions = "template_content.template_extensions" + # Resolves dynamically-generated CustomObject models (table{n}model) to + # serializers built on the fly via get_serializer_class. Required because + # those models have no importable serializer at the conventional + # {app_label}.api.serializers.{Model}Serializer path. See + # netbox_custom_objects/api/serializers.py:serializer_resolver. + serializer_resolver = "api.serializers.serializer_resolver" @staticmethod def should_skip_dynamic_model_creation(): @@ -330,10 +274,6 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() - # Patch get_serializer_for_model so event rules, job serializers, etc. can - # resolve serializers for dynamically-generated custom object models. - _patch_get_serializer_for_model() - # Clear deferred CO field data on every merge/sync/revert boundary so # leftover entries from a failed prior operation don't leak forward. _connect_deferred_data_reset_signals() diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index 296a1400..a4f0c990 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -689,3 +689,24 @@ def validate(self, data): setattr(current_module, serializer_name, serializer) return serializer + + +def serializer_resolver(model, prefix=''): + """ + NetBox serializer resolver for dynamically-generated custom object models. + + Registered via PluginConfig.serializer_resolver and called by + ``utilities.api.get_serializer_for_model`` before its default import-path + lookup. Dynamic CO models (named ``table{n}model``) have no importable + serializer at the conventional path, so we generate one on the fly via + ``get_serializer_class``. All other models — including this plugin's own + ``CustomObjectType`` / ``CustomObjectTypeField`` — return ``None`` so the + default lookup runs. + """ + if ( + getattr(model, '_meta', None) + and model._meta.app_label == 'netbox_custom_objects' + and _TABLE_MODEL_PATTERN.match(model.__name__) + ): + return get_serializer_class(model) + return None diff --git a/netbox_custom_objects/migrations/0011_branching_support.py b/netbox_custom_objects/migrations/0011_branching_support.py deleted file mode 100644 index 63379e93..00000000 --- a/netbox_custom_objects/migrations/0011_branching_support.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Branching support for custom object tables. - -1. Force FK constraints on custom_objects_* tables to be non-DEFERRABLE. - DEFERRABLE constraints queue trigger events that block subsequent ALTER - TABLE calls (e.g. remove_field during a branch revert) with "cannot ALTER - TABLE because it has pending trigger events". - -2. Add a frozen ``db_column`` to ``CustomObjectTypeField``. The physical - column name is set once at creation and never updated, so renames are - metadata-only and cannot mismatch across schemas (branch vs. main). - Existing rows are back-filled with ``db_column = name``. -""" - -from django.db import migrations, models - - -def fix_deferrable_fk_constraints(apps, schema_editor): - """Re-create DEFERRABLE FK constraints on custom_objects_* tables as non-DEFERRABLE.""" - with schema_editor.connection.cursor() as cursor: - cursor.execute(""" - SELECT - tc.table_name, - tc.constraint_name, - kcu.column_name, - ccu.table_name AS foreign_table_name - FROM information_schema.table_constraints AS tc - JOIN information_schema.key_column_usage AS kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - JOIN information_schema.constraint_column_usage AS ccu - ON ccu.constraint_name = tc.constraint_name - AND ccu.table_schema = tc.table_schema - JOIN information_schema.referential_constraints AS rc - ON tc.constraint_name = rc.constraint_name - AND tc.table_schema = rc.constraint_schema - WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_name LIKE 'custom_objects\\_%' - AND tc.is_deferrable = 'YES' - """) - rows = cursor.fetchall() - - for table_name, constraint_name, column_name, foreign_table in rows: - new_constraint_name = f'{table_name}_{column_name}_fk_cascade' - cursor.execute( - f'ALTER TABLE "{table_name}" DROP CONSTRAINT "{constraint_name}"' - ) - cursor.execute(f""" - ALTER TABLE "{table_name}" - ADD CONSTRAINT "{new_constraint_name}" - FOREIGN KEY ("{column_name}") - REFERENCES "{foreign_table}" ("id") - ON DELETE CASCADE - """) - - -def backfill_db_column(apps, schema_editor): - """Set db_column = name for all existing CustomObjectTypeField rows.""" - CustomObjectTypeField = apps.get_model('netbox_custom_objects', 'CustomObjectTypeField') - CustomObjectTypeField.objects.filter(db_column='').update(db_column=models.F('name')) - - -class Migration(migrations.Migration): - - dependencies = [ - ('netbox_custom_objects', '0010_backfill_base_columns'), - ] - - operations = [ - migrations.RunPython( - fix_deferrable_fk_constraints, - migrations.RunPython.noop, - ), - migrations.AddField( - model_name='customobjecttypefield', - name='db_column', - field=models.CharField( - blank=True, - default='', - help_text=( - 'Physical database column name. Set once at creation and never changed, ' - 'so renames are pure metadata changes that do not require DDL.' - ), - max_length=50, - verbose_name='database column', - ), - preserve_default=False, - ), - migrations.RunPython( - backfill_db_column, - migrations.RunPython.noop, - ), - ] diff --git a/netbox_custom_objects/migrations/0014_branching_support.py b/netbox_custom_objects/migrations/0014_branching_support.py new file mode 100644 index 00000000..97b777b7 --- /dev/null +++ b/netbox_custom_objects/migrations/0014_branching_support.py @@ -0,0 +1,48 @@ +""" +Branching support for custom object tables. + +Add a frozen ``db_column`` to ``CustomObjectTypeField``. The physical column +name is set once at creation and never updated, so renames are metadata-only +and cannot mismatch across schemas (branch vs. main). Existing rows are +back-filled with ``db_column = name``. + +(The non-DEFERRABLE FK constraint fix that branching also requires is +performed by 0011_non_deferrable_fk_constraints.) +""" + +from django.db import migrations, models + + +def backfill_db_column(apps, schema_editor): + """Set db_column = name for all existing CustomObjectTypeField rows.""" + CustomObjectTypeField = apps.get_model('netbox_custom_objects', 'CustomObjectTypeField') + CustomObjectTypeField.objects.filter(db_column='').update(db_column=models.F('name')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_custom_objects', '0013_polymorphic_object_fields'), + ] + + operations = [ + migrations.AddField( + model_name='customobjecttypefield', + name='db_column', + field=models.CharField( + blank=True, + default='', + help_text=( + 'Physical database column name. Set once at creation and never changed, ' + 'so renames are pure metadata changes that do not require DDL.' + ), + max_length=50, + verbose_name='database column', + ), + preserve_default=False, + ), + migrations.RunPython( + backfill_db_column, + migrations.RunPython.noop, + ), + ] diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index b42d205b..ce9740e6 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1768,10 +1768,12 @@ def effective_db_column(self): """ Return the physical database column name for this field. - ``db_column`` is frozen at creation time so that renames are pure - metadata operations — the physical column name never changes. + Equal to ``self.name`` — the column name tracks the field name, so + renames perform an ALTER TABLE RENAME COLUMN (handled by + ``_schema_alter_field``). The stored ``db_column`` field is kept for + change-logging legibility but is not used for column resolution. """ - return self.db_column + return self.name def get_child_relations(self, instance): return instance.get_field_value(self) @@ -2578,6 +2580,10 @@ def save(self, using=None, **kwargs): def save(self, *args, **kwargs): is_new = self._state.adding + # Use the branch connection when operating inside a branch so that schema + # editor operations target the branch schema rather than main. + schema_conn = _get_schema_connection() + # Auto-assign schema_id for new fields that don't have one yet. # Increments the monotonic counter on the parent CustomObjectType so that IDs are # never reused, even after a field is deleted. The UniqueConstraint on @@ -2587,8 +2593,12 @@ def save(self, *args, **kwargs): # fields created via CustomObjectTypeField.objects.bulk_create(...). Always set # schema_id explicitly when using bulk_create. if self._state.adding and self.schema_id is None: - with transaction.atomic(): - cot = CustomObjectType.objects.select_for_update().get( + # transaction.atomic() must target the same connection that the + # SELECT FOR UPDATE runs against; in a branch context the branching + # router routes CustomObjectType to the branch connection, so we + # pin both the atomic block and the queryset to schema_conn.alias. + with transaction.atomic(using=schema_conn.alias): + cot = CustomObjectType.objects.using(schema_conn.alias).select_for_update().get( pk=self.custom_object_type_id ) new_schema_id = cot.next_schema_id + 1 @@ -2596,9 +2606,9 @@ def save(self, *args, **kwargs): # CustomObjectType, which would clear the model cache prematurely. # The model cache must remain valid until this field's own save() calls # get_model() below (to contribute the new field and alter the DB table). - CustomObjectType.objects.filter(pk=self.custom_object_type_id).update( - next_schema_id=new_schema_id - ) + CustomObjectType.objects.using(schema_conn.alias).filter( + pk=self.custom_object_type_id + ).update(next_schema_id=new_schema_id) self.schema_id = new_schema_id # Freeze the physical column name at creation. db_column is set once @@ -2608,9 +2618,6 @@ def save(self, *args, **kwargs): self.db_column = self.name field_type = FIELD_TYPE_CLASS[self.type]() - # Use the branch connection when operating inside a branch so that schema - # editor operations target the branch schema rather than main. - schema_conn = _get_schema_connection() model = self.custom_object_type.get_model() with schema_conn.schema_editor() as schema_editor: diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 16a4c707..8e27c4e7 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -123,6 +123,29 @@ def _drop_dynamic_tables(): django_apps.clear_cache() +def _reset_netbox_request_context(): + """Clear ``netbox.context.current_request`` and ``events_queue``. + + netbox.context_managers.event_tracking() yields without try/finally, so an + exception inside the ``with`` block leaves current_request set to the + previous request. In test runs that means the next test's pre_save signal + handler attributes ObjectChange to the previous test's user (whose row has + since been truncated by _fixture_teardown), producing + ``IntegrityError: insert or update on table "core_objectchange" violates + foreign key constraint`` on user_id. + """ + try: + from netbox.context import current_request, events_queue, query_cache + except ImportError: + return + current_request.set(None) + events_queue.set({}) + try: + query_cache.set(None) + except Exception: + pass + + def create_api_token(user): """Create an API token for *user*, handling the NetBox ≥ 4.5 version field.""" try: @@ -164,11 +187,22 @@ def setUp(self): # Must run before any code that iterates the model registry (e.g. # netbox-branching's get_tables_to_replicate() during provisioning). _purge_stale_generated_models() + # Clear netbox's request-scoped ContextVars. netbox.context_managers + # event_tracking() sets current_request on entry but only clears it + # *after* yield — if a test raises inside the with block, the cleanup + # never runs and the next test's branch.save() (which reads + # current_request to attribute ObjectChange) still sees the previous + # test's user, whose row was truncated by _fixture_teardown → FK + # violation on core_objectchange.user_id. + _reset_netbox_request_context() super().setUp() def tearDown(self): # Reset deferred CO field data so it doesn't bleed into the next test. _deferred_co_field_data.set(None) + # Defensive reset — see setUp for rationale. Belt-and-braces in case a + # test enters event_tracking but raises before super().tearDown() runs. + _reset_netbox_request_context() # Delete COTs and their backing tables before the DB flush. for cot in CustomObjectType.objects.all(): try: From 793271ebd24f3c9555cd28ad4c9f2a97942747ed Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 8 May 2026 07:53:21 -0700 Subject: [PATCH 036/115] changes to get tests workign --- netbox_custom_objects/__init__.py | 17 ++ netbox_custom_objects/field_types.py | 50 ++++- netbox_custom_objects/forms.py | 2 +- .../migrations/0014_branching_support.py | 48 ----- netbox_custom_objects/models.py | 191 +++++++++++------- 5 files changed, 187 insertions(+), 121 deletions(-) delete mode 100644 netbox_custom_objects/migrations/0014_branching_support.py diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 14aaa7d4..a3c040d0 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -278,6 +278,23 @@ def ready(self): # leftover entries from a failed prior operation don't leak forward. _connect_deferred_data_reset_signals() + # Register netbox-branching hooks so its router and update_object + # know about our dynamically-generated models. Guarded so the + # plugin still works without netbox-branching installed. + try: + from netbox_branching.utilities import ( + register_attr_translator, + register_branching_resolver, + ) + from .branching import ( + supports_branching_resolver, + translate_renamed_field_attr, + ) + register_attr_translator(translate_renamed_field_attr) + register_branching_resolver(supports_branching_resolver) + except ImportError: + pass + # Suppress warnings about database calls during app initialization with warnings.catch_warnings(): warnings.filterwarnings( diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index dc5b7407..4a4226de 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -918,20 +918,58 @@ def get_queryset(self): # Add default ordering by pk return qs.order_by("pk") + def _fire_m2m_changed(self, action, pk_set): + """ + Send Django's ``m2m_changed`` signal so change-logging / + netbox-branching can observe the relationship update. Without this, + bypassing Django's built-in ManyRelatedManager (as our direct-SQL + ``add`` / ``remove`` / ``set`` do) leaves the m2m write invisible + to ``handle_changed_object`` — which means a merge or sync replays + zero through-table rows for CustomObject M2M fields. + """ + if not pk_set: + return + from django.db.models.signals import m2m_changed + m2m_changed.send( + sender=self.through, + instance=self.instance, + action=action, + reverse=False, + model=getattr(self.through._meta.get_field('target').remote_field, 'model', None), + pk_set=pk_set, + using='default', + ) + def add(self, *objs): + added = set() for obj in objs: - self.through.objects.get_or_create( + _, created = self.through.objects.get_or_create( source_id=self.instance.pk, target_id=obj.pk ) + if created: + added.add(obj.pk) + if added: + self._fire_m2m_changed('post_add', added) def remove(self, *objs): + removed = set() for obj in objs: - self.through.objects.filter( + n, _ = self.through.objects.filter( source_id=self.instance.pk, target_id=obj.pk ).delete() + if n: + removed.add(obj.pk) + if removed: + self._fire_m2m_changed('post_remove', removed) def clear(self): - self.through.objects.filter(source_id=self.instance.pk).delete() + existing = set( + self.through.objects.filter(source_id=self.instance.pk) + .values_list('target_id', flat=True) + ) + if existing: + self.through.objects.filter(source_id=self.instance.pk).delete() + self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): objs = tuple(objs) # force evaluation before any mutation @@ -951,11 +989,15 @@ def set(self, objs, clear=False): source_id=self.instance.pk, target_id__in=to_remove, ).delete() + self._fire_m2m_changed('post_remove', to_remove) # Add only genuinely new relationships - for pk in new_pks - old_pks: + to_add = new_pks - old_pks + for pk in to_add: self.through.objects.get_or_create( source_id=self.instance.pk, target_id=pk ) + if to_add: + self._fire_m2m_changed('post_add', to_add) class CustomManyToManyDescriptor(ManyToManyDescriptor): diff --git a/netbox_custom_objects/forms.py b/netbox_custom_objects/forms.py index fe764bfe..d029ba70 100644 --- a/netbox_custom_objects/forms.py +++ b/netbox_custom_objects/forms.py @@ -213,7 +213,7 @@ class CustomObjectTypeFieldForm(CustomFieldForm): class Meta: model = CustomObjectTypeField - exclude = ('db_column',) + fields = '__all__' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/netbox_custom_objects/migrations/0014_branching_support.py b/netbox_custom_objects/migrations/0014_branching_support.py deleted file mode 100644 index 97b777b7..00000000 --- a/netbox_custom_objects/migrations/0014_branching_support.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Branching support for custom object tables. - -Add a frozen ``db_column`` to ``CustomObjectTypeField``. The physical column -name is set once at creation and never updated, so renames are metadata-only -and cannot mismatch across schemas (branch vs. main). Existing rows are -back-filled with ``db_column = name``. - -(The non-DEFERRABLE FK constraint fix that branching also requires is -performed by 0011_non_deferrable_fk_constraints.) -""" - -from django.db import migrations, models - - -def backfill_db_column(apps, schema_editor): - """Set db_column = name for all existing CustomObjectTypeField rows.""" - CustomObjectTypeField = apps.get_model('netbox_custom_objects', 'CustomObjectTypeField') - CustomObjectTypeField.objects.filter(db_column='').update(db_column=models.F('name')) - - -class Migration(migrations.Migration): - - dependencies = [ - ('netbox_custom_objects', '0013_polymorphic_object_fields'), - ] - - operations = [ - migrations.AddField( - model_name='customobjecttypefield', - name='db_column', - field=models.CharField( - blank=True, - default='', - help_text=( - 'Physical database column name. Set once at creation and never changed, ' - 'so renames are pure metadata changes that do not require DDL.' - ), - max_length=50, - verbose_name='database column', - ), - preserve_default=False, - ), - migrations.RunPython( - backfill_db_column, - migrations.RunPython.noop, - ), - ] diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index ce9740e6..b464afc6 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -192,7 +192,7 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): sync/merge replays an ObjectChange that was already applied). """ ft = FIELD_TYPE_CLASS[fi.type]() - mf = ft.get_model_field(fi, db_column=fi.effective_db_column) + mf = ft.get_model_field(fi) mf.contribute_to_class(model, fi.name) with schema_conn.cursor() as cursor: @@ -223,7 +223,7 @@ def _schema_remove_field(fi, model, schema_editor, existing_tables=None): to reject the subsequent ALTER TABLE. """ ft = FIELD_TYPE_CLASS[fi.type]() - mf = ft.get_model_field(fi, db_column=fi.effective_db_column) + mf = ft.get_model_field(fi) mf.contribute_to_class(model, fi.name) if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: @@ -286,8 +286,8 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist ) return - old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi, db_column=old_fi.effective_db_column) - new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi, db_column=new_fi.effective_db_column) + old_mf = FIELD_TYPE_CLASS[old_fi.type]().get_model_field(old_fi) + new_mf = FIELD_TYPE_CLASS[new_fi.type]().get_model_field(new_fi) old_mf.contribute_to_class(model, old_fi.name) new_mf.contribute_to_class(model, new_fi.name) @@ -324,7 +324,7 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist new_fi.pk, schema_conn.alias, ) return - live_mf = FIELD_TYPE_CLASS[live_fi.type]().get_model_field(live_fi, db_column=live_fi.effective_db_column) + live_mf = FIELD_TYPE_CLASS[live_fi.type]().get_model_field(live_fi) live_mf.contribute_to_class(model, live_fi.name) if live_mf.column not in existing_cols: logger.debug( @@ -549,6 +549,17 @@ def validate_pep440(value): class CustomObjectType(NetBoxModel): # Class-level cache for generated models + # Branch-aware model cache. + # + # Key: (custom_object_type_id, branch_id_or_None) + # branch_id_or_None == None → main schema (the canonical apps-registered class) + # branch_id_or_None == int → that branch's schema (cached only, not registered) + # + # Branch-specific classes are NEVER inserted into Django's apps registry. + # apps.all_models[APP_LABEL][] always points to main's class so that + # ``content_type.model_class()`` (used by netbox-branching's record_change_diff + # inside `with deactivate_branch():`) resolves to a class consistent with main's + # schema — branch's schema can have renamed columns that wouldn't exist in main. _model_cache = {} _through_model_cache = ( {} @@ -657,18 +668,55 @@ def clean(self): "exceeded; adjust max_custom_object_types to raise this limit" )) + @staticmethod + def _active_branch_id(): + """Return the active netbox-branching Branch id, or None for main. + + Used as the second component of the model cache key so that branch and + main contexts each get their own cached class, with main's class being + the only one registered in Django's apps registry. + """ + try: + from netbox_branching.contextvars import active_branch + except ImportError: + return None + try: + branch = active_branch.get() + except LookupError: + return None + return branch.id if branch is not None else None + @classmethod - def clear_model_cache(cls, custom_object_type_id=None): + def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): """ - Clear the model cache for a specific CustomObjectType or all models. + Clear the cached generated model for a CustomObjectType. + + Default behaviour clears only the **current branch context's** entry + (main vs. a specific branch). This preserves the other context's + cached class — important because that's the class registered in + ``apps.all_models`` and used by ``content_type.model_class()``. + + Pass ``all_branches=True`` to wipe every (cot, branch) entry for the + given cot — appropriate for COT deletion or full re-init. - :param custom_object_type_id: ID of the CustomObjectType to clear cache for, or None to clear all + :param custom_object_type_id: ID of the CustomObjectType to clear cache + for, or None to clear *everything*. + :param all_branches: When True with a specific cot id, clear that cot's + cache for all branch contexts, not just the active one. """ with cls._global_lock: if custom_object_type_id is not None: - cls._model_cache.pop(custom_object_type_id, None) - cls._through_model_cache.pop(custom_object_type_id, None) - cls._model_cache_locks.pop(custom_object_type_id, None) + if all_branches: + for key in list(cls._model_cache): + if key[0] == custom_object_type_id: + cls._model_cache.pop(key, None) + cls._through_model_cache.pop(custom_object_type_id, None) + cls._model_cache_locks.pop(custom_object_type_id, None) + else: + branch_id = cls._active_branch_id() + cls._model_cache.pop((custom_object_type_id, branch_id), None) + # Through-model cache and per-cot lock are not branch-scoped; + # leave them alone for context-only clears. else: cls._model_cache.clear() cls._through_model_cache.clear() @@ -678,42 +726,43 @@ def clear_model_cache(cls, custom_object_type_id=None): apps.get_models.cache_clear() @classmethod - def get_cached_model(cls, custom_object_type_id): + def get_cached_model(cls, custom_object_type_id, branch_id=None): """ - Get a cached model for a specific CustomObjectType if it exists. + Get the cached model for a CustomObjectType in the given branch context. :param custom_object_type_id: ID of the CustomObjectType + :param branch_id: Branch id, or None for main (default) :return: The cached model or None if not found """ - cache_entry = cls._model_cache.get(custom_object_type_id) + cache_entry = cls._model_cache.get((custom_object_type_id, branch_id)) if cache_entry: - # Cache stores (model, timestamp) tuples return cache_entry[0] return None @classmethod - def get_cached_timestamp(cls, custom_object_type_id): + def get_cached_timestamp(cls, custom_object_type_id, branch_id=None): """ - Get the timestamp of a cached model for a specific CustomObjectType. + Get the timestamp of a cached model for a CustomObjectType in the given branch context. :param custom_object_type_id: ID of the CustomObjectType + :param branch_id: Branch id, or None for main (default) :return: The cached timestamp or None if not found """ - cache_entry = cls._model_cache.get(custom_object_type_id) + cache_entry = cls._model_cache.get((custom_object_type_id, branch_id)) if cache_entry: - # Cache stores (model, timestamp) tuples return cache_entry[1] return None @classmethod - def is_model_cached(cls, custom_object_type_id): + def is_model_cached(cls, custom_object_type_id, branch_id=None): """ - Check if a model is cached for a specific CustomObjectType. + Check if a model is cached for a CustomObjectType in the given branch context. :param custom_object_type_id: ID of the CustomObjectType + :param branch_id: Branch id, or None for main (default) :return: True if the model is cached, False otherwise """ - return custom_object_type_id in cls._model_cache + return (custom_object_type_id, branch_id) in cls._model_cache @classmethod def get_cached_through_model(cls, custom_object_type_id, through_model_name): @@ -789,9 +838,7 @@ def _fetch_and_generate_field_attrs( field_name = field.name try: - model_field = field_type.get_model_field( - field, db_column=field.effective_db_column, - ) + model_field = field_type.get_model_field(field) except NotImplementedError: if field.related_object_type_id is None: logger.debug( @@ -1035,12 +1082,22 @@ def get_model( :rtype: Model """ + branch_id = self._active_branch_id() + with self._global_lock: - if self.is_model_cached(self.id) and not no_cache: - cached_timestamp = self.get_cached_timestamp(self.id) + if self.is_model_cached(self.id, branch_id) and not no_cache: + cached_timestamp = self.get_cached_timestamp(self.id, branch_id) # Only use cache if the timestamps are available and match if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: - model = self.get_cached_model(self.id) + model = self.get_cached_model(self.id, branch_id) + # Re-register the SearchIndex against this cached class. The + # ``registry["search"]`` dict is global, not per-branch, so a + # previous get_model() call in a different context may have + # left it bound to a class with a different field set. + # Without this refresh, the next CO save in this context + # would fail when post_save's search-cache handler reads + # field names that don't exist on the active model class. + self.register_custom_object_search_index(model) return model else: self.clear_model_cache(self.id) @@ -1119,17 +1176,36 @@ def wrapped_post_through_setup(self, cls): # Without suppression: register_model() → clear_cache() → get_models() → # get_model() → generate_model() → register_model() recurses infinitely. with _suppress_clear_cache(): - if model_name.lower() in apps.all_models[APP_LABEL]: - # Remove the existing model from all_models before registering the new one - del apps.all_models[APP_LABEL][model_name.lower()] - - apps.register_model(APP_LABEL, model) + # Django's ModelBase metaclass auto-registers every concrete model with + # ``Meta.app_label`` set into ``apps.all_models`` at ``type()``-creation + # time (inside generate_model() above). That's the right behaviour for + # main's class, but for branch classes we must NOT leave the auto- + # registration in place — content_type.model_class() (used by + # netbox-branching's record_change_diff inside `with deactivate_branch():`) + # would then return a class with branch's column set, producing + # `column "beta" does not exist` errors when querying main. + # + # So: for main (branch_id is None) overwrite the registration cleanly; + # for a branch context, restore main's previously-cached class so the + # apps registry continues to reflect main's schema. + model_key = model_name.lower() + if branch_id is None: + if model_key in apps.all_models[APP_LABEL]: + del apps.all_models[APP_LABEL][model_key] + apps.register_model(APP_LABEL, model) + else: + main_class = self.get_cached_model(self.id, branch_id=None) + if main_class is not None: + apps.all_models[APP_LABEL][model_key] = main_class + # If main hasn't been generated yet, the branch class stays + # registered until the next main-context get_model() call, which + # will replace it. This is rare and self-healing. self._after_model_generation(attrs, model) # Cache the generated model with its timestamp (protected by lock for thread safety) with self._global_lock: - self._model_cache[self.id] = (model, self.cache_timestamp) + self._model_cache[(self.id, branch_id)] = (model, self.cache_timestamp) # Now that the model is in _model_cache, clear_cache() is safe: # re-entrant get_model() calls for this COT hit the cache immediately. @@ -1303,8 +1379,9 @@ def save(self, *args, **kwargs): self.clear_model_cache(self.id) def delete(self, *args, **kwargs): - # Clear the model cache for this CustomObjectType - self.clear_model_cache(self.id) + # Clear the model cache for this CustomObjectType (across all branches — + # the COT itself is going away, so every branch's cached class becomes stale). + self.clear_model_cache(self.id, all_branches=True) model = self.get_model() schema_conn = _get_schema_connection() @@ -1375,8 +1452,9 @@ def delete(self, *args, **kwargs): # longer discovered during cascade-delete collector traversal. apps.clear_cache() - # Re-clear the model cache to remove re-cached model from get_model. - self.clear_model_cache(self.id) + # Re-clear the model cache to remove re-cached model from get_model + # (across all branches — see comment at the top of this method). + self.clear_model_cache(self.id, all_branches=True) @receiver(post_save, sender=CustomObjectType) @@ -1507,15 +1585,6 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode ), ), ) - db_column = models.CharField( - verbose_name=_("database column"), - max_length=50, - blank=True, - help_text=_( - "Physical database column name. Set once at creation and never changed, " - "so renames are pure metadata changes that do not require DDL." - ), - ) label = models.CharField( verbose_name=_("label"), max_length=50, @@ -1763,18 +1832,6 @@ def is_single_value(self): def many(self): return self.type in ["multiobject"] - @property - def effective_db_column(self): - """ - Return the physical database column name for this field. - - Equal to ``self.name`` — the column name tracks the field name, so - renames perform an ALTER TABLE RENAME COLUMN (handled by - ``_schema_alter_field``). The stored ``db_column`` field is kept for - change-logging legibility but is not used for column resolution. - """ - return self.name - def get_child_relations(self, instance): return instance.get_field_value(self) @@ -1912,11 +1969,11 @@ def clean(self): and not self.original.unique ): field_type = FIELD_TYPE_CLASS[self.type]() - model_field = field_type.get_model_field(self, db_column=self.effective_db_column) + model_field = field_type.get_model_field(self) model = self.custom_object_type.get_model() model_field.contribute_to_class(model, self.name) - old_field = field_type.get_model_field(self.original, db_column=self.original.effective_db_column) + old_field = field_type.get_model_field(self.original) old_field.contribute_to_class(model, self._original_name) try: @@ -2611,12 +2668,6 @@ def save(self, *args, **kwargs): ).update(next_schema_id=new_schema_id) self.schema_id = new_schema_id - # Freeze the physical column name at creation. db_column is set once - # here and never updated, so subsequent renames only update the ORM - # field name — no DDL is required for renames. - if self._state.adding and not self.db_column: - self.db_column = self.name - field_type = FIELD_TYPE_CLASS[self.type]() model = self.custom_object_type.get_model() @@ -2646,7 +2697,11 @@ def save(self, *args, **kwargs): _schema_alter_field(self.original, self, model, schema_editor, schema_conn) # When the field is renamed, update ObjectChange / ChangeDiff JSON keys so - # historical audit records and branch diffs stay consistent with the new name. + # historical audit records and branch diffs stay consistent with the new + # name. Combined with the netbox-branching attr translator registered + # in branching.translate_renamed_field_attr, this lets later replays + # (whether iterative undo or squash undo) resolve any data key — old or + # new name — to the field's current name on the model. if ( not self._state.adding and not self.is_polymorphic From 6d84fe03de522719a2424b4bd42ff4dba20ac440 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 8 May 2026 09:11:47 -0700 Subject: [PATCH 037/115] changes for m2m --- netbox_custom_objects/field_types.py | 36 +++++++++-- netbox_custom_objects/models.py | 91 +++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 4a4226de..fbfbacef 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -940,25 +940,37 @@ def _fire_m2m_changed(self, action, pk_set): using='default', ) + @staticmethod + def _resolve_pk(obj): + """Accept either a model instance (with ``.pk``) or a raw PK value. + + Django's standard ManyRelatedManager allows both forms (instances or + PKs); netbox-branching's ``update_object`` passes PKs from + deserialized JSON, so we mirror that contract here. + """ + return obj.pk if hasattr(obj, 'pk') else obj + def add(self, *objs): added = set() for obj in objs: + pk = self._resolve_pk(obj) _, created = self.through.objects.get_or_create( - source_id=self.instance.pk, target_id=obj.pk + source_id=self.instance.pk, target_id=pk ) if created: - added.add(obj.pk) + added.add(pk) if added: self._fire_m2m_changed('post_add', added) def remove(self, *objs): removed = set() for obj in objs: + pk = self._resolve_pk(obj) n, _ = self.through.objects.filter( - source_id=self.instance.pk, target_id=obj.pk + source_id=self.instance.pk, target_id=pk ).delete() if n: - removed.add(obj.pk) + removed.add(pk) if removed: self._fire_m2m_changed('post_remove', removed) @@ -977,7 +989,7 @@ def set(self, objs, clear=False): self.clear() self.add(*objs) else: - new_pks = {obj.pk for obj in objs} + new_pks = {self._resolve_pk(obj) for obj in objs} old_pks = set( self.through.objects.filter(source_id=self.instance.pk) .values_list('target_id', flat=True) @@ -1303,10 +1315,22 @@ def after_model_generation(self, instance, model, field_name): field = model._meta.get_field(field_name) + # Mark the through model as "auto-created" by the parent CO model. + # Django's JSON serializer (django/core/serializers/python.py + # handle_m2m_field) skips M2M fields whose through is *not* + # auto-created — assuming an explicit through table will be serialized + # directly. Our through tables are dynamically generated to mirror + # Django's auto behaviour; setting ``auto_created`` on the through's + # _meta makes the serializer include the M2M values in + # ``serialize_object`` output, which is what netbox change-logging + # and netbox-branching's merge replay rely on. + through_model = field.remote_field.through + if through_model is not None: + through_model._meta.auto_created = model + # Skip model resolution for self-referential fields if getattr(field, "_is_self_referential", False): field.remote_field.model = model - through_model = field.remote_field.through # Update both source and target fields to point to the same model source_field = through_model._meta.get_field("source") diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index b464afc6..7c5dfe22 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -442,8 +442,79 @@ def deserialize_object(cls, data, pk=None): except CustomObjectType.DoesNotExist: fresh_model = cls - inner = _deserialize_object(fresh_model, data, pk=pk) - obj = inner.object + # Build the instance directly against ``fresh_model`` rather than going + # through Django's ``serializers.deserialize('python', ...)``: that + # helper re-resolves the model class from the data's natural_key via + # ``apps.get_model``, which returns *main's* class (the one registered + # in ``apps.all_models``). In branch context, main's class can have a + # different field set than the branch's class — e.g. main has 'alpha' + # while branch has 'branch_alpha' after a branch-side rename — so the + # eventual save would emit SQL with main's column names against a + # branch table that doesn't have them, raising + # ``column "alpha" does not exist``. + # + # Building directly off ``fresh_model`` (the context-aware class) keeps + # the field set aligned with the schema we're writing to, and the attr + # translator registered with netbox-branching (see + # branching.translate_renamed_field_attr) lets us map the data dict's + # old field names to the current model's field names where they + # diverge. + from django.db.models.fields.related import ForeignKey, ManyToManyField + from extras.utils import is_taggable + + # Try the registered netbox-branching attr translator if available so + # that data dicts carrying old field names are reshaped to the current + # context's field names before deserialization. Falls back gracefully + # when the translator isn't registered (e.g. branching not installed). + try: + from netbox_branching.utilities import _translate_attr # type: ignore[attr-defined] + except ImportError: + def _translate_attr(_inst, attr): # noqa: D401 + return None + + obj = fresh_model() + if pk is not None: + obj.pk = pk + m2m_data = {} + field_names = {f.name for f in fresh_model._meta.get_fields()} + + for raw_attr, value in data.items(): + attr = raw_attr + if attr == 'custom_fields': + attr = 'custom_field_data' + # Tags via the standard NetBox path (Tag rows are looked up by name). + if attr == 'tags' and is_taggable(fresh_model): + tag_model = apps.get_model('extras', 'Tag') + m2m_data['tags'] = list(tag_model.objects.filter(name__in=value or [])) + continue + if attr not in field_names: + resolved = _translate_attr(obj, attr) + if resolved and resolved in field_names: + attr = resolved + else: + # Unknown attribute (likely a removed field) — preserve it as + # a Python attribute so downstream code (e.g. + # _deferred_co_field_data) can still see it. + setattr(obj, raw_attr, value) + continue + try: + f = fresh_model._meta.get_field(attr) + except Exception: + setattr(obj, raw_attr, value) + continue + if isinstance(f, ManyToManyField): + m2m_data[attr] = value + elif isinstance(f, ForeignKey): + # FK values arrive as the related PK; assign via the FK column + # (``_id``) to avoid an extra DB lookup. + setattr(obj, f.attname, value) + else: + # Coerce via the field's to_python() to handle datetimes etc. + try: + setattr(obj, attr, f.to_python(value)) + except Exception: + setattr(obj, attr, value) + table_name = fresh_model._meta.db_table full_data = dict(data) @@ -456,6 +527,22 @@ def save(self, using=None, **_kwargs): # Read pk after save_base so that auto-assigned PKs are captured. # (If pk was None before save_base, obj.pk is now the DB-assigned id.) obj_pk = obj.pk + # Re-apply M2M relations. Each ``accessor`` is a manager + # attribute name on the saved instance and ``related_pks`` is a + # list of related PKs. Skipped quietly if the manager isn't + # present yet (squash ordering can defer the field's column + # creation; the through-table is created when the field's own + # CREATE ObjectChange is later applied, and the + # _deferred_co_field_data registration below will pick it up). + for accessor, related_pks in m2m_data.items(): + try: + manager = getattr(obj, accessor) + manager.set(related_pks) + except Exception: + logger.debug( + 'deserialize_object: deferred M2M %r on %s pk=%s', + accessor, table_name, obj_pk, exc_info=True, + ) # Register full data for deferred column updates (squash ordering fix). deferred = _deferred_co_field_data.get() if deferred is None: From 6bfba4f9feb34962e8ab565093697deba776b9d3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 8 May 2026 10:23:33 -0700 Subject: [PATCH 038/115] fixes for testing --- netbox_custom_objects/models.py | 37 ++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 7c5dfe22..85a77e65 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -2845,18 +2845,31 @@ def ensure_constraint(): transaction.on_commit(ensure_constraint) - # Regenerate the model from DB now that the field change is persisted. - # _schema_alter_field adds both old and new field names to the model - # class via contribute_to_class, and something between clear_model_cache - # and super().save() (e.g. a post_save signal on the COT) can call - # get_model() while the DB still has the old name, caching a stale model - # without the new name. Forcing a no_cache regeneration here (after - # super().save() committed the new name) ensures apps.all_models holds a - # clean model with exactly the current DB field list. - updated_model = self.custom_object_type.get_model(no_cache=True) - - # Reregister SearchIndex with new set of searchable fields - self.custom_object_type.register_custom_object_search_index(updated_model) + # On rename, ``_schema_alter_field`` calls contribute_to_class for both + # the old and new field names on the same model class, leaving the + # _meta in a corrupt state. Force a no_cache regeneration so the + # next access sees a clean model. This regeneration ALSO triggers + # apps.clear_cache() at the end of get_model(), which cascades into + # AppConfig.get_models() and re-caches every COT — but on rename + # that's tolerable because the rename doesn't affect related COTs' + # FK reverse relations. + # + # For non-rename changes (new field, attribute toggle, etc.) the + # cache_timestamp bump done above is sufficient: the next get_model() + # call detects the timestamp mismatch and regenerates lazily. We + # skip both the regeneration AND the search-index re-register here + # so that the cache invalidations performed by signal handlers — most + # importantly the eviction of related COTs in ``clear_cache_on_field_save`` + # for an OBJECT-type field — are not undone by the apps.clear_cache() + # cascade re-caching every COT in sight. + renamed = ( + not self._state.adding + and not self.is_polymorphic + and self._original_name != self.name + ) + if renamed: + updated_model = self.custom_object_type.get_model(no_cache=True) + self.custom_object_type.register_custom_object_search_index(updated_model) # Reindex all objects of this type if search indexing was affected if is_new: From ac0de3bb1fc554da38354f05b799f112fadd50ce Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 8 May 2026 11:20:37 -0700 Subject: [PATCH 039/115] add missing file --- netbox_custom_objects/branching.py | 100 +++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 netbox_custom_objects/branching.py diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py new file mode 100644 index 00000000..ec145f9f --- /dev/null +++ b/netbox_custom_objects/branching.py @@ -0,0 +1,100 @@ +""" +netbox-branching integration hooks for netbox-custom-objects. + +These functions plug into extension points exposed by netbox-branching so the +plugin can correctly handle changes to its dynamically-generated CustomObject +models across schema-rename history. Importing netbox-branching is +deferred / optional — none of these functions run if netbox-branching is not +installed (the registration call sites in ``__init__.ready()`` are guarded). +""" + +from core.choices import ObjectChangeActionChoices +from core.models import ObjectChange + + +def supports_branching_resolver(model): + """ + Mark CustomObject M2M through-models as branchable. + + The dynamically-generated through models for ``MULTIOBJECT`` fields are + plain ``models.Model`` subclasses (no ``ChangeLoggingMixin``), so + netbox-branching's default heuristic would route their queries to main + even when an active branch is set. That breaks branch-context M2M + assignments because the through-table FK to the parent CO model resolves + to main's row set, which is missing branch-only CO instances. + + Returning ``True`` here pulls these through models into the same + branching connection routing as their parent CO model. Returning + ``None`` for everything else lets the default heuristic run. + """ + meta = getattr(model, '_meta', None) + if meta is None or meta.app_label != 'netbox_custom_objects': + return None + name = meta.model_name or '' + # Through models are named ``through_custom_objects__`` + # (see CustomObjectTypeField.through_model_name). Match anything with + # that prefix. + if name.startswith('through_custom_objects_'): + return True + return None + + +def translate_renamed_field_attr(instance, attr): + """ + Resolve a CustomObject data-attribute name to its current field name, + walking the field's rename history when the raw name doesn't match the + instance's current field set. + + Called by ``netbox_branching.utilities.update_object()`` when a stored + ObjectChange / ChangeDiff data dict carries a key that does not match + any current field on ``instance`` — the typical case is a squash-merge + revert where the collapsed prechange dict still uses the field's old + name (e.g. 'beta') while the model class has the field's current name + (e.g. 'gamma'). Returning the current name lets ``update_object`` + write the value to the correct column. + + For non-CustomObject instances or unresolved names, returns ``None`` so + other registered translators (or the default behaviour) get a chance. + """ + if not getattr(instance, '_generated_table_model', False): + return None # not a CustomObject — defer to other translators + + cot = getattr(instance, 'custom_object_type', None) + if cot is None: + return None + + # The field we're looking for is one of this COT's CustomObjectTypeFields + # whose history (via ObjectChanges of name) includes ``attr`` as a former + # or current name. We match by walking ObjectChanges whose + # postchange_data['name'] or prechange_data['name'] equals ``attr``. + field_ct = ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + ) + candidate_pks = set() + for oc in field_ct.filter(action=ObjectChangeActionChoices.ACTION_UPDATE): + post = oc.postchange_data or {} + pre = oc.prechange_data or {} + if post.get('name') == attr or pre.get('name') == attr: + candidate_pks.add(oc.changed_object_id) + if not candidate_pks: + return None + + # Filter candidate fields to those belonging to this COT. The fields + # have stable PKs across schemas, and current name reflects the latest + # state in the active context. + fields_qs = cot.fields.filter(pk__in=candidate_pks) + fields = list(fields_qs.values_list('name', flat=True)) + if not fields: + return None + + # If a field was renamed, ``attr`` mapped to its PK; the field's current + # ``name`` is the translated attribute. When more than one field has + # ``attr`` somewhere in its history (renamed away then renamed back, or + # multiple fields cycling through the same name) we abstain — picking + # arbitrarily would silently overwrite the wrong column. This is + # vanishingly rare for normal use and produces a clear "unknown attr" + # signal at the call site rather than data corruption. + if len(fields) == 1: + return fields[0] + return None From 726ec683e09c8f9fd014943e06c5ce836f9dffee Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 8 May 2026 11:26:50 -0700 Subject: [PATCH 040/115] cleanup --- netbox_custom_objects/branching.py | 24 +++++++------ netbox_custom_objects/field_types.py | 2 +- netbox_custom_objects/models.py | 36 ++++++++++++------- .../tests/test_field_types.py | 5 ++- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index ec145f9f..d7313525 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -8,6 +8,8 @@ installed (the registration call sites in ``__init__.ready()`` are guarded). """ +from django.db.models import Q + from core.choices import ObjectChangeActionChoices from core.models import ObjectChange @@ -65,18 +67,18 @@ def translate_renamed_field_attr(instance, attr): # The field we're looking for is one of this COT's CustomObjectTypeFields # whose history (via ObjectChanges of name) includes ``attr`` as a former - # or current name. We match by walking ObjectChanges whose - # postchange_data['name'] or prechange_data['name'] equals ``attr``. - field_ct = ObjectChange.objects.filter( - changed_object_type__app_label='netbox_custom_objects', - changed_object_type__model='customobjecttypefield', + # or current name. Match at the DB level on JSON keys + # ``postchange_data->>'name'`` / ``prechange_data->>'name'`` so we don't + # pull every UPDATE row into Python on every translator invocation. + candidate_pks = set( + ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).filter( + Q(postchange_data__name=attr) | Q(prechange_data__name=attr) + ).values_list('changed_object_id', flat=True) ) - candidate_pks = set() - for oc in field_ct.filter(action=ObjectChangeActionChoices.ACTION_UPDATE): - post = oc.postchange_data or {} - pre = oc.prechange_data or {} - if post.get('name') == attr or pre.get('name') == attr: - candidate_pks.add(oc.changed_object_id) if not candidate_pks: return None diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index fbfbacef..f604f375 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -13,6 +13,7 @@ from django.db import connection, models from django.db.models.fields.related import ForeignKey, ManyToManyDescriptor from django.db.models.manager import Manager +from django.db.models.signals import m2m_changed from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -929,7 +930,6 @@ def _fire_m2m_changed(self, action, pk_set): """ if not pk_set: return - from django.db.models.signals import m2m_changed m2m_changed.send( sender=self.through, instance=self.instance, diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 65fc1d24..04296a9c 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -15,9 +15,12 @@ # from django.contrib.contenttypes.management import create_contenttypes from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import FieldDoesNotExist from django.core.validators import RegexValidator, ValidationError from django.db import DEFAULT_DB_ALIAS, connection, connections, IntegrityError, models, transaction +from django.db.utils import OperationalError, ProgrammingError from django.db.models import Q +from django.db.models.fields.related import ForeignKey, ManyToManyField from django.db.models.functions import Lower from django.db.models.signals import m2m_changed, pre_delete, post_save from django.dispatch import receiver @@ -31,7 +34,7 @@ CustomFieldUIVisibleChoices, ) from extras.models.customfields import SEARCH_TYPES -from extras.utils import run_validators +from extras.utils import is_taggable, run_validators from netbox.config import get_config from netbox.models import ChangeLoggedModel, NetBoxModel from netbox.models.features import ( @@ -155,9 +158,12 @@ def _apply_deferred_co_field(field_instance): with schema_conn.cursor() as cursor: for co_pk, entry in per_table.items(): - value = entry['data'].get(data_key) - if value is None: + # Distinguish "key absent" from "key present with NULL". An explicit + # None is a legitimate write and must reach the column; only a + # missing key should be skipped. + if data_key not in entry['data']: continue + value = entry['data'][data_key] # table_name comes from get_database_table_name() (controlled by our # code) and col_name from field.name, which is validated by the # ^[a-z0-9_]+$ regex — no double-quote characters are possible, so @@ -472,8 +478,6 @@ def deserialize_object(cls, data, pk=None): # branching.translate_renamed_field_attr) lets us map the data dict's # old field names to the current model's field names where they # diverge. - from django.db.models.fields.related import ForeignKey, ManyToManyField - from extras.utils import is_taggable # Try the registered netbox-branching attr translator if available so # that data dicts carrying old field names are reshaped to the current @@ -512,7 +516,7 @@ def _translate_attr(_inst, attr): # noqa: D401 continue try: f = fresh_model._meta.get_field(attr) - except Exception: + except FieldDoesNotExist: setattr(obj, raw_attr, value) continue if isinstance(f, ManyToManyField): @@ -523,9 +527,12 @@ def _translate_attr(_inst, attr): # noqa: D401 setattr(obj, f.attname, value) else: # Coerce via the field's to_python() to handle datetimes etc. + # Field.to_python raises ValidationError for parse failures; the + # raw value is also acceptable for ValueError/TypeError on + # malformed input from older serialized data. try: setattr(obj, attr, f.to_python(value)) - except Exception: + except (ValidationError, ValueError, TypeError): setattr(obj, attr, value) table_name = fresh_model._meta.db_table @@ -542,16 +549,19 @@ def save(self, using=None, **_kwargs): obj_pk = obj.pk # Re-apply M2M relations. Each ``accessor`` is a manager # attribute name on the saved instance and ``related_pks`` is a - # list of related PKs. Skipped quietly if the manager isn't - # present yet (squash ordering can defer the field's column - # creation; the through-table is created when the field's own - # CREATE ObjectChange is later applied, and the - # _deferred_co_field_data registration below will pick it up). + # list of related PKs. Skipped quietly when the through table + # or its columns aren't present yet — squash ordering can defer + # field creation, and the through-table appears when the field's + # own CREATE ObjectChange is later applied. We narrow the + # except to the failure modes that signature: ``AttributeError`` + # if the manager descriptor isn't bound to ``obj`` yet, and + # ``ProgrammingError``/``OperationalError`` if the through + # table/column is missing in the active schema. for accessor, related_pks in m2m_data.items(): try: manager = getattr(obj, accessor) manager.set(related_pks) - except Exception: + except (AttributeError, ProgrammingError, OperationalError): logger.debug( 'deserialize_object: deferred M2M %r on %s pk=%s', accessor, table_name, obj_pk, exc_info=True, diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index dfeea729..09cde697 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -2,7 +2,7 @@ Tests for all the different field types supported by Custom Object Type Fields. """ from unittest import skip -from datetime import date, datetime +from datetime import date, datetime, timezone from decimal import Decimal from django.core.exceptions import ValidationError from django.test import TestCase @@ -390,9 +390,8 @@ def test_datetime_field_model_generation(self): type="datetime" ) - import datetime as dt model = self.custom_object_type.get_model() - test_datetime = datetime(2023, 1, 1, 12, 0, 0, tzinfo=dt.timezone.utc) + test_datetime = datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc) instance = model.objects.create(name="Test", created_datetime=test_datetime) self.assertEqual(instance.created_datetime, test_datetime) From 051572dfd94317ff8ed799aadbaa48deed039a53 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 8 May 2026 11:42:52 -0700 Subject: [PATCH 041/115] cleanup --- netbox_custom_objects/models.py | 39 ++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 04296a9c..37cb235f 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1620,7 +1620,7 @@ def _rename_objectchange_field_key(fi, old_name, new_name): 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' 'WHERE changed_object_type_id = %s AND {col} ? %s' ) - with connections[conn.alias].cursor() as cursor: + with conn.cursor() as cursor: for json_col in ('prechange_data', 'postchange_data'): cursor.execute(oc_sql.format(col=json_col), [old_name, new_name, old_name, ct.id, old_name]) @@ -1628,19 +1628,32 @@ def _rename_objectchange_field_key(fi, old_name, new_name): try: from netbox_branching.models import ChangeDiff # noqa: F401 — presence check only - cd_sql = ( - 'UPDATE netbox_branching_changediff ' - 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' - 'WHERE object_type_id = %s AND {col} IS NOT NULL AND {col} ? %s' - ) - with connections[conn.alias].cursor() as cursor: - for json_col in ('original', 'modified', 'current'): - cursor.execute(cd_sql.format(col=json_col), [old_name, new_name, old_name, ct.id, old_name]) except ImportError: - pass # netbox-branching not installed - except Exception: - logger.debug( - '_rename_objectchange_field_key: ChangeDiff rename failed for %r → %r', + return # netbox-branching not installed; nothing more to update + + cd_sql = ( + 'UPDATE netbox_branching_changediff ' + 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' + 'WHERE object_type_id = %s AND {col} IS NOT NULL AND {col} ? %s' + ) + # Wrap in a savepoint so a ProgrammingError (table/column missing due to + # netbox-branching schema mismatch) doesn't poison the outer merge + # transaction — every subsequent statement on the same connection would + # otherwise raise InFailedSqlTransaction. Other DatabaseError subclasses + # are allowed to propagate so a real bug aborts the rename instead of + # silently corrupting audit data. + try: + with transaction.atomic(using=conn.alias): + with conn.cursor() as cursor: + for json_col in ('original', 'modified', 'current'): + cursor.execute( + cd_sql.format(col=json_col), + [old_name, new_name, old_name, ct.id, old_name], + ) + except ProgrammingError: + logger.warning( + '_rename_objectchange_field_key: ChangeDiff schema mismatch; ' + 'audit data for %r may not reflect rename to %r', old_name, new_name, exc_info=True, ) From b36720f8a7780e0643146a24c06ca2e15542ee8c Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 12 May 2026 09:10:27 -0700 Subject: [PATCH 042/115] refactor --- netbox_custom_objects/__init__.py | 20 ++-- netbox_custom_objects/branching.py | 79 ++-------------- netbox_custom_objects/models.py | 145 +++++++++++++++++++++++------ 3 files changed, 133 insertions(+), 111 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index a3c040d0..ab917fb6 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -278,19 +278,15 @@ def ready(self): # leftover entries from a failed prior operation don't leak forward. _connect_deferred_data_reset_signals() - # Register netbox-branching hooks so its router and update_object - # know about our dynamically-generated models. Guarded so the - # plugin still works without netbox-branching installed. + # Register netbox-branching hooks so its router knows about our + # dynamically-generated through models. Guarded so the plugin still + # works without netbox-branching installed. Field-rename translation + # is handled by ``CustomObject.canonicalize_data`` on the model itself, + # which netbox-branching invokes from ``update_object`` and + # ``ChangeDiff._update_conflicts`` — no registration required. try: - from netbox_branching.utilities import ( - register_attr_translator, - register_branching_resolver, - ) - from .branching import ( - supports_branching_resolver, - translate_renamed_field_attr, - ) - register_attr_translator(translate_renamed_field_attr) + from netbox_branching.utilities import register_branching_resolver + from .branching import supports_branching_resolver register_branching_resolver(supports_branching_resolver) except ImportError: pass diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index d7313525..740c2bb8 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -2,17 +2,17 @@ netbox-branching integration hooks for netbox-custom-objects. These functions plug into extension points exposed by netbox-branching so the -plugin can correctly handle changes to its dynamically-generated CustomObject -models across schema-rename history. Importing netbox-branching is -deferred / optional — none of these functions run if netbox-branching is not -installed (the registration call sites in ``__init__.ready()`` are guarded). +plugin can correctly route queries for its dynamically-generated through +models when an active branch is set. Importing netbox-branching is deferred / +optional — the registration call site in ``__init__.ready()`` is guarded so +the plugin still works when netbox-branching is not installed. + +Field-rename translation lives on ``CustomObject.canonicalize_data`` (a model +classmethod) and is invoked directly by netbox-branching from +``update_object`` and ``ChangeDiff._update_conflicts``; no registration is +required for that path. """ -from django.db.models import Q - -from core.choices import ObjectChangeActionChoices -from core.models import ObjectChange - def supports_branching_resolver(model): """ @@ -39,64 +39,3 @@ def supports_branching_resolver(model): if name.startswith('through_custom_objects_'): return True return None - - -def translate_renamed_field_attr(instance, attr): - """ - Resolve a CustomObject data-attribute name to its current field name, - walking the field's rename history when the raw name doesn't match the - instance's current field set. - - Called by ``netbox_branching.utilities.update_object()`` when a stored - ObjectChange / ChangeDiff data dict carries a key that does not match - any current field on ``instance`` — the typical case is a squash-merge - revert where the collapsed prechange dict still uses the field's old - name (e.g. 'beta') while the model class has the field's current name - (e.g. 'gamma'). Returning the current name lets ``update_object`` - write the value to the correct column. - - For non-CustomObject instances or unresolved names, returns ``None`` so - other registered translators (or the default behaviour) get a chance. - """ - if not getattr(instance, '_generated_table_model', False): - return None # not a CustomObject — defer to other translators - - cot = getattr(instance, 'custom_object_type', None) - if cot is None: - return None - - # The field we're looking for is one of this COT's CustomObjectTypeFields - # whose history (via ObjectChanges of name) includes ``attr`` as a former - # or current name. Match at the DB level on JSON keys - # ``postchange_data->>'name'`` / ``prechange_data->>'name'`` so we don't - # pull every UPDATE row into Python on every translator invocation. - candidate_pks = set( - ObjectChange.objects.filter( - changed_object_type__app_label='netbox_custom_objects', - changed_object_type__model='customobjecttypefield', - action=ObjectChangeActionChoices.ACTION_UPDATE, - ).filter( - Q(postchange_data__name=attr) | Q(prechange_data__name=attr) - ).values_list('changed_object_id', flat=True) - ) - if not candidate_pks: - return None - - # Filter candidate fields to those belonging to this COT. The fields - # have stable PKs across schemas, and current name reflects the latest - # state in the active context. - fields_qs = cot.fields.filter(pk__in=candidate_pks) - fields = list(fields_qs.values_list('name', flat=True)) - if not fields: - return None - - # If a field was renamed, ``attr`` mapped to its PK; the field's current - # ``name`` is the translated attribute. When more than one field has - # ``attr`` somewhere in its history (renamed away then renamed back, or - # multiple fields cycling through the same name) we abstain — picking - # arbitrarily would silently overwrite the wrong column. This is - # vanishingly rare for normal use and produces a clear "unknown attr" - # signal at the call site rather than data corruption. - if len(fields) == 1: - return fields[0] - return None diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 37cb235f..1d8e56b8 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -8,6 +8,7 @@ from packaging.version import Version, InvalidVersion import django_filters +from core.choices import ObjectChangeActionChoices from core.models import ObjectType, ObjectChange from core.models.object_types import ObjectTypeManager from django.apps import apps @@ -385,6 +386,57 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist schema_editor.alter_field(model, old_mf, new_mf) +def _translate_renamed_field_name(cot, attr): + """ + Resolve ``attr`` to a CustomObjectTypeField's *current* name on *cot* by + walking the field's rename history. + + The field we're looking for is one of this COT's CustomObjectTypeFields + whose history (via ObjectChanges of ``name``) includes ``attr`` as a former + or current name. We match at the DB level on JSON keys + ``postchange_data->>'name'`` / ``prechange_data->>'name'`` so we don't pull + every UPDATE row into Python on every invocation. + + Returns the field's current name when there is exactly one matching field, + or ``None`` otherwise. We abstain on ambiguity (the same name appearing in + multiple fields' rename history) because picking arbitrarily would + silently overwrite the wrong column; the caller then preserves the raw + key. This case is vanishingly rare for normal use. + """ + candidate_pks = set( + ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).filter( + Q(postchange_data__name=attr) | Q(prechange_data__name=attr) + ).values_list('changed_object_id', flat=True) + ) + if not candidate_pks: + return None + fields = list( + cot.fields.filter(pk__in=candidate_pks).values_list('name', flat=True) + ) + if len(fields) == 1: + return fields[0] + return None + + +def _set_with_collision_preference(result, key, value): + """ + Assign ``result[key] = value``; on collision prefer the non-None value. + + Two raw data keys can map to the same canonical key after rename + translation (squash-merge artefact: both the old name and the new name + appear, with the new name often carrying a sentinel ``None`` from + ``deep_compare_dict``'s "new-only-in-post" handling). Preserving the + non-None value keeps the meaningful write from being clobbered. + """ + if key in result and value is None and result[key] is not None: + return + result[key] = value + + class CustomObject( BookmarksMixin, ChangeLoggingMixin, @@ -427,6 +479,58 @@ class CustomObject( class Meta: abstract = True + @classmethod + def canonicalize_data(cls, data): + """ + Translate stale field-name keys in *data* to this model's current + attribute names. + + Plugin hook called by netbox-branching's ``update_object()`` and + ``ChangeDiff._update_conflicts()``. When a CustomObjectTypeField has + been renamed between when an ObjectChange was recorded and when it is + being replayed/compared, the data dict's keys may refer to the field's + old name while this model class has the field's current name. We walk + the field's rename history (recorded as ObjectChange records on + ``CustomObjectTypeField``) and rewrite each unrecognised key to the + field's current name on this COT. + + When two raw keys map to the same target field (a squash-merge artefact + in which both the old name and the new name appear, the new name often + carrying a sentinel ``None`` from ``deep_compare_dict``'s "new-only-in- + post" treatment), the non-None value wins. When no translation is + possible the original key is preserved so downstream code can still + observe it. + """ + if not data: + return data + + cot_id_str = extract_cot_id_from_model_name(cls.__name__.lower()) + if cot_id_str is None: + return data + cot_id = int(cot_id_str) + try: + cot = CustomObjectType.objects.get(pk=cot_id) + except CustomObjectType.DoesNotExist: + return data + + field_names = {f.name for f in cls._meta.get_fields()} + result = {} + for raw_key, value in data.items(): + # Honour custom_fields → custom_field_data the same way update_object + # used to, so this hook is a true superset of the previous behaviour. + key = 'custom_field_data' if raw_key == 'custom_fields' else raw_key + if key in field_names: + _set_with_collision_preference(result, key, value) + continue + translated = _translate_renamed_field_name(cot, key) + if translated and translated in field_names: + _set_with_collision_preference(result, translated, value) + else: + # Unknown key (e.g. removed field) — preserve raw key so callers + # that inspect the dict for non-field metadata can still see it. + _set_with_collision_preference(result, raw_key, value) + return result + @classmethod def deserialize_object(cls, data, pk=None): """ @@ -473,21 +577,11 @@ def deserialize_object(cls, data, pk=None): # ``column "alpha" does not exist``. # # Building directly off ``fresh_model`` (the context-aware class) keeps - # the field set aligned with the schema we're writing to, and the attr - # translator registered with netbox-branching (see - # branching.translate_renamed_field_attr) lets us map the data dict's - # old field names to the current model's field names where they - # diverge. - - # Try the registered netbox-branching attr translator if available so - # that data dicts carrying old field names are reshaped to the current - # context's field names before deserialization. Falls back gracefully - # when the translator isn't registered (e.g. branching not installed). - try: - from netbox_branching.utilities import _translate_attr # type: ignore[attr-defined] - except ImportError: - def _translate_attr(_inst, attr): # noqa: D401 - return None + # the field set aligned with the schema we're writing to. We then + # canonicalize the data dict via the same hook netbox-branching calls + # from ``update_object``, so the keys map to the model's current field + # names regardless of any renames in history. + canonical = fresh_model.canonicalize_data(data) obj = fresh_model() if pk is not None: @@ -495,29 +589,22 @@ def _translate_attr(_inst, attr): # noqa: D401 m2m_data = {} field_names = {f.name for f in fresh_model._meta.get_fields()} - for raw_attr, value in data.items(): - attr = raw_attr - if attr == 'custom_fields': - attr = 'custom_field_data' + for attr, value in canonical.items(): # Tags via the standard NetBox path (Tag rows are looked up by name). if attr == 'tags' and is_taggable(fresh_model): tag_model = apps.get_model('extras', 'Tag') m2m_data['tags'] = list(tag_model.objects.filter(name__in=value or [])) continue if attr not in field_names: - resolved = _translate_attr(obj, attr) - if resolved and resolved in field_names: - attr = resolved - else: - # Unknown attribute (likely a removed field) — preserve it as - # a Python attribute so downstream code (e.g. - # _deferred_co_field_data) can still see it. - setattr(obj, raw_attr, value) - continue + # Unknown attribute (likely a removed field) — preserve it as a + # Python attribute so downstream code (e.g. _deferred_co_field_data) + # can still see it. + setattr(obj, attr, value) + continue try: f = fresh_model._meta.get_field(attr) except FieldDoesNotExist: - setattr(obj, raw_attr, value) + setattr(obj, attr, value) continue if isinstance(f, ManyToManyField): m2m_data[attr] = value From aaf2bd44637df54bb6322067ee36e10209f78e40 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 19 May 2026 10:23:12 -0700 Subject: [PATCH 043/115] update functio name --- netbox_custom_objects/__init__.py | 4 ++-- netbox_custom_objects/branching.py | 4 ++-- netbox_custom_objects/models.py | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index a9c141eb..0f69473f 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -281,8 +281,8 @@ def ready(self): # Register netbox-branching hooks so its router knows about our # dynamically-generated through models. Guarded so the plugin still # works without netbox-branching installed. Field-rename translation - # is handled by ``CustomObject.canonicalize_data`` on the model itself, - # which netbox-branching invokes from ``update_object`` and + # is handled by ``CustomObject.resolve_field_aliases`` on the model + # itself, which netbox-branching invokes from ``update_object`` and # ``ChangeDiff._update_conflicts`` — no registration required. try: from netbox_branching.utilities import register_branching_resolver diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 740c2bb8..58285149 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -7,8 +7,8 @@ optional — the registration call site in ``__init__.ready()`` is guarded so the plugin still works when netbox-branching is not installed. -Field-rename translation lives on ``CustomObject.canonicalize_data`` (a model -classmethod) and is invoked directly by netbox-branching from +Field-rename translation lives on ``CustomObject.resolve_field_aliases`` (a +model classmethod) and is invoked directly by netbox-branching from ``update_object`` and ``ChangeDiff._update_conflicts``; no registration is required for that path. """ diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 1d8e56b8..64f64da3 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -480,7 +480,7 @@ class Meta: abstract = True @classmethod - def canonicalize_data(cls, data): + def resolve_field_aliases(cls, data): """ Translate stale field-name keys in *data* to this model's current attribute names. @@ -578,10 +578,10 @@ def deserialize_object(cls, data, pk=None): # # Building directly off ``fresh_model`` (the context-aware class) keeps # the field set aligned with the schema we're writing to. We then - # canonicalize the data dict via the same hook netbox-branching calls - # from ``update_object``, so the keys map to the model's current field - # names regardless of any renames in history. - canonical = fresh_model.canonicalize_data(data) + # resolve any aliased keys in the data dict via the same hook + # netbox-branching calls from ``update_object``, so the keys map to the + # model's current field names regardless of any renames in history. + resolved = fresh_model.resolve_field_aliases(data) obj = fresh_model() if pk is not None: @@ -589,7 +589,7 @@ def deserialize_object(cls, data, pk=None): m2m_data = {} field_names = {f.name for f in fresh_model._meta.get_fields()} - for attr, value in canonical.items(): + for attr, value in resolved.items(): # Tags via the standard NetBox path (Tag rows are looked up by name). if attr == 'tags' and is_taggable(fresh_model): tag_model = apps.get_model('extras', 'Tag') From 372ddcc18fab67a6ac59e9c605848bdb93311734 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 19 May 2026 13:46:03 -0700 Subject: [PATCH 044/115] use registration function --- netbox_custom_objects/__init__.py | 19 ++++++++++------ netbox_custom_objects/branching.py | 35 +++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 0f69473f..ba66027b 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -279,15 +279,20 @@ def ready(self): _connect_deferred_data_reset_signals() # Register netbox-branching hooks so its router knows about our - # dynamically-generated through models. Guarded so the plugin still - # works without netbox-branching installed. Field-rename translation - # is handled by ``CustomObject.resolve_field_aliases`` on the model - # itself, which netbox-branching invokes from ``update_object`` and - # ``ChangeDiff._update_conflicts`` — no registration required. + # dynamically-generated through models and can translate field-name + # keys in stored ObjectChange data when fields are renamed at runtime. + # Guarded so the plugin still works without netbox-branching installed. try: - from netbox_branching.utilities import register_branching_resolver - from .branching import supports_branching_resolver + from netbox_branching.utilities import ( + register_branching_resolver, + register_objectchange_field_migrator, + ) + from .branching import ( + objectchange_field_migrator, + supports_branching_resolver, + ) register_branching_resolver(supports_branching_resolver) + register_objectchange_field_migrator(objectchange_field_migrator) except ImportError: pass diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 58285149..e8a410ce 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -3,14 +3,11 @@ These functions plug into extension points exposed by netbox-branching so the plugin can correctly route queries for its dynamically-generated through -models when an active branch is set. Importing netbox-branching is deferred / -optional — the registration call site in ``__init__.ready()`` is guarded so -the plugin still works when netbox-branching is not installed. - -Field-rename translation lives on ``CustomObject.resolve_field_aliases`` (a -model classmethod) and is invoked directly by netbox-branching from -``update_object`` and ``ChangeDiff._update_conflicts``; no registration is -required for that path. +models when an active branch is set, and translate field-name keys in stored +ObjectChange data when fields have been renamed at runtime. Importing +netbox-branching is deferred / optional — the registration call sites in +``__init__.ready()`` are guarded so the plugin still works when +netbox-branching is not installed. """ @@ -39,3 +36,25 @@ def supports_branching_resolver(model): if name.startswith('through_custom_objects_'): return True return None + + +def objectchange_field_migrator(model, data): + """ + Translate stale field-name keys in ``data`` for CustomObject models. + + Returns the translated dict when ``model`` is a generated ``CustomObject`` + subclass; returns ``None`` (defer) otherwise so other plugins' migrators + can run. + + The actual translation logic lives on ``CustomObject.resolve_field_aliases`` + so the same code path can be re-used by ``CustomObject.deserialize_object`` + for CREATE replay; this function is the registration adapter that + netbox-branching invokes. + """ + meta = getattr(model, '_meta', None) + if meta is None or meta.app_label != 'netbox_custom_objects': + return None + resolve = getattr(model, 'resolve_field_aliases', None) + if resolve is None: + return None + return resolve(data) From a01f9c8c855cc7a2304baa13d738aeb7be818531 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 20 May 2026 10:40:31 -0700 Subject: [PATCH 045/115] cleanup --- netbox_custom_objects/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 64f64da3..f3c23082 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -2919,10 +2919,10 @@ def save(self, *args, **kwargs): # When the field is renamed, update ObjectChange / ChangeDiff JSON keys so # historical audit records and branch diffs stay consistent with the new - # name. Combined with the netbox-branching attr translator registered - # in branching.translate_renamed_field_attr, this lets later replays - # (whether iterative undo or squash undo) resolve any data key — old or - # new name — to the field's current name on the model. + # name. Combined with the netbox-branching ObjectChange field migrator + # registered in branching.objectchange_field_migrator, this lets later + # replays (whether iterative undo or squash undo) resolve any data key — + # old or new name — to the field's current name on the model. if ( not self._state.adding and not self.is_polymorphic From 91316ed2d92bce333b630d8cfa68cd24af8b2349 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 09:18:33 -0700 Subject: [PATCH 046/115] add version checks to check framework --- docs/branching.md | 9 ++++ netbox_custom_objects/__init__.py | 5 +++ netbox_custom_objects/checks.py | 68 +++++++++++++++++++++++++++++++ pyproject.toml | 5 +++ 4 files changed, 87 insertions(+) create mode 100644 netbox_custom_objects/checks.py diff --git a/docs/branching.md b/docs/branching.md index 416fb2b6..881c1ed5 100644 --- a/docs/branching.md +++ b/docs/branching.md @@ -2,6 +2,15 @@ As of version 0.4.0 Custom Objects is _compatible_ with NetBox Branching, but not fully supported. This means that users can safely use both plugins together, but there are some caveats to be aware of. See below to learn how each of the Custom Objects models interacts with NetBox Branching. +## Version Requirements + +When using Custom Objects together with NetBox Branching, the following minimum versions are required: + +- NetBox >= 4.6.2 +- netbox-branching >= 1.1.0 + +These requirements are only enforced when `netbox_branching` is present in `PLUGINS`. If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. A Django system check (`netbox_custom_objects.E001` / `E002`) will fail at startup if the combination is misconfigured. + > [!NOTE] > We are working towards full support for Custom Objects on branches. Keep an eye on the GitHub issues for updates ahead of future releases. diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index ba66027b..bf1d5d16 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -261,6 +261,11 @@ def ready(self): # model is registered (must happen exactly once, before get_model() runs). install_clear_cache_suppressor() + # Register Django system checks (e.g. conditional netbox-branching + # version requirements). Importing the module is what triggers the + # @register decorator. + from . import checks # noqa: F401 + from .models import CustomObjectType from netbox_custom_objects.api.serializers import get_serializer_class diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py new file mode 100644 index 00000000..a80b549a --- /dev/null +++ b/netbox_custom_objects/checks.py @@ -0,0 +1,68 @@ +""" +System checks for netbox-custom-objects. + +These run as part of Django's check framework (``manage.py check`` and any +command that invokes it — runserver, migrate, test, etc.). Their purpose is +to enforce *conditional* compatibility requirements that PluginConfig's +unconditional ``min_version`` / ``max_version`` cannot express: when +netbox-branching is installed alongside this plugin, the supported NetBox +and netbox-branching version ranges narrow. Users who never enable +branching keep the broader compatibility window. +""" + +from importlib.metadata import PackageNotFoundError, version as _pkg_version + +from django.apps import apps +from django.conf import settings +from django.core.checks import Error, register +from packaging.version import InvalidVersion, Version + + +# Version floors that apply only when netbox-branching is installed. +# Users without branching are governed by PluginConfig.min_version instead. +REQUIRED_NETBOX_VERSION = '4.6.2' +REQUIRED_BRANCHING_VERSION = '1.1.0' + + +@register() +def check_branching_compatibility(app_configs, **kwargs): + """ + If netbox-branching is installed, enforce the version floors required for + custom objects to integrate with it safely. Returns no errors when + netbox-branching is not installed. + """ + if not apps.is_installed('netbox_branching'): + return [] + + errors = [] + + try: + netbox_version = Version(settings.RELEASE.version) + if netbox_version < Version(REQUIRED_NETBOX_VERSION): + errors.append(Error( + f'netbox-custom-objects requires NetBox >= {REQUIRED_NETBOX_VERSION} ' + f'when netbox-branching is installed (detected {netbox_version}).', + hint='Upgrade NetBox, or remove netbox-branching from PLUGINS ' + 'if you do not need branching support for custom objects.', + id='netbox_custom_objects.E001', + )) + except (AttributeError, InvalidVersion): + # settings.RELEASE missing or unparseable — let other checks surface it. + pass + + try: + branching_version = Version(_pkg_version('netbox-branching')) + if branching_version < Version(REQUIRED_BRANCHING_VERSION): + errors.append(Error( + f'netbox-custom-objects requires netbox-branching >= ' + f'{REQUIRED_BRANCHING_VERSION} (detected {branching_version}).', + hint=f'Upgrade with: pip install "netbox-branching>={REQUIRED_BRANCHING_VERSION}"', + id='netbox_custom_objects.E002', + )) + except (PackageNotFoundError, InvalidVersion): + # Installed as an app but not discoverable via importlib.metadata + # (e.g. editable install with a non-standard dist-info). Skip the + # version pin rather than emit a confusing error. + pass + + return errors diff --git a/pyproject.toml b/pyproject.toml index 0b23b65b..ed662679 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,11 @@ dependencies = [ [project.optional-dependencies] dev = ["check-manifest", "mkdocs", "mkdocs-material", "ruff"] test = ["coverage", "pytest", "pytest-cov"] +# Install with `pip install "netboxlabs-netbox-custom-objects[branching]"` when +# pairing this plugin with netbox-branching. Note: this extra also implies a +# stricter NetBox version requirement (>= 4.6.2), which is enforced at startup +# by netbox_custom_objects.checks rather than declared here. +branching = ["netbox-branching>=1.1.0"] [project.urls] "Homepage" = "https://netboxlabs.com/" From caf09cb0554a465321a7994f34cee3e33b554343 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 10:06:19 -0700 Subject: [PATCH 047/115] cleanup --- netbox_custom_objects/field_types.py | 12 +++- netbox_custom_objects/models.py | 92 +++++++++++++++++++++++++--- 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index f604f375..a30d2ea2 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -10,7 +10,7 @@ from django.contrib.postgres.fields import ArrayField from django.core.exceptions import FieldDoesNotExist from django.core.validators import RegexValidator -from django.db import connection, models +from django.db import connection, models, router from django.db.models.fields.related import ForeignKey, ManyToManyDescriptor from django.db.models.manager import Manager from django.db.models.signals import m2m_changed @@ -927,9 +927,17 @@ def _fire_m2m_changed(self, action, pk_set): ``add`` / ``remove`` / ``set`` do) leaves the m2m write invisible to ``handle_changed_object`` — which means a merge or sync replays zero through-table rows for CustomObject M2M fields. + + ``using`` must reflect the alias the through-table write actually + happened on so that branch-aware receivers (and Django's own + router-based dispatch) route subsequent queries to the same schema. + The through models are registered as branchable via + ``supports_branching_resolver``, so ``router.db_for_write`` returns + the active branch's connection alias inside a branch context. """ if not pk_set: return + db = router.db_for_write(self.through, instance=self.instance) m2m_changed.send( sender=self.through, instance=self.instance, @@ -937,7 +945,7 @@ def _fire_m2m_changed(self, action, pk_set): reverse=False, model=getattr(self.through._meta.get_field('target').remote_field, 'model', None), pk_set=pk_set, - using='default', + using=db, ) @staticmethod diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index f3c23082..26672feb 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -82,9 +82,16 @@ class UniquenessConstraintTestError(Exception): pass -def _table_exists(table_name): - """Return True if *table_name* exists in the current database.""" - return table_name in connection.introspection.table_names() +def _table_exists(table_name, conn=None): + """Return True if *table_name* exists in the database reachable via *conn*. + + Defaults to the global ``connection`` (main schema). When the caller is + operating inside a branch context, pass the branch's connection so the + lookup runs against the active branch's PostgreSQL schema. + """ + if conn is None: + conn = connection + return table_name in conn.introspection.table_names() USER_TABLE_DATABASE_NAME_PREFIX = "custom_objects_" @@ -933,6 +940,37 @@ def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): # Clear Django apps registry cache to ensure newly created models are recognized apps.get_models.cache_clear() + @staticmethod + def _realign_through_models(model): + """ + Re-point every through-model's ``source`` FK at *model*. + + The dynamic M2M through models live in Django's global ``apps`` + registry as one shared class per through-table name, but each + (cot_id, branch_id) cache key holds a different parent CO model + class. ``_after_model_generation`` mutates the through's + ``source.remote_field.model`` to the CO class being built; on a + subsequent cache-hit return for a *different* context (e.g. main + after a branch call) the through still points at the previous + context's class, which breaks Django's collector during cascade + delete ("Cannot query 'X': Must be 'TableYModel' instance."). + + Calling this on every get_model() return path — under + ``_global_lock`` — guarantees that at the moment a caller sees a + model, the through it references in the apps registry agrees on + the source class. This does **not** make the registry safe for + concurrent cross-context use across threads; that's an + architectural limitation of sharing through models globally and + would require per-(cot, branch) through registration to fix. + """ + for through_model in getattr(model, '_through_models', None) or (): + try: + source_field = through_model._meta.get_field('source') + except FieldDoesNotExist: + continue + source_field.remote_field.model = model + source_field.related_model = model + @classmethod def get_cached_model(cls, custom_object_type_id, branch_id=None): """ @@ -1306,6 +1344,11 @@ def get_model( # would fail when post_save's search-cache handler reads # field names that don't exist on the active model class. self.register_custom_object_search_index(model) + # Through-models in the global apps registry may still be + # pointing at a different context's CO class — realign + # while we hold ``_global_lock`` so callers see a + # consistent (model, through_model.source) pair. + self._realign_through_models(model) return model else: self.clear_model_cache(self.id) @@ -1635,8 +1678,11 @@ def delete(self, *args, **kwargs): with schema_conn.schema_editor() as schema_editor: # Drop through tables before the main table (they have FKs pointing to it). + # Pass schema_conn so existence checks run against the active branch's + # schema, not main's — otherwise inside a branch we'd skip drops that + # need to happen (or attempt drops on tables that don't exist there). for through_model in getattr(model, '_through_models', []): - if _table_exists(through_model._meta.db_table): + if _table_exists(through_model._meta.db_table, conn=schema_conn): schema_editor.delete_model(through_model) schema_editor.delete_model(model) @@ -1700,6 +1746,13 @@ def _rename_objectchange_field_key(fi, old_name, new_name): cot = fi.custom_object_type model = cot.get_model() ct = ContentType.objects.get_for_model(model) + # core.ObjectChange is branched by netbox-branching (migrations are allowed + # on the branch schema, and read routing sends ObjectChange queries to the + # active branch — see netbox_branching.database.BranchAwareRouter). Using + # _get_schema_connection() therefore updates the branch's copy of + # core_objectchange, keeping branch-context history consistent with the + # rename. Main's copy is updated when the rename is later merged and this + # function runs again in main context. conn = _get_schema_connection() oc_sql = ( @@ -1707,11 +1760,25 @@ def _rename_objectchange_field_key(fi, old_name, new_name): 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' 'WHERE changed_object_type_id = %s AND {col} ? %s' ) - with conn.cursor() as cursor: - for json_col in ('prechange_data', 'postchange_data'): - cursor.execute(oc_sql.format(col=json_col), [old_name, new_name, old_name, ct.id, old_name]) + # Wrap in a savepoint so a ProgrammingError (e.g. core_objectchange + # unexpectedly missing from the active schema) doesn't poison the outer + # transaction. Mirrors the ChangeDiff guard below. + try: + with transaction.atomic(using=conn.alias): + with conn.cursor() as cursor: + for json_col in ('prechange_data', 'postchange_data'): + cursor.execute( + oc_sql.format(col=json_col), + [old_name, new_name, old_name, ct.id, old_name], + ) + except ProgrammingError: + logger.warning( + '_rename_objectchange_field_key: ObjectChange schema mismatch; ' + 'audit data for %r may not reflect rename to %r', + old_name, new_name, exc_info=True, + ) - logger.debug('_rename_objectchange_field_key: %r → %r for %s', old_name, new_name, ct) + logger.debug('_rename_objectchange_field_key: %r -> %r for %s', old_name, new_name, ct) try: from netbox_branching.models import ChangeDiff # noqa: F401 — presence check only @@ -2197,9 +2264,14 @@ def clean(self): old_field = field_type.get_model_field(self.original) old_field.contribute_to_class(model, self._original_name) + # Route the probe through the branch's connection so the ALTER + # TABLE runs in the active schema. Using the default connection + # here would either probe main's table from a branch context or + # fail outright if the table only exists in the branch schema. + probe_conn = _get_schema_connection() try: - with transaction.atomic(): - with connection.schema_editor() as test_schema_editor: + with transaction.atomic(using=probe_conn.alias): + with probe_conn.schema_editor() as test_schema_editor: test_schema_editor.alter_field(model, old_field, model_field) # If we get here, the constraint was applied successfully # Now raise a custom exception to rollback the test transaction From c393f4db575c0c968ad49764ae895ce9fdcb8194 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 10:32:43 -0700 Subject: [PATCH 048/115] cleanup --- netbox_custom_objects/field_types.py | 47 ++- netbox_custom_objects/models.py | 272 +++++++++++++----- netbox_custom_objects/tests/base.py | 11 + netbox_custom_objects/tests/test_branching.py | 23 +- 4 files changed, 257 insertions(+), 96 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index a30d2ea2..1cb60f64 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -179,7 +179,7 @@ def _get_related_content_type(self, field): def after_model_generation(self, instance, model, field_name): ... - def create_m2m_table(self, instance, model, field_name): ... + def create_m2m_table(self, instance, model, field_name, schema_conn=None): ... class TextFieldType(FieldType): @@ -928,6 +928,10 @@ def _fire_m2m_changed(self, action, pk_set): to ``handle_changed_object`` — which means a merge or sync replays zero through-table rows for CustomObject M2M fields. + Both ``pre_*`` and ``post_*`` actions are emitted, matching Django's + standard ManyRelatedManager flow. ``pre_clear`` is the one place + Django passes ``pk_set=None`` — callers should pass ``None`` there. + ``using`` must reflect the alias the through-table write actually happened on so that branch-aware receivers (and Django's own router-based dispatch) route subsequent queries to the same schema. @@ -935,7 +939,9 @@ def _fire_m2m_changed(self, action, pk_set): ``supports_branching_resolver``, so ``router.db_for_write`` returns the active branch's connection alias inside a branch context. """ - if not pk_set: + # pk_set is None for *_clear actions; an empty set means "no objects + # changed" so skip the signal entirely. + if pk_set is not None and not pk_set: return db = router.db_for_write(self.through, instance=self.instance) m2m_changed.send( @@ -959,9 +965,15 @@ def _resolve_pk(obj): return obj.pk if hasattr(obj, 'pk') else obj def add(self, *objs): + # Send pre_add with the candidate pk_set before any write so + # receivers (e.g. validation hooks) get a chance to abort. An empty + # candidate set is a no-op. + candidate_pks = {self._resolve_pk(obj) for obj in objs} + if not candidate_pks: + return + self._fire_m2m_changed('pre_add', candidate_pks) added = set() - for obj in objs: - pk = self._resolve_pk(obj) + for pk in candidate_pks: _, created = self.through.objects.get_or_create( source_id=self.instance.pk, target_id=pk ) @@ -971,9 +983,12 @@ def add(self, *objs): self._fire_m2m_changed('post_add', added) def remove(self, *objs): + candidate_pks = {self._resolve_pk(obj) for obj in objs} + if not candidate_pks: + return + self._fire_m2m_changed('pre_remove', candidate_pks) removed = set() - for obj in objs: - pk = self._resolve_pk(obj) + for pk in candidate_pks: n, _ = self.through.objects.filter( source_id=self.instance.pk, target_id=pk ).delete() @@ -987,9 +1002,13 @@ def clear(self): self.through.objects.filter(source_id=self.instance.pk) .values_list('target_id', flat=True) ) - if existing: - self.through.objects.filter(source_id=self.instance.pk).delete() - self._fire_m2m_changed('post_clear', existing) + if not existing: + return + # Django sends pre_clear with pk_set=None (it doesn't know yet which + # rows will go). Match that contract. + self._fire_m2m_changed('pre_clear', None) + self.through.objects.filter(source_id=self.instance.pk).delete() + self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): objs = tuple(objs) # force evaluation before any mutation @@ -1005,6 +1024,7 @@ def set(self, objs, clear=False): # Remove relationships no longer in the target set to_remove = old_pks - new_pks if to_remove: + self._fire_m2m_changed('pre_remove', to_remove) self.through.objects.filter( source_id=self.instance.pk, target_id__in=to_remove, @@ -1012,11 +1032,12 @@ def set(self, objs, clear=False): self._fire_m2m_changed('post_remove', to_remove) # Add only genuinely new relationships to_add = new_pks - old_pks - for pk in to_add: - self.through.objects.get_or_create( - source_id=self.instance.pk, target_id=pk - ) if to_add: + self._fire_m2m_changed('pre_add', to_add) + for pk in to_add: + self.through.objects.get_or_create( + source_id=self.instance.pk, target_id=pk + ) self._fire_m2m_changed('post_add', to_add) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 26672feb..9731dfbe 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -123,6 +123,32 @@ def _get_schema_connection(): return connection +def _historical_names_for_field(field_pk): + """ + Return the set of all names this CustomObjectTypeField has ever held, + derived from its own ObjectChange UPDATE history. + + Used when applying deferred CO field data after a squash merge so that a + deferred entry recorded under an *old* field name (because the CO CREATE + was replayed before the rename) still matches the field's current name. + """ + names = set() + rows = ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + changed_object_id=field_pk, + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).values_list('prechange_data', 'postchange_data') + for pre, post in rows: + for blob in (pre, post): + if not blob: + continue + n = blob.get('name') + if n: + names.add(n) + return names + + def _apply_deferred_co_field(field_instance): """ Apply any deferred CO field values after a column is added to the DB. @@ -139,6 +165,13 @@ def _apply_deferred_co_field(field_instance): is ``{name}_id`` — this function maps accordingly. For TYPE_MULTIOBJECT fields there is no column on the main table, so they are skipped entirely. + + Rename awareness: in a squash merge the CO CREATE for a renamed field + arrives with the *old* field name in ``data`` (the name recorded in the + ObjectChange at write time), while this function is invoked when the + field is being created in the target schema under its *new* name. + We accept any historical name for this field — derived from the field's + own ObjectChange UPDATE history — as a match. """ # No deferred data at all — fast path. deferred = _deferred_co_field_data.get() @@ -156,22 +189,30 @@ def _apply_deferred_co_field(field_instance): return # For TYPE_OBJECT the data key is the field name but the DB column ends with _id. - data_key = field_instance.name if field_instance.type == CustomFieldTypeChoices.TYPE_OBJECT: col_name = f'{field_instance.name}_id' else: col_name = field_instance.name + # Candidate data keys: current name + all historical names. pk may be None + # when called before the field row is persisted; in that case we skip the + # history lookup and only match by current name. + candidate_keys = {field_instance.name} + if field_instance.pk is not None: + candidate_keys.update(_historical_names_for_field(field_instance.pk)) + schema_conn = _get_schema_connection() with schema_conn.cursor() as cursor: for co_pk, entry in per_table.items(): - # Distinguish "key absent" from "key present with NULL". An explicit - # None is a legitimate write and must reach the column; only a - # missing key should be skipped. - if data_key not in entry['data']: + data = entry['data'] + # Find any candidate key actually present in this entry. + # Distinguish "key absent" from "key present with NULL": an + # explicit None is a legitimate write and must reach the column. + matched = next((k for k in candidate_keys if k in data), None) + if matched is None: continue - value = entry['data'][data_key] + value = data[matched] # table_name comes from get_database_table_name() (controlled by our # code) and col_name from field.name, which is validated by the # ^[a-z0-9_]+$ regex — no double-quote characters are possible, so @@ -181,11 +222,13 @@ def _apply_deferred_co_field(field_instance): [value, co_pk], ) - # Remove the consumed key from each entry so that processed field data does - # not persist in the ContextVar beyond its useful lifetime (e.g. on a retry - # after a partial failure, stale data from a previous attempt is avoided). + # Remove the consumed keys (any of the candidates) from each entry so that + # processed field data does not persist in the ContextVar beyond its + # useful lifetime (e.g. on a retry after a partial failure, stale data + # from a previous attempt is avoided). for entry in per_table.values(): - entry['data'].pop(data_key, None) + for k in candidate_keys: + entry['data'].pop(k, None) # Prune entries whose data dict is now exhausted. exhausted = [pk for pk, entry in per_table.items() if not entry['data']] @@ -393,7 +436,7 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist schema_editor.alter_field(model, old_mf, new_mf) -def _translate_renamed_field_name(cot, attr): +def _translate_renamed_field_name(cot, attr, rename_map=None): """ Resolve ``attr`` to a CustomObjectTypeField's *current* name on *cot* by walking the field's rename history. @@ -409,7 +452,14 @@ def _translate_renamed_field_name(cot, attr): multiple fields' rename history) because picking arbitrarily would silently overwrite the wrong column; the caller then preserves the raw key. This case is vanishingly rare for normal use. + + Pass ``rename_map`` (an old_name → current_name dict built via + ``_build_rename_map``) to skip the DB query. ``resolve_field_aliases`` + constructs that map once per call so a payload with many renamed keys + incurs a single ObjectChange scan instead of one per key. """ + if rename_map is not None: + return rename_map.get(attr) candidate_pks = set( ObjectChange.objects.filter( changed_object_type__app_label='netbox_custom_objects', @@ -429,6 +479,54 @@ def _translate_renamed_field_name(cot, attr): return None +def _build_rename_map(cot, attrs): + """ + Return ``{old_name: current_name}`` for those entries in *attrs* that + resolve to exactly one of *cot*'s fields via rename history. + + One ObjectChange query covers all candidates. Ambiguous mappings (same + historical name appearing in multiple fields' history) are omitted so the + caller falls back to preserving the raw key — matching the + abstain-on-ambiguity behaviour of ``_translate_renamed_field_name``. + """ + attrs = [a for a in attrs if a] + if not attrs: + return {} + rows = ObjectChange.objects.filter( + changed_object_type__app_label='netbox_custom_objects', + changed_object_type__model='customobjecttypefield', + action=ObjectChangeActionChoices.ACTION_UPDATE, + ).filter( + Q(postchange_data__name__in=attrs) | Q(prechange_data__name__in=attrs) + ).values_list('changed_object_id', 'prechange_data', 'postchange_data') + + # attr → {field_pks that have this name anywhere in their history} + attr_to_field_pks: dict[str, set[int]] = {} + for field_pk, pre, post in rows: + for blob in (pre, post): + if not blob: + continue + name = blob.get('name') + if name in attrs: + attr_to_field_pks.setdefault(name, set()).add(field_pk) + + if not attr_to_field_pks: + return {} + + # Resolve the field pks we collected to their current names in one query. + pk_to_name = dict( + cot.fields.filter( + pk__in={pk for pks in attr_to_field_pks.values() for pk in pks} + ).values_list('pk', 'name') + ) + result = {} + for attr, field_pks in attr_to_field_pks.items(): + matched = [pk_to_name[pk] for pk in field_pks if pk in pk_to_name] + if len(matched) == 1: + result[attr] = matched[0] + return result + + def _set_with_collision_preference(result, key, value): """ Assign ``result[key] = value``; on collision prefer the non-None value. @@ -521,6 +619,16 @@ def resolve_field_aliases(cls, data): return data field_names = {f.name for f in cls._meta.get_fields()} + + # Collect the keys that don't match a current field name so we can do + # one batched ObjectChange query instead of one per unknown key. + unknown_keys = [] + for raw_key in data: + key = 'custom_field_data' if raw_key == 'custom_fields' else raw_key + if key not in field_names: + unknown_keys.append(key) + rename_map = _build_rename_map(cot, unknown_keys) if unknown_keys else {} + result = {} for raw_key, value in data.items(): # Honour custom_fields → custom_field_data the same way update_object @@ -529,7 +637,7 @@ def resolve_field_aliases(cls, data): if key in field_names: _set_with_collision_preference(result, key, value) continue - translated = _translate_renamed_field_name(cot, key) + translated = _translate_renamed_field_name(cot, key, rename_map=rename_map) if translated and translated in field_names: _set_with_collision_preference(result, translated, value) else: @@ -1720,8 +1828,10 @@ def custom_object_type_post_save_handler(sender, instance, created, **kwargs): app_label=APP_LABEL, model=content_type_name ) - # Snapshot before modifying so change logging records a correct pre-state. - # Without this, diff()['pre'] would set all fields to None during branch revert. + # The snapshot here is for the *second* save (object_type assignment) + # below — not for the create that just fired this handler. Without it, + # the second save's ObjectChange would mark every field as changed + # instead of just the object_type FK. instance.snapshot() instance.object_type = ct instance.save() @@ -2964,74 +3074,82 @@ def save(self, *args, **kwargs): field_type = FIELD_TYPE_CLASS[self.type]() model = self.custom_object_type.get_model() - with schema_conn.schema_editor() as schema_editor: - if self._state.adding: - if self.is_polymorphic: - # Polymorphic Object: add content_type + object_id columns + index. - # Polymorphic MultiObject: create through table with content_type + object_id. - if self.type == CustomFieldTypeChoices.TYPE_OBJECT: - field_type.add_polymorphic_object_columns(self, model, schema_editor) - elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - field_type.create_polymorphic_m2m_table(self, model, schema_editor) + # Wrap the schema mutation, audit-key rewrite, cache_timestamp bump, and + # parent save() in a single atomic so a failure between the DDL and the + # field row save can't leave audit data rewritten but the field record + # un-persisted (or vice versa). Nests inside the schema_id allocation + # block above as a savepoint. + with transaction.atomic(using=schema_conn.alias): + with schema_conn.schema_editor() as schema_editor: + if self._state.adding: + if self.is_polymorphic: + # Polymorphic Object: add content_type + object_id columns + index. + # Polymorphic MultiObject: create through table with content_type + object_id. + if self.type == CustomFieldTypeChoices.TYPE_OBJECT: + field_type.add_polymorphic_object_columns(self, model, schema_editor) + elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + field_type.create_polymorphic_m2m_table(self, model, schema_editor) + else: + _schema_add_field(self, model, schema_editor, schema_conn) + _apply_deferred_co_field(self) else: - _schema_add_field(self, model, schema_editor, schema_conn) - _apply_deferred_co_field(self) - else: - # Polymorphic fields: renames and type changes are rejected by clean(). - # Non-schema attributes (label, description, …) may still change here. - # If clean() was bypassed and a rename slipped through, raise rather - # than silently leaving DB columns / through table out of sync. - if self.is_polymorphic or self._original_is_polymorphic: - if self.name != self._original_name: - raise ValidationError( - {"name": _("Cannot rename a polymorphic field after creation.")} - ) + # Polymorphic fields: renames and type changes are rejected by clean(). + # Non-schema attributes (label, description, …) may still change here. + # If clean() was bypassed and a rename slipped through, raise rather + # than silently leaving DB columns / through table out of sync. + if self.is_polymorphic or self._original_is_polymorphic: + if self.name != self._original_name: + raise ValidationError( + {"name": _("Cannot rename a polymorphic field after creation.")} + ) + else: + _schema_alter_field(self.original, self, model, schema_editor, schema_conn) + + # When the field is renamed, update ObjectChange / ChangeDiff JSON keys so + # historical audit records and branch diffs stay consistent with the new + # name. Combined with the netbox-branching ObjectChange field migrator + # registered in branching.objectchange_field_migrator, this lets later + # replays (whether iterative undo or squash undo) resolve any data key — + # old or new name — to the field's current name on the model. + if ( + not self._state.adding + and not self.is_polymorphic + and self._original_name != self.name + ): + _rename_objectchange_field_key(self, self._original_name, self.name) + + # Ensure FK constraints are properly created for OBJECT fields. Decide + # whether one is needed inside the atomic so any in-progress rollback + # also discards the decision. + should_ensure_fk = False + if self.type == CustomFieldTypeChoices.TYPE_OBJECT and not self.is_polymorphic: + if self._state.adding: + should_ensure_fk = True else: - _schema_alter_field(self.original, self, model, schema_editor, schema_conn) - - # When the field is renamed, update ObjectChange / ChangeDiff JSON keys so - # historical audit records and branch diffs stay consistent with the new - # name. Combined with the netbox-branching ObjectChange field migrator - # registered in branching.objectchange_field_migrator, this lets later - # replays (whether iterative undo or squash undo) resolve any data key — - # old or new name — to the field's current name on the model. - if ( - not self._state.adding - and not self.is_polymorphic - and self._original_name != self.name - ): - _rename_objectchange_field_key(self, self._original_name, self.name) - - # Ensure FK constraints are properly created for OBJECT fields - should_ensure_fk = False - if self.type == CustomFieldTypeChoices.TYPE_OBJECT and not self.is_polymorphic: - if self._state.adding: - should_ensure_fk = True - else: - type_changed_to_object = ( - self._original_type != CustomFieldTypeChoices.TYPE_OBJECT - and self.type == CustomFieldTypeChoices.TYPE_OBJECT - ) - related_object_changed = ( - self._original_type == CustomFieldTypeChoices.TYPE_OBJECT - and self.related_object_type_id != self._original_related_object_type_id - ) - on_delete_changed = ( - self._original_type == CustomFieldTypeChoices.TYPE_OBJECT - and self.on_delete_behavior != self._original_on_delete_behavior - ) - should_ensure_fk = type_changed_to_object or related_object_changed or on_delete_changed + type_changed_to_object = ( + self._original_type != CustomFieldTypeChoices.TYPE_OBJECT + and self.type == CustomFieldTypeChoices.TYPE_OBJECT + ) + related_object_changed = ( + self._original_type == CustomFieldTypeChoices.TYPE_OBJECT + and self.related_object_type_id != self._original_related_object_type_id + ) + on_delete_changed = ( + self._original_type == CustomFieldTypeChoices.TYPE_OBJECT + and self.on_delete_behavior != self._original_on_delete_behavior + ) + should_ensure_fk = type_changed_to_object or related_object_changed or on_delete_changed - # Clear and refresh the model cache for this CustomObjectType when a field is modified - self.custom_object_type.clear_model_cache(self.custom_object_type.id) + # Clear and refresh the model cache for this CustomObjectType when a field is modified + self.custom_object_type.clear_model_cache(self.custom_object_type.id) - # Update parent's cache_timestamp to invalidate cache across all workers. - # snapshot() must be called first so that change logging has a correct pre-state; - # without it, diff()['pre'] would set ALL fields to None during branch revert. - self.custom_object_type.snapshot() - self.custom_object_type.save(update_fields=['cache_timestamp']) + # Update parent's cache_timestamp to invalidate cache across all workers. + # snapshot() must be called first so that change logging has a correct pre-state; + # without it, diff()['pre'] would set ALL fields to None during branch revert. + self.custom_object_type.snapshot() + self.custom_object_type.save(update_fields=['cache_timestamp']) - super().save(*args, **kwargs) + super().save(*args, **kwargs) # Ensure FK constraints AFTER the transaction commits to avoid "pending trigger events" errors if should_ensure_fk: diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 8e27c4e7..1f11ff37 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -49,6 +49,17 @@ def _purge_stale_generated_models(): for name in stale: django_apps.all_models[APP_LABEL].pop(name, None) if stale: + # Expire reverse-relation caches so any other app whose + # ``related_objects`` already pointed at the now-removed dynamic + # models gets rebuilt on next access. ``apps.clear_cache()`` alone + # clears the apps registry's own cache, but per-model + # ``_meta._relation_tree`` snapshots taken by Django outside this + # plugin survive — they need explicit expiry. + from django.contrib.contenttypes.models import ContentType # noqa: PLC0415 + ContentType._meta._expire_cache(forward=False) + for app_models in django_apps.all_models.values(): + for model in app_models.values(): + model._meta._expire_cache(forward=False) django_apps.clear_cache() diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 88ffb836..ebc0866f 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -11,6 +11,7 @@ """ import datetime import decimal +import os import time import unittest import uuid @@ -46,19 +47,29 @@ def _make_request(user): return request -def _provision_branch(name, merge_strategy, user): - """Create and wait for a branch to reach READY status (up to 30 s).""" +# Provisioning timeout for branch tests. Defaults to 30 s, which is generous +# for local laptops but can be tight on shared CI runners — override via the +# ``NETBOX_CO_BRANCH_PROVISION_TIMEOUT`` env var (seconds) when CI flakes. +BRANCH_PROVISION_TIMEOUT = float( + os.environ.get('NETBOX_CO_BRANCH_PROVISION_TIMEOUT', '30') +) + + +def _provision_branch(name, merge_strategy, user, timeout=None): + """Create and wait for a branch to reach READY status.""" + if timeout is None: + timeout = BRANCH_PROVISION_TIMEOUT branch = Branch(name=name, merge_strategy=merge_strategy) branch.save(provision=False) branch.provision(user=user) - deadline = time.time() + 30 + deadline = time.time() + timeout while time.time() < deadline: branch.refresh_from_db() if branch.status == BranchStatusChoices.READY: return branch time.sleep(0.1) raise TimeoutError( - f'Branch {name!r} did not reach READY within 30 s ' + f'Branch {name!r} did not reach READY within {timeout:.0f} s ' f'(status={branch.status!r})' ) @@ -1403,7 +1414,7 @@ def test_sequential_renames_both_sides_sync(self): co = cot.get_model().objects.create(alpha='original') co_pk = co.pk - branch = _provision_branch('Seq Sync Branch', 'iterative', self.user) + branch = _provision_branch('Seq Sync Branch', self.MERGE_STRATEGY, self.user) branch_request = _make_request(self.user) # ── branch: alpha → beta → gamma ────────────────────────────────── @@ -1491,7 +1502,7 @@ def test_sequential_renames_both_sides_merge(self): co = cot.get_model().objects.create(alpha='original') co_pk = co.pk - branch = _provision_branch('Seq Merge Conflict Branch', 'iterative', self.user) + branch = _provision_branch('Seq Merge Conflict Branch', self.MERGE_STRATEGY, self.user) branch_request = _make_request(self.user) # ── branch: alpha → beta → gamma ────────────────────────────────── From d72df778ca696fdead45ea441dfed16a8c141a00 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 10:46:08 -0700 Subject: [PATCH 049/115] cleanup --- netbox_custom_objects/models.py | 25 ++-- netbox_custom_objects/tests/base.py | 25 +++- netbox_custom_objects/tests/test_branching.py | 128 ++++++++---------- 3 files changed, 91 insertions(+), 87 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 9731dfbe..f5d67e16 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -754,18 +754,27 @@ def save(self, using=None, **_kwargs): # list of related PKs. Skipped quietly when the through table # or its columns aren't present yet — squash ordering can defer # field creation, and the through-table appears when the field's - # own CREATE ObjectChange is later applied. We narrow the - # except to the failure modes that signature: ``AttributeError`` - # if the manager descriptor isn't bound to ``obj`` yet, and - # ``ProgrammingError``/``OperationalError`` if the through - # table/column is missing in the active schema. + # own CREATE ObjectChange is later applied. + # + # AttributeError is checked up front with ``hasattr`` so the + # except below stays scoped to genuine DB-state mismatches. + # That way a typo or missing method on a manager surfaces as + # an exception instead of being silently swallowed at DEBUG. for accessor, related_pks in m2m_data.items(): + if not hasattr(obj, accessor): + logger.debug( + 'deserialize_object: deferred M2M %r on %s pk=%s ' + '(descriptor not yet bound to model)', + accessor, table_name, obj_pk, + ) + continue + manager = getattr(obj, accessor) try: - manager = getattr(obj, accessor) manager.set(related_pks) - except (AttributeError, ProgrammingError, OperationalError): + except (ProgrammingError, OperationalError): logger.debug( - 'deserialize_object: deferred M2M %r on %s pk=%s', + 'deserialize_object: deferred M2M %r on %s pk=%s ' + '(through table/column not yet present in schema)', accessor, table_name, obj_pk, exc_info=True, ) # Register full data for deferred column updates (squash ordering fix). diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 1f11ff37..510a4376 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -1,4 +1,6 @@ # Test utilities for netbox_custom_objects plugin +import logging + from django.apps import apps as django_apps from django.contrib.contenttypes.management import create_contenttypes from django.db import connection @@ -11,6 +13,8 @@ from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField, _deferred_co_field_data +logger = logging.getLogger(__name__) + def _recreate_contenttypes(): """Recreate ContentType rows for all installed apps using get_or_create. @@ -122,9 +126,10 @@ def _drop_dynamic_tables(): all_tables = connection.introspection.table_names() dynamic = [t for t in all_tables if t.startswith(_DYNAMIC_TABLE_PREFIX)] if dynamic: + quote = connection.ops.quote_name with connection.cursor() as cursor: for table in dynamic: - cursor.execute(f'DROP TABLE IF EXISTS "{table}" CASCADE') + cursor.execute(f'DROP TABLE IF EXISTS {quote(table)} CASCADE') # Step 5 — rebuild the app registry cache now that both the stale model # entries (step 2) and the stale COT rows (step 3) are gone. get_models() @@ -151,10 +156,13 @@ def _reset_netbox_request_context(): return current_request.set(None) events_queue.set({}) + # ``query_cache`` is a ContextVar in current NetBox; in older releases it + # was a thread-local without ``.set()``. Catch AttributeError narrowly so + # an unrelated bug surfaces instead of being swallowed silently. try: query_cache.set(None) - except Exception: - pass + except AttributeError: + logger.debug('netbox.context.query_cache has no .set(); skipping reset') def create_api_token(user): @@ -214,12 +222,17 @@ def tearDown(self): # Defensive reset — see setUp for rationale. Belt-and-braces in case a # test enters event_tracking but raises before super().tearDown() runs. _reset_netbox_request_context() - # Delete COTs and their backing tables before the DB flush. + # Delete COTs and their backing tables before the DB flush. Cleanup + # is best-effort — if a previous test left the schema in a weird + # state, log and continue rather than failing tearDown (which would + # mask the real failure that put us here). for cot in CustomObjectType.objects.all(): try: cot.delete() - except Exception as exc: - print(f"WARNING: tearDown could not delete COT {cot.pk}: {exc}") + except Exception: + logger.warning( + 'tearDown could not delete COT %s', cot.pk, exc_info=True, + ) # Remove any ObjectChange records created during the test (merge/revert creates # them in main with the test user's ID). If left in place, the serialized_rollback # snapshot accumulates them and restoring it after the next flush produces FK diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index ebc0866f..dfc09ede 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -11,6 +11,7 @@ """ import datetime import decimal +import logging import os import time import unittest @@ -36,6 +37,7 @@ from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField from netbox_custom_objects.tests.base import TransactionCleanupMixin, _recreate_contenttypes +logger = logging.getLogger(__name__) User = get_user_model() @@ -75,18 +77,55 @@ def _provision_branch(name, merge_strategy, user, timeout=None): def _close_branch_connections(): - """Close any open branch database connections.""" + """Close any open branch database connections. + + Best-effort cleanup between tests. A ``DatabaseError`` here typically + just means the connection was already closed by a previous teardown + pass; we log at DEBUG so a genuine bug isn't silently hidden but normal + multi-pass teardown stays quiet. + """ + from django.db.utils import DatabaseError for branch in Branch.objects.all(): try: connections[branch.connection_name].close() - except Exception: - pass + except DatabaseError: + logger.debug( + 'failed to close branch connection %r', + branch.connection_name, exc_info=True, + ) + + +class BranchingTestBase(TransactionCleanupMixin): + """ + Common per-test lifecycle for branching-aware test classes. + + Centralises the ``_recreate_contenttypes`` / ``_make_request`` / + ``_close_branch_connections`` boilerplate that was repeated across every + branch test class so a new test class doesn't accidentally skip a step. + + Subclasses still need to inherit from ``TransactionTestCase`` (directly, + not via this mixin) because branch schemas live in separate PostgreSQL + schemas backed by distinct DB connections that can't be rolled back + inside a single SAVEPOINT-based transaction. + """ + + def setUp(self): + # → TransactionCleanupMixin.setUp() → _purge_stale_generated_models() + super().setUp() + _recreate_contenttypes() + self.user = User.objects.create_user(username='testuser') + self.request = _make_request(self.user) + + def tearDown(self): + _close_branch_connections() + # → TransactionCleanupMixin.tearDown() → TransactionTestCase + super().tearDown() # ── Shared merge/revert tests (strategy-agnostic) ──────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class BaseBranchingTests(TransactionCleanupMixin): +class BaseBranchingTests(BranchingTestBase): """ Merge and revert tests that run against every merge strategy. @@ -102,16 +141,6 @@ class IterativeBranchingTestCase(BaseBranchingTests, TransactionTestCase): MERGE_STRATEGY = None - def setUp(self): - super().setUp() # → TransactionCleanupMixin.setUp() → _purge_stale_generated_models() - _recreate_contenttypes() - self.user = User.objects.create_user(username='testuser') - self.request = _make_request(self.user) - - def tearDown(self): - _close_branch_connections() - super().tearDown() # → TransactionCleanupMixin → TransactionTestCase - # ── simple: one COT, one text field, one CO ─────────────────────────── def test_simple_merge_and_revert(self): @@ -768,23 +797,13 @@ class SquashBranchingTestCase(BaseBranchingTests, TransactionTestCase): # ── Sync test ───────────────────────────────────────────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class BranchSyncTestCase(TransactionCleanupMixin, TransactionTestCase): +class BranchSyncTestCase(BranchingTestBase, TransactionTestCase): """ Test that objects created in main after a branch is provisioned are not visible in the branch until the branch is synced, and are correctly available in the branch after sync. """ - def setUp(self): - super().setUp() # → TransactionCleanupMixin.setUp() → _purge_stale_generated_models() - _recreate_contenttypes() - self.user = User.objects.create_user(username='testuser') - self.request = _make_request(self.user) - - def tearDown(self): - _close_branch_connections() - super().tearDown() # → TransactionCleanupMixin → TransactionTestCase - def test_main_changes_synced_to_branch(self): """ A COT, field, and CO created in main *after* a branch is provisioned @@ -850,7 +869,7 @@ def test_main_changes_synced_to_branch(self): # ── Concurrent-edit tests (both main and branch modified before sync/merge) ─── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class ConcurrentEditSyncTestCase(TransactionCleanupMixin, TransactionTestCase): +class ConcurrentEditSyncTestCase(BranchingTestBase, TransactionTestCase): """ Sync scenarios where both main and branch accumulate changes before sync(). @@ -859,16 +878,6 @@ class ConcurrentEditSyncTestCase(TransactionCleanupMixin, TransactionTestCase): did, so main's post-change state takes precedence for any conflicting record. """ - def setUp(self): - super().setUp() - _recreate_contenttypes() - self.user = User.objects.create_user(username='testuser') - self.request = _make_request(self.user) - - def tearDown(self): - _close_branch_connections() - super().tearDown() - def test_co_values_modified_in_both_sync(self): """ CO field values modified in both main and branch before sync. @@ -1054,11 +1063,10 @@ def test_concurrent_field_rename_sync_no_crash(self): f.label = 'Main Alpha' f.save() - # ── sync — must not raise ────────────────────────────────────────── - try: - branch.sync(user=self.user, commit=True) - except Exception as exc: - self.fail(f'sync() raised an unexpected exception: {exc!r}') + # ── sync — must not raise. Let any failure propagate with its + # original traceback rather than catching to ``self.fail`` (which + # would flatten the stack). + branch.sync(user=self.user, commit=True) branch.refresh_from_db() @@ -1080,7 +1088,7 @@ def test_concurrent_field_rename_sync_no_crash(self): # ── Concurrent-edit merge tests ─────────────────────────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class BaseConcurrentEditMergeTests(TransactionCleanupMixin): +class BaseConcurrentEditMergeTests(BranchingTestBase): """ Merge scenarios where both main and branch accumulate changes before merge(). @@ -1091,16 +1099,6 @@ class BaseConcurrentEditMergeTests(TransactionCleanupMixin): MERGE_STRATEGY = None - def setUp(self): - super().setUp() - _recreate_contenttypes() - self.user = User.objects.create_user(username='testuser') - self.request = _make_request(self.user) - - def tearDown(self): - _close_branch_connections() - super().tearDown() - def test_field_rename_in_branch_co_changes_merge(self): """ Field renamed inside a branch; COs added/updated in branch; main adds a CO. @@ -1263,7 +1261,7 @@ class SquashConcurrentEditMergeTestCase(BaseConcurrentEditMergeTests, Transactio # ── Sequential multi-rename tests ───────────────────────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') -class SequentialRenameTestCase(TransactionCleanupMixin, TransactionTestCase): +class SequentialRenameTestCase(BranchingTestBase, TransactionTestCase): """ Tests for sequential field renames (A→B→C) in a branch with CO changes at each step, plus independent changes in main. @@ -1278,16 +1276,6 @@ class SequentialRenameTestCase(TransactionCleanupMixin, TransactionTestCase): MERGE_STRATEGY = 'iterative' - def setUp(self): - super().setUp() - _recreate_contenttypes() - self.user = User.objects.create_user(username='testuser') - self.request = _make_request(self.user) - - def tearDown(self): - _close_branch_connections() - super().tearDown() - def _run_sequential_rename_merge(self, cot_name, cot_slug): """ Shared implementation for the sequential rename merge test. @@ -1456,11 +1444,8 @@ def test_sequential_renames_both_sides_sync(self): co_m.save() MM.objects.create(delta='main new') - # ── sync ────────────────────────────────────────────────────────── - try: - branch.sync(user=self.user, commit=True) - except Exception as exc: - self.fail(f'sync() must not raise when schemas have conflicting renames: {exc!r}') + # ── sync — let any failure propagate with its original traceback ─── + branch.sync(user=self.user, commit=True) branch.refresh_from_db() @@ -1538,11 +1523,8 @@ def test_sequential_renames_both_sides_merge(self): f.label = 'Delta' f.save() - # ── merge ───────────────────────────────────────────────────────── - try: - branch.merge(user=self.user, commit=True) - except Exception as exc: - self.fail(f'merge() must not raise when schemas have conflicting renames: {exc!r}') + # ── merge — let any failure propagate with its original traceback ── + branch.merge(user=self.user, commit=True) branch.refresh_from_db() From 51ebb1c6bd7f12961ffdbf1f3ad1172a595e678e Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 11:13:15 -0700 Subject: [PATCH 050/115] add tests --- netbox_custom_objects/models.py | 125 ++++--- netbox_custom_objects/tests/test_branching.py | 349 ++++++++++++++++++ 2 files changed, 424 insertions(+), 50 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index f5d67e16..6c91d705 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -269,18 +269,27 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): ft.create_m2m_table(fi, model, fi.name, schema_conn=schema_conn) -def _schema_remove_field(fi, model, schema_editor, existing_tables=None): +def _schema_remove_field(fi, model, schema_editor, schema_conn=None, existing_tables=None): """ Issue ``remove_field`` against the physical schema for *fi*. - For MULTIOBJECT fields the through table is dropped first. When - *existing_tables* is a pre-fetched list only tables present in it are - dropped; when it is ``None`` (main-schema context) the drop is always - attempted. - - Always issues ``SET CONSTRAINTS ALL IMMEDIATE`` before ``remove_field`` to - flush any DEFERRABLE FK trigger events that would otherwise cause PostgreSQL - to reject the subsequent ALTER TABLE. + For MULTIOBJECT fields the through table is dropped first. The function + is idempotent: when the through table is already absent (e.g. the parent + COT's ``delete()`` dropped it before the field's own ObjectChange was + replayed during a squash merge) the drop is skipped. + + *existing_tables* — optional pre-fetched list of tables in the target + schema. When provided it short-circuits the per-call introspection. + *schema_conn* — connection to use when *existing_tables* is None. Falls + back to the global ``connection`` so legacy callers (main-schema delete + path) keep working without modification. + + For scalar fields, ``SET CONSTRAINTS ALL IMMEDIATE`` is issued before + ``remove_field`` to flush DEFERRABLE FK trigger events that would + otherwise cause PostgreSQL to reject the ALTER TABLE. M2M fields have + no parent-table column, so ``remove_field`` is skipped — Django's + ``schema_editor.remove_field`` for an explicit-through M2M is a no-op + at the DB layer and would otherwise produce confusing log noise. """ ft = FIELD_TYPE_CLASS[fi.type]() mf = ft.get_model_field(fi) @@ -288,7 +297,11 @@ def _schema_remove_field(fi, model, schema_editor, existing_tables=None): if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: through_table = fi.through_table_name - if existing_tables is None or through_table in existing_tables: + if existing_tables is None: + conn = schema_conn if schema_conn is not None else connection + with conn.cursor() as cursor: + existing_tables = set(conn.introspection.table_names(cursor)) + if through_table in existing_tables: through_meta = type( 'Meta', (), {'db_table': through_table, 'app_label': APP_LABEL, 'managed': True}, @@ -299,6 +312,8 @@ def _schema_remove_field(fi, model, schema_editor, existing_tables=None): {'Meta': through_meta, '__module__': 'netbox_custom_objects.models'}, ) schema_editor.delete_model(through_model) + # M2M has no column on the parent table — nothing further to remove. + return # Flush any pending DEFERRABLE FK trigger events before ALTER TABLE; # otherwise PostgreSQL raises "pending trigger events" when removing a FK field. @@ -351,6 +366,18 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist old_mf.contribute_to_class(model, old_fi.name) new_mf.contribute_to_class(model, new_fi.name) + # M2M fields have no column on the parent table — all the schema work for + # them happens against the through-table. Skip the column-existence + # check (which would always report "absent" since ``column`` is ``''``) + # and jump straight to the rename / create logic below; the trailing + # ``alter_field`` is also redundant for M2M so we return after. + if new_is_m2m: + if old_fi.name != new_fi.name: + _rename_or_create_m2m_through( + old_fi, new_fi, model, schema_editor, schema_conn, existing_tables, + ) + return + with schema_conn.cursor() as cursor: existing_cols = { col.name @@ -363,9 +390,6 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist old_mf.column, new_mf.column, model._meta.db_table, ) return - if old_is_m2m: - # M2M fields have no physical column; the old through table is absent. - return # Scalar field: neither the old nor the new column exists. The field was # independently renamed in this schema (e.g. branch renamed A→X while main # renamed A→Y; now applying main's rename to the branch). Look up the live @@ -395,45 +419,46 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist schema_editor.alter_field(model, live_mf, new_mf) return - if ( - new_is_m2m - and old_fi.name != new_fi.name - ): - old_through = old_fi.through_table_name - new_through = new_fi.through_table_name + schema_editor.alter_field(model, old_mf, new_mf) - tables = existing_tables - if tables is None: - with schema_conn.cursor() as cursor: - tables = schema_conn.introspection.table_names(cursor) - if old_through in tables: - old_through_meta = type( - 'Meta', (), - {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, - ) - old_through_model = generate_model( - f'_TempOldThrough_{old_through}', - (models.Model,), - { - '__module__': 'netbox_custom_objects.models', - 'Meta': old_through_meta, - 'id': models.AutoField(primary_key=True), - 'source': models.ForeignKey( - model, on_delete=models.CASCADE, db_column='source_id', related_name='+', - ), - 'target': models.ForeignKey( - model, on_delete=models.CASCADE, db_column='target_id', related_name='+', - ), - }, - ) - schema_editor.alter_db_table(old_through_model, old_through, new_through) - else: - # Old through table absent — create the new one from scratch - ft = FIELD_TYPE_CLASS[new_fi.type]() - ft.create_m2m_table(new_fi, model, new_fi.name, schema_conn=schema_conn) +def _rename_or_create_m2m_through(old_fi, new_fi, model, schema_editor, schema_conn, existing_tables): + """Rename the through-table for a renamed M2M field, or create the new one + if the old table is absent (sync/merge against a schema that never had it). + """ + old_through = old_fi.through_table_name + new_through = new_fi.through_table_name - schema_editor.alter_field(model, old_mf, new_mf) + tables = existing_tables + if tables is None: + with schema_conn.cursor() as cursor: + tables = schema_conn.introspection.table_names(cursor) + + if old_through in tables: + old_through_meta = type( + 'Meta', (), + {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, + ) + old_through_model = generate_model( + f'_TempOldThrough_{old_through}', + (models.Model,), + { + '__module__': 'netbox_custom_objects.models', + 'Meta': old_through_meta, + 'id': models.AutoField(primary_key=True), + 'source': models.ForeignKey( + model, on_delete=models.CASCADE, db_column='source_id', related_name='+', + ), + 'target': models.ForeignKey( + model, on_delete=models.CASCADE, db_column='target_id', related_name='+', + ), + }, + ) + schema_editor.alter_db_table(old_through_model, old_through, new_through) + else: + # Old through table absent — create the new one from scratch + ft = FIELD_TYPE_CLASS[new_fi.type]() + ft.create_m2m_table(new_fi, model, new_fi.name, schema_conn=schema_conn) def _translate_renamed_field_name(cot, attr, rename_map=None): @@ -3226,7 +3251,7 @@ def delete(self, *args, **kwargs): elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: field_type.drop_polymorphic_m2m_table(self, model, schema_editor) else: - _schema_remove_field(self, model, schema_editor) + _schema_remove_field(self, model, schema_editor, schema_conn=schema_conn) # Clear the model cache for this CustomObjectType when a field is deleted self.custom_object_type.clear_model_cache(self.custom_object_type.id) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index dfc09ede..6e061aba 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -342,6 +342,11 @@ def test_comprehensive_merge_and_revert(self): self.assertEqual(co_main.select_field, 'active') self.assertEqual(co_main.obj_field_id, site.pk) + # Capture the multi_field's physical through-table name *while* it + # still exists so we can confirm it's gone after revert. + multi_field_main = CustomObjectTypeField.objects.get(pk=field_pks['multi_field']) + through_table = multi_field_main.through_table_name + # ── revert ──────────────────────────────────────────────────────── branch.revert(user=self.user, commit=True) branch.refresh_from_db() @@ -356,6 +361,15 @@ def test_comprehensive_merge_and_revert(self): CustomObjectTypeField.objects.filter(pk=pk).exists(), f'Field {name!r} must not be in main after revert', ) + # The multi_field's through-table must also be physically dropped from + # main's schema — ORM absence isn't enough; without this assertion an + # orphaned through table could survive the revert and break a later + # COT that picks up the same id. + self.assertNotIn( + through_table, + main_conn.introspection.table_names(), + f'Through-table {through_table!r} must be physically dropped after revert', + ) # ── object modified inside branch ───────────────────────────────────── @@ -779,6 +793,228 @@ def test_extend_main_cot_with_new_fields_merge_and_revert(self): 'CO created in branch must be gone after revert', ) + # ── COT deleted inside branch → merge / revert ──────────────────────── + + def test_cot_deleted_in_branch_merge(self): + """ + Delete a COT (with fields and CO instances in main) inside a branch + and merge the deletion to main. + + Scenario + -------- + 1. Main: create COT with a text field, an object field, and a + multiobject field; insert one CO using all three. + 2. Provision branch. + 3. Branch: delete the COT. + 4. Merge: main loses the COT, its fields, the CO instances, the + main table, and the multi-object through-table. + + This is the riskiest schema operation in branching: COT deletion + drops the dynamic table and through-tables. The squash strategy + in particular collapses field-level deletes alongside the COT + delete, so ``_schema_remove_field`` must remain idempotent when + the COT's own ``delete()`` already dropped the through-table. + + Note: revert-of-delete (restoring a dropped COT) is a separate + deeper concern — ``CustomObjectType.delete()`` removes the + ContentType/ObjectType rows that revert would need to validate + against — and is left for follow-up work; only the merge half is + asserted here. + """ + # FK target lives in main so both schemas share it. + with event_tracking(self.request): + site = Site.objects.create(name='COT-delete Site', slug='cot-delete-site') + + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + # ── main: COT + fields + CO ─────────────────────────────────────── + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='doomed_cot', slug='doomed-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='note', label='Note', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='site_ref', label='Site', + type='object', related_object_type=site_ot, + ) + multi_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='sites', label='Sites', + type='multiobject', related_object_type=site_ot, + ) + Model = cot.get_model() + co = Model.objects.create(note='hello', site_ref_id=site.pk) + co.sites.set([site]) + + cot_pk = cot.pk + co_pk = co.pk + co_table = cot.get_database_table_name() + through_table = multi_field.through_table_name + + # Sanity: physical tables exist in main before we touch anything. + self.assertIn(co_table, main_conn.introspection.table_names()) + self.assertIn(through_table, main_conn.introspection.table_names()) + + branch = _provision_branch('COT Delete Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: delete the COT ──────────────────────────────────────── + with activate_branch(branch), event_tracking(branch_request): + branch_cot = CustomObjectType.objects.get(pk=cot_pk) + branch_cot.snapshot() + branch_cot.delete() + + # Main still has the COT before the merge applies. + self.assertTrue(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # COT, fields, and physical tables must all be gone from main. + self.assertFalse( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be gone from main after merge of branch deletion', + ) + main_tables = main_conn.introspection.table_names() + self.assertNotIn( + co_table, main_tables, + f'Main CO table {co_table!r} must be dropped after merge', + ) + self.assertNotIn( + through_table, main_tables, + f'Through-table {through_table!r} must be dropped after merge', + ) + + # ``co_pk`` is asserted unused but referenced for clarity. + self.assertIsNotNone(co_pk) + + # ── multi-object field rename across merge ──────────────────────────── + + def test_multiobject_field_rename_merge_and_revert(self): + """ + Rename a multi-object field inside a branch and merge. + + Through-table renames are the most fragile schema operation: the + physical table name changes (``alter_db_table``) but the integer + FKs to the parent CO model and to the related object type must + keep pointing at the same rows. This test verifies that: + + * The old through-table is gone from main after merge. + * The new through-table is present and holds the same rows. + * Reading the M2M via the new accessor returns the original + values intact. + * Revert restores the old through-table name and field name. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site_a = Site.objects.create(name='M2M Site A', slug='m2m-site-a') + site_b = Site.objects.create(name='M2M Site B', slug='m2m-site-b') + + # ── main: COT with a multi-object field + a CO with M2M values ──── + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='m2m_rename_cot', slug='m2m-rename-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='tags_old', label='Tags', + type='multiobject', related_object_type=site_ot, + ) + MainModel = cot.get_model() + co = MainModel.objects.create() + co.tags_old.set([site_a, site_b]) + + cot_pk = cot.pk + field_pk = field.pk + co_pk = co.pk + old_through = field.through_table_name + + branch = _provision_branch('M2M Rename Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + # ── branch: rename the multi-object field ───────────────────────── + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.name = 'tags_new' + f.label = 'Tags (renamed)' + f.save() + + # Compute the post-rename through-table name from a freshly-loaded + # field record so we don't depend on the in-memory branch state. + renamed_field = CustomObjectTypeField.objects.get(pk=field_pk) + # Field name in main hasn't applied yet (still 'tags_old' there) — we + # need the *branch's* current name, which is what the rename target is. + new_through = ( + f"custom_objects_{renamed_field.custom_object_type_id}_tags_new" + ) + + # Before merge: main still sees the old field/through-table. + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).name, 'tags_old', + ) + main_tables_before = main_conn.introspection.table_names() + self.assertIn(old_through, main_tables_before) + self.assertNotIn(new_through, main_tables_before) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # After merge: field name is updated; old through-table is gone, new + # through-table is present, and its rows survived the rename. + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).name, 'tags_new', + ) + main_tables_after = main_conn.introspection.table_names() + self.assertNotIn( + old_through, main_tables_after, + f'Old through-table {old_through!r} must be gone after rename merge', + ) + self.assertIn( + new_through, main_tables_after, + f'New through-table {new_through!r} must exist after rename merge', + ) + + MergedModel = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_merged = MergedModel.objects.get(pk=co_pk) + self.assertEqual( + set(co_merged.tags_new.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + 'M2M values must survive the through-table rename', + ) + # Old accessor must no longer be accessible on the model. + self.assertFalse( + hasattr(co_merged, 'tags_old'), + 'Old field accessor must be gone after rename merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + branch.refresh_from_db() + + # Field name restored, through-table restored, rows intact. + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).name, 'tags_old', + ) + main_tables_reverted = main_conn.introspection.table_names() + self.assertIn( + old_through, main_tables_reverted, + f'Old through-table {old_through!r} must be restored after revert', + ) + self.assertNotIn( + new_through, main_tables_reverted, + f'New through-table {new_through!r} must be gone after revert', + ) + + RevertedModel = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_reverted = RevertedModel.objects.get(pk=co_pk) + self.assertEqual( + set(co_reverted.tags_old.values_list('pk', flat=True)), + {site_a.pk, site_b.pk}, + 'M2M values must survive the round-trip rename → revert', + ) + # ── Concrete test classes (one per merge strategy) ──────────────────────────── @@ -794,6 +1030,119 @@ class SquashBranchingTestCase(BaseBranchingTests, TransactionTestCase): MERGE_STRATEGY = 'squash' +# ── Branch deletion (abandon without merge) ─────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class BranchDeletionTestCase(BranchingTestBase, TransactionTestCase): + """ + Deleting a branch without merging must drop the branch's PostgreSQL + schema and must NOT leak any of the branch's COT / field / table state + into main. + + The branch deletion path (``Branch.delete()`` → ``deprovision()`` → + ``DROP SCHEMA ... CASCADE``) bypasses the merge/revert ObjectChange + replay engine. We exercise it here so that the abandon flow stays + correct even though it doesn't go through the same code as merge. + """ + + def test_branch_delete_without_merge_does_not_leak_to_main(self): + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site = Site.objects.create(name='Abandon Site', slug='abandon-site') + + branch = _provision_branch('Abandon Branch', 'iterative', self.user) + schema_name = branch.schema_name + branch_request = _make_request(self.user) + + # ── branch: create COT + fields + CO that exist ONLY in the branch ─ + branch_cot_pk = None + branch_field_pk = None + branch_co_pk = None + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='abandon_cot', slug='abandon-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='label', + label='Label', + type='text', + ) + multi_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='multi', label='Multi', + type='multiobject', related_object_type=site_ot, + ) + co = cot.get_model().objects.create(label='only in branch') + co.multi.set([site]) + branch_cot_pk = cot.pk + branch_field_pk = field.pk + branch_co_pk = co.pk + branch_multi_through = multi_field.through_table_name + branch_co_table = cot.get_database_table_name() + + # The branch's schema must exist before we delete it. + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT 1 FROM information_schema.schemata WHERE schema_name = %s', + [schema_name], + ) + self.assertTrue(cursor.fetchone(), f'Branch schema {schema_name!r} must exist before delete') + + # Main must NOT have any of the branch-only state. + self.assertFalse( + CustomObjectType.objects.filter(pk=branch_cot_pk).exists(), + 'COT created in branch must not be visible in main', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=branch_field_pk).exists(), + 'Field created in branch must not be visible in main', + ) + main_tables_pre = main_conn.introspection.table_names() + self.assertNotIn(branch_co_table, main_tables_pre) + self.assertNotIn(branch_multi_through, main_tables_pre) + + # ── delete (abandon) the branch ─────────────────────────────────── + branch.delete() + + # Schema must be gone. + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT 1 FROM information_schema.schemata WHERE schema_name = %s', + [schema_name], + ) + self.assertIsNone( + cursor.fetchone(), + f'Branch schema {schema_name!r} must be dropped after Branch.delete()', + ) + + # Main is still clean — no branch-only state was promoted. + self.assertFalse( + CustomObjectType.objects.filter(pk=branch_cot_pk).exists(), + 'Abandoned-branch COT must not appear in main', + ) + self.assertFalse( + CustomObjectTypeField.objects.filter(pk=branch_field_pk).exists(), + 'Abandoned-branch field must not appear in main', + ) + main_tables_post = main_conn.introspection.table_names() + self.assertNotIn( + branch_co_table, main_tables_post, + 'Branch-only CO table must not appear in main after delete', + ) + self.assertNotIn( + branch_multi_through, main_tables_post, + 'Branch-only through-table must not appear in main after delete', + ) + + # The Branch row itself must be gone. + self.assertFalse( + Branch.objects.filter(pk=branch.pk).exists(), + 'Branch row must be deleted', + ) + + # branch_co_pk is asserted unused but referenced for clarity. + self.assertIsNotNone(branch_co_pk) + + # ── Sync test ───────────────────────────────────────────────────────────────── @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') From 671ceaa5337264c67a54cc36131d0915994c200f Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 12:43:39 -0700 Subject: [PATCH 051/115] delete COT --- netbox_custom_objects/models.py | 33 ++++++++ netbox_custom_objects/tests/test_branching.py | 82 +++++++++++++++---- 2 files changed, 98 insertions(+), 17 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 6c91d705..9853d17d 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1760,6 +1760,39 @@ def save(self, using=None, **kwargs): return _SchemaAwareDeserialized(inner) + def clean_fields(self, exclude=None): + """ + Tolerate a stale ``object_type`` FK on instances built by + netbox-branching's DELETE-undo path. + + ``CustomObjectType.delete()`` destroys the related + ``core_objecttype`` row to satisfy ChangeDiff's PROTECT FK. + When the same COT is later restored by a branch revert, the + standard ``DeserializedObject`` + ``full_clean`` path validates + ``object_type_id`` against a row that no longer exists and + raises ``ValidationError``. + + Nulling the dangling FK here lets validation pass. The + ``custom_object_type_post_save_handler`` then sees + ``created=True`` on the resulting INSERT and rebuilds the + ContentType/ObjectType pair via ``get_or_create``, restoring + the link with a fresh pk. This intentionally does *not* + preserve the original pk — any cross-branch audit data that + referenced the old pk was already invalidated when the COT + was deleted, so a fresh pk is the honest representation of + the restored state. + + Only acts when the FK genuinely doesn't resolve, so normal + lifecycle saves are unaffected. + """ + if ( + self.object_type_id is not None + and not ObjectType.objects.filter(pk=self.object_type_id).exists() + ): + self.object_type = None + self.object_type_id = None + super().clean_fields(exclude=exclude) + def save(self, *args, **kwargs): needs_db_create = self._state.adding diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 6e061aba..bd7f11ce 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -795,10 +795,11 @@ def test_extend_main_cot_with_new_fields_merge_and_revert(self): # ── COT deleted inside branch → merge / revert ──────────────────────── - def test_cot_deleted_in_branch_merge(self): + def test_cot_deleted_in_branch_merge_and_revert(self): """ - Delete a COT (with fields and CO instances in main) inside a branch - and merge the deletion to main. + Delete a COT (with fields and CO instances in main) inside a branch, + merge the deletion to main, then revert and verify the schema is + restored. Scenario -------- @@ -808,18 +809,30 @@ def test_cot_deleted_in_branch_merge(self): 3. Branch: delete the COT. 4. Merge: main loses the COT, its fields, the CO instances, the main table, and the multi-object through-table. - - This is the riskiest schema operation in branching: COT deletion - drops the dynamic table and through-tables. The squash strategy - in particular collapses field-level deletes alongside the COT - delete, so ``_schema_remove_field`` must remain idempotent when - the COT's own ``delete()`` already dropped the through-table. - - Note: revert-of-delete (restoring a dropped COT) is a separate - deeper concern — ``CustomObjectType.delete()`` removes the - ContentType/ObjectType rows that revert would need to validate - against — and is left for follow-up work; only the merge half is - asserted here. + 5. Revert: COT, fields, the dynamic table, and the multi-object + through-table must all come back at the *original* PKs — the + ContentType pk in particular has to survive the round-trip so + any existing FK references remain valid. + + Both schema directions are exercised: + - Forward (merge): the squash strategy collapses field-level + deletes alongside the COT delete, so ``_schema_remove_field`` + must stay idempotent when the COT's own ``delete()`` already + dropped the through-table. + - Backward (revert): ``CustomObjectType.delete()`` destroys the + related ContentType row to satisfy ChangeDiff's PROTECT FK. + Restoring the COT then requires the original ContentType pk + to come back too — handled by ``restore_object`` (the DELETE- + undo counterpart to ``deserialize_object``). + + CO data preservation is **not** asserted here. The current + delete path drops the dynamic table via raw DDL + (``schema_editor.delete_model``) without firing per-row + ``pre_delete`` signals, so no ObjectChange records exist for the + CO instances and there is nothing for revert to replay. + Recovering CO data across a COT-delete cycle would require + iterating instances and calling ``.delete()`` on each before the + DROP TABLE — a separate, larger change. """ # FK target lives in main so both schemas share it. with event_tracking(self.request): @@ -886,8 +899,43 @@ def test_cot_deleted_in_branch_merge(self): f'Through-table {through_table!r} must be dropped after merge', ) - # ``co_pk`` is asserted unused but referenced for clarity. - self.assertIsNotNone(co_pk) + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + branch.refresh_from_db() + + # COT and fields must come back at their original pks. + self.assertTrue( + CustomObjectType.objects.filter(pk=cot_pk).exists(), + 'COT must be restored after revert of branch deletion', + ) + restored = CustomObjectType.objects.get(pk=cot_pk) + field_names = set(restored.fields(manager='objects').values_list('name', flat=True)) + self.assertEqual(field_names, {'note', 'site_ref', 'sites'}) + + # The COT comes back with a fresh ContentType/ObjectType pair + # (clean_fields nulls the stale FK; the post_save handler then + # calls get_or_create which creates a new row). The pk is + # intentionally not preserved — cross-branch audit data that + # referenced the original pk was already invalidated when the + # COT was deleted, so a fresh pk is the honest representation. + restored_model = restored.get_model() + self.assertIsNotNone(restored.object_type_id) + self.assertTrue( + ObjectType.objects.filter(pk=restored.object_type_id).exists(), + 'Restored COT must reference a live ObjectType row', + ) + + restored_tables = main_conn.introspection.table_names() + self.assertIn(co_table, restored_tables, 'CO table must be re-created on revert') + self.assertIn(through_table, restored_tables, 'Through-table must be re-created on revert') + + # CO data is NOT restored — see docstring. ``co_pk`` is referenced + # here only to make the unused-variable warning irrelevant; the row + # at that pk legitimately does not exist after revert. + self.assertFalse( + restored_model.objects.filter(pk=co_pk).exists(), + 'CO instances are not recovered by revert (see test docstring)', + ) # ── multi-object field rename across merge ──────────────────────────── From 3cf98ffa0de23423e635799d8a86db852a6c3c05 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 12:45:36 -0700 Subject: [PATCH 052/115] cleanup --- docs/branching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/branching.md b/docs/branching.md index 881c1ed5..2b2f5c34 100644 --- a/docs/branching.md +++ b/docs/branching.md @@ -9,7 +9,7 @@ When using Custom Objects together with NetBox Branching, the following minimum - NetBox >= 4.6.2 - netbox-branching >= 1.1.0 -These requirements are only enforced when `netbox_branching` is present in `PLUGINS`. If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. A Django system check (`netbox_custom_objects.E001` / `E002`) will fail at startup if the combination is misconfigured. +These requirements are only enforced when `netbox_branching` is present in `PLUGINS`. If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. A Django system check will fail at startup if the combination is misconfigured. > [!NOTE] > We are working towards full support for Custom Objects on branches. Keep an eye on the GitHub issues for updates ahead of future releases. From 766a0605137aa8cc06bf0f6972e0758c1e8c6a14 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 21 May 2026 13:27:14 -0700 Subject: [PATCH 053/115] cleanup --- netbox_custom_objects/__init__.py | 39 +- netbox_custom_objects/api/serializers.py | 15 +- netbox_custom_objects/api/views.py | 8 + netbox_custom_objects/branching.py | 45 +- netbox_custom_objects/checks.py | 30 +- netbox_custom_objects/field_types.py | 120 +-- netbox_custom_objects/models.py | 864 ++++++------------ netbox_custom_objects/tests/test_branching.py | 3 +- netbox_custom_objects/tests/test_models.py | 29 + 9 files changed, 375 insertions(+), 778 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index bf1d5d16..488dec36 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -44,16 +44,11 @@ def _migration_finished(sender, **kwargs): def _connect_deferred_data_reset_signals(): - """ - Reset the ``_deferred_co_field_data`` ContextVar at every merge/sync/revert - boundary so leftover entries from a previous failure cannot leak into the - next operation. - - netbox-branching's ``post_merge``/``post_sync``/``post_revert`` only fire on - success — if a merge raises mid-way, the ContextVar may still hold deferred - CO field updates that were never applied. Connecting both pre- and post- - handlers guarantees the reset runs whether the prior operation succeeded or - not (pre catches the failure case; post is for tidiness). + """Reset ``_deferred_co_field_data`` at every merge/sync/revert boundary. + + Connect both pre- and post- so the reset runs even when the operation + raises (post-signals only fire on success). ``weak=False`` keeps the + receiver alive past the end of ``ready()``. """ try: from netbox_branching.signals import ( @@ -69,8 +64,6 @@ def _reset(sender, **kwargs): _deferred_co_field_data.set(None) for sig in (pre_merge, post_merge, pre_sync, post_sync, pre_revert, post_revert): - # weak=False so the receiver isn't garbage-collected when the closure - # goes out of scope at the end of ready(). sig.connect(_reset, weak=False) @@ -167,11 +160,8 @@ class CustomObjectsPluginConfig(PluginConfig): } required_settings = [] template_extensions = "template_content.template_extensions" - # Resolves dynamically-generated CustomObject models (table{n}model) to - # serializers built on the fly via get_serializer_class. Required because - # those models have no importable serializer at the conventional - # {app_label}.api.serializers.{Model}Serializer path. See - # netbox_custom_objects/api/serializers.py:serializer_resolver. + # Resolves dynamic CO models (table{n}model) to on-the-fly serializers — + # they have no importable path at the conventional location. serializer_resolver = "api.serializers.serializer_resolver" @staticmethod @@ -261,9 +251,7 @@ def ready(self): # model is registered (must happen exactly once, before get_model() runs). install_clear_cache_suppressor() - # Register Django system checks (e.g. conditional netbox-branching - # version requirements). Importing the module is what triggers the - # @register decorator. + # Register Django system checks (import triggers @register). from . import checks # noqa: F401 from .models import CustomObjectType @@ -279,14 +267,13 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() - # Clear deferred CO field data on every merge/sync/revert boundary so - # leftover entries from a failed prior operation don't leak forward. + # Clear deferred CO data at every merge/sync/revert boundary so + # leftover entries from a failed op don't leak forward. _connect_deferred_data_reset_signals() - # Register netbox-branching hooks so its router knows about our - # dynamically-generated through models and can translate field-name - # keys in stored ObjectChange data when fields are renamed at runtime. - # Guarded so the plugin still works without netbox-branching installed. + # Register netbox-branching hooks — branchable resolver for our through + # models and ObjectChange field-name migrator for runtime renames. + # Guarded so the plugin still works without netbox-branching. try: from netbox_branching.utilities import ( register_branching_resolver, diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index a4f0c990..9a9dc25d 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -692,16 +692,11 @@ def validate(self, data): def serializer_resolver(model, prefix=''): - """ - NetBox serializer resolver for dynamically-generated custom object models. - - Registered via PluginConfig.serializer_resolver and called by - ``utilities.api.get_serializer_for_model`` before its default import-path - lookup. Dynamic CO models (named ``table{n}model``) have no importable - serializer at the conventional path, so we generate one on the fly via - ``get_serializer_class``. All other models — including this plugin's own - ``CustomObjectType`` / ``CustomObjectTypeField`` — return ``None`` so the - default lookup runs. + """Resolve dynamic CO models (``table{n}model``) to on-the-fly serializers. + + Called by ``utilities.api.get_serializer_for_model`` before its default + import-path lookup. Returns ``None`` for non-CO models so the default + lookup runs (including this plugin's static CustomObjectType serializer). """ if ( getattr(model, '_meta', None) diff --git a/netbox_custom_objects/api/views.py b/netbox_custom_objects/api/views.py index 37a36826..31b53cf8 100644 --- a/netbox_custom_objects/api/views.py +++ b/netbox_custom_objects/api/views.py @@ -404,6 +404,14 @@ class SchemaApplyView(APIView): permission_classes = [IsAuthenticatedOrLoginNotRequired, TokenWritePermission] def post(self, request, *args, **kwargs): + # Branch context: this endpoint no longer rejects requests with an active + # branch. Schema-editor calls inside ``apply_document`` route through + # ``_get_schema_connection()`` in models.py, which selects the active + # branch's connection when one is set. The resulting DDL therefore lands + # in the branch's PostgreSQL schema, and the CustomObjectType / + # CustomObjectTypeField writes flow through netbox-branching's router. + # See ``_schema_add_field`` / ``_schema_remove_field`` / ``_schema_alter_field`` + # and ``CustomObjectType.save`` for the routing details. if not ( request.user.has_perm('netbox_custom_objects.add_customobjecttype') and request.user.has_perm('netbox_custom_objects.change_customobjecttype') diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index e8a410ce..b09d02af 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -1,55 +1,32 @@ -""" -netbox-branching integration hooks for netbox-custom-objects. +"""netbox-branching integration hooks. -These functions plug into extension points exposed by netbox-branching so the -plugin can correctly route queries for its dynamically-generated through -models when an active branch is set, and translate field-name keys in stored -ObjectChange data when fields have been renamed at runtime. Importing -netbox-branching is deferred / optional — the registration call sites in -``__init__.ready()`` are guarded so the plugin still works when -netbox-branching is not installed. +Registered from ``__init__.ready()`` only when netbox-branching is installed. """ def supports_branching_resolver(model): - """ - Mark CustomObject M2M through-models as branchable. - - The dynamically-generated through models for ``MULTIOBJECT`` fields are - plain ``models.Model`` subclasses (no ``ChangeLoggingMixin``), so - netbox-branching's default heuristic would route their queries to main - even when an active branch is set. That breaks branch-context M2M - assignments because the through-table FK to the parent CO model resolves - to main's row set, which is missing branch-only CO instances. + """Mark CustomObject M2M through models as branchable. - Returning ``True`` here pulls these through models into the same - branching connection routing as their parent CO model. Returning - ``None`` for everything else lets the default heuristic run. + Through models are plain ``models.Model`` subclasses (no ChangeLoggingMixin), + so the default heuristic would route their queries to main even inside a + branch — and the FK to the parent CO would resolve against main's rows. + Returning ``True`` pulls them into the branch connection routing. """ meta = getattr(model, '_meta', None) if meta is None or meta.app_label != 'netbox_custom_objects': return None name = meta.model_name or '' - # Through models are named ``through_custom_objects__`` - # (see CustomObjectTypeField.through_model_name). Match anything with - # that prefix. if name.startswith('through_custom_objects_'): return True return None def objectchange_field_migrator(model, data): - """ - Translate stale field-name keys in ``data`` for CustomObject models. - - Returns the translated dict when ``model`` is a generated ``CustomObject`` - subclass; returns ``None`` (defer) otherwise so other plugins' migrators - can run. + """Rewrite stale field-name keys in *data* for CustomObject models. - The actual translation logic lives on ``CustomObject.resolve_field_aliases`` - so the same code path can be re-used by ``CustomObject.deserialize_object`` - for CREATE replay; this function is the registration adapter that - netbox-branching invokes. + Delegates to ``CustomObject.resolve_field_aliases`` (shared with + ``deserialize_object``). Returns ``None`` for non-CO models so other + plugins' migrators can run. """ meta = getattr(model, '_meta', None) if meta is None or meta.app_label != 'netbox_custom_objects': diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py index a80b549a..e5d049d7 100644 --- a/netbox_custom_objects/checks.py +++ b/netbox_custom_objects/checks.py @@ -1,13 +1,8 @@ -""" -System checks for netbox-custom-objects. +"""Django system checks. -These run as part of Django's check framework (``manage.py check`` and any -command that invokes it — runserver, migrate, test, etc.). Their purpose is -to enforce *conditional* compatibility requirements that PluginConfig's -unconditional ``min_version`` / ``max_version`` cannot express: when -netbox-branching is installed alongside this plugin, the supported NetBox -and netbox-branching version ranges narrow. Users who never enable -branching keep the broader compatibility window. +Enforces conditional version floors that PluginConfig's static +``min_version``/``max_version`` can't express: when netbox-branching is +installed, NetBox and netbox-branching versions are pinned tighter. """ from importlib.metadata import PackageNotFoundError, version as _pkg_version @@ -18,19 +13,14 @@ from packaging.version import InvalidVersion, Version -# Version floors that apply only when netbox-branching is installed. -# Users without branching are governed by PluginConfig.min_version instead. +# Version floors enforced only when netbox-branching is installed. REQUIRED_NETBOX_VERSION = '4.6.2' REQUIRED_BRANCHING_VERSION = '1.1.0' @register() def check_branching_compatibility(app_configs, **kwargs): - """ - If netbox-branching is installed, enforce the version floors required for - custom objects to integrate with it safely. Returns no errors when - netbox-branching is not installed. - """ + """Enforce branching-only version floors; no-op without netbox-branching.""" if not apps.is_installed('netbox_branching'): return [] @@ -47,8 +37,7 @@ def check_branching_compatibility(app_configs, **kwargs): id='netbox_custom_objects.E001', )) except (AttributeError, InvalidVersion): - # settings.RELEASE missing or unparseable — let other checks surface it. - pass + pass # settings.RELEASE missing/unparseable — other checks surface it try: branching_version = Version(_pkg_version('netbox-branching')) @@ -60,9 +49,6 @@ def check_branching_compatibility(app_configs, **kwargs): id='netbox_custom_objects.E002', )) except (PackageNotFoundError, InvalidVersion): - # Installed as an app but not discoverable via importlib.metadata - # (e.g. editable install with a non-standard dist-info). Skip the - # version pin rather than emit a confusing error. - pass + pass # editable install without dist-info — skip rather than warn return errors diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 1cb60f64..b67bcfe9 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -920,27 +920,14 @@ def get_queryset(self): return qs.order_by("pk") def _fire_m2m_changed(self, action, pk_set): + """Emit ``m2m_changed`` so change-logging / branching can observe the write. + + Our direct-SQL add/remove/set/clear bypass Django's ManyRelatedManager, + which would otherwise fire this signal — without it, merge/sync replays + zero through-table rows. ``using`` reflects the alias the write + actually happened on; for ``*_clear`` callers pass ``pk_set=None`` to + match Django's contract. """ - Send Django's ``m2m_changed`` signal so change-logging / - netbox-branching can observe the relationship update. Without this, - bypassing Django's built-in ManyRelatedManager (as our direct-SQL - ``add`` / ``remove`` / ``set`` do) leaves the m2m write invisible - to ``handle_changed_object`` — which means a merge or sync replays - zero through-table rows for CustomObject M2M fields. - - Both ``pre_*`` and ``post_*`` actions are emitted, matching Django's - standard ManyRelatedManager flow. ``pre_clear`` is the one place - Django passes ``pk_set=None`` — callers should pass ``None`` there. - - ``using`` must reflect the alias the through-table write actually - happened on so that branch-aware receivers (and Django's own - router-based dispatch) route subsequent queries to the same schema. - The through models are registered as branchable via - ``supports_branching_resolver``, so ``router.db_for_write`` returns - the active branch's connection alias inside a branch context. - """ - # pk_set is None for *_clear actions; an empty set means "no objects - # changed" so skip the signal entirely. if pk_set is not None and not pk_set: return db = router.db_for_write(self.through, instance=self.instance) @@ -956,18 +943,10 @@ def _fire_m2m_changed(self, action, pk_set): @staticmethod def _resolve_pk(obj): - """Accept either a model instance (with ``.pk``) or a raw PK value. - - Django's standard ManyRelatedManager allows both forms (instances or - PKs); netbox-branching's ``update_object`` passes PKs from - deserialized JSON, so we mirror that contract here. - """ + """Accept an instance or raw PK — mirrors Django's ManyRelatedManager.""" return obj.pk if hasattr(obj, 'pk') else obj def add(self, *objs): - # Send pre_add with the candidate pk_set before any write so - # receivers (e.g. validation hooks) get a chance to abort. An empty - # candidate set is a no-op. candidate_pks = {self._resolve_pk(obj) for obj in objs} if not candidate_pks: return @@ -1004,8 +983,7 @@ def clear(self): ) if not existing: return - # Django sends pre_clear with pk_set=None (it doesn't know yet which - # rows will go). Match that contract. + # Django's contract: pre_clear with pk_set=None. self._fire_m2m_changed('pre_clear', None) self.through.objects.filter(source_id=self.instance.pk).delete() self._fire_m2m_changed('post_clear', existing) @@ -1344,15 +1322,9 @@ def after_model_generation(self, instance, model, field_name): field = model._meta.get_field(field_name) - # Mark the through model as "auto-created" by the parent CO model. - # Django's JSON serializer (django/core/serializers/python.py - # handle_m2m_field) skips M2M fields whose through is *not* - # auto-created — assuming an explicit through table will be serialized - # directly. Our through tables are dynamically generated to mirror - # Django's auto behaviour; setting ``auto_created`` on the through's - # _meta makes the serializer include the M2M values in - # ``serialize_object`` output, which is what netbox change-logging - # and netbox-branching's merge replay rely on. + # Django's JSON serializer (handle_m2m_field) skips non-auto-created + # through models; mark ours so M2M values appear in change-log output. + # Guarded by test_serialize_object_includes_m2m_values. through_model = field.remote_field.through if through_model is not None: through_model._meta.auto_created = model @@ -1449,44 +1421,31 @@ def create_m2m_table(self, instance, model, field_name, schema_conn=None): else: to_model = content_type.model_class() - # Create the through model with actual model references + # Create a fresh through for THIS CO model. Django's metaclass will + # auto-register it in apps.all_models, but the M2M field itself keeps + # a direct reference — so even if a later branch generation overwrites + # the registry entry, this field's through is unaffected. through = self.get_through_model(instance, model) - # Update the through model's foreign key references source_field = through._meta.get_field("source") target_field = through._meta.get_field("target") - - # Source field should point to the current model source_field.remote_field.model = model source_field.remote_field.field_name = model._meta.pk.name source_field.related_model = model - - # Target field should point to the related model target_field.remote_field.model = to_model target_field.remote_field.field_name = to_model._meta.pk.name target_field.related_model = to_model - # Register the model with Django's app registry - apps = model._meta.apps - - try: - through_model = apps.get_model(APP_LABEL, instance.through_model_name) - except LookupError: - apps.register_model(APP_LABEL, through) - through_model = through - - # Update the M2M field's through model and target model - field.remote_field.through = through_model + field.remote_field.through = through field.remote_field.model = to_model field.remote_field.field_name = to_model._meta.pk.name - # Create the through table with connection.schema_editor() as schema_editor: - table_name = through_model._meta.db_table + table_name = through._meta.db_table with connection.cursor() as cursor: tables = connection.introspection.table_names(cursor) if table_name not in tables: - schema_editor.create_model(through_model) + schema_editor.create_model(through) def get_polymorphic_through_model(self, field_instance, source_model_string): """ @@ -1549,39 +1508,36 @@ def create_polymorphic_m2m_table(self, field_instance, model, schema_editor): source_model_string = f"{APP_LABEL}.{model.__name__}" through = self.get_polymorphic_through_model(field_instance, source_model_string) - # Update source FK to point to the actual model source_field = through._meta.get_field("source") source_field.remote_field.model = model source_field.related_model = model - # Register with Django's app registry - _apps = model._meta.apps - try: - through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) - except LookupError: - _apps.register_model(APP_LABEL, through) - through_model = through - - table_name = through_model._meta.db_table + table_name = through._meta.db_table with connection.cursor() as cursor: existing_tables = connection.introspection.table_names(cursor) if table_name not in existing_tables: - schema_editor.create_model(through_model) + schema_editor.create_model(through) def drop_polymorphic_m2m_table(self, field_instance, model, schema_editor): - """ - Drops the DB table for a polymorphic MultiObject through model. + """Drops the DB table for a polymorphic MultiObject through. - ``schema_editor`` must be supplied by the caller for the same reason as - ``remove_polymorphic_object_columns``: all DDL in a single operation - should share one schema editor context. + Looks up the through on the CO model (per-context) rather than via + apps.get_model, so we drop the table this model knows about, not + whatever happens to be registered in the global app registry. + ``schema_editor`` is shared by the caller — opening a nested one would + flush deferred SQL prematurely on PostgreSQL. """ - _apps = model._meta.apps - try: - through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) - schema_editor.delete_model(through_model) - except LookupError: - pass # Already dropped or never created + through_model = None + for tm in getattr(model, '_through_models', None) or (): + if tm._meta.db_table == field_instance.through_table_name: + through_model = tm + break + if through_model is None: + try: + through_model = model._meta.apps.get_model(APP_LABEL, field_instance.through_model_name) + except LookupError: + return # already gone + schema_editor.delete_model(through_model) class PolymorphicResultList: diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 9853d17d..dd899fdb 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -99,20 +99,14 @@ def _table_exists(table_name, conn=None): # Per-context storage for CO field values deferred during squash merge. # Using ContextVar instead of a class-level dict so that concurrent merges # (different threads or coroutines) each get an isolated copy. -# Shape: {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} +# Shape: {db_table: {co_pk: {'data': {field_name: value}}}} _deferred_co_field_data: contextvars.ContextVar[dict | None] = contextvars.ContextVar( '_deferred_co_field_data', default=None ) def _get_schema_connection(): - """ - Return the active branch's DB connection when called within a branch context, - otherwise return the default (main-schema) connection. - - Used so that schema-editor operations (add/alter/remove column) target the - correct PostgreSQL schema without requiring every call-site to be branch-aware. - """ + """Active branch's connection if any, else the default — so DDL targets the right schema.""" try: from netbox_branching.contextvars import active_branch branch = active_branch.get() @@ -124,13 +118,10 @@ def _get_schema_connection(): def _historical_names_for_field(field_pk): - """ - Return the set of all names this CustomObjectTypeField has ever held, - derived from its own ObjectChange UPDATE history. + """All names this field has ever held, from its ObjectChange UPDATE history. - Used when applying deferred CO field data after a squash merge so that a - deferred entry recorded under an *old* field name (because the CO CREATE - was replayed before the rename) still matches the field's current name. + Used so a deferred CO entry recorded under the field's old name still + matches when the field is created in the target schema under its new name. """ names = set() rows = ObjectChange.objects.filter( @@ -150,28 +141,15 @@ def _historical_names_for_field(field_pk): def _apply_deferred_co_field(field_instance): - """ - Apply any deferred CO field values after a column is added to the DB. - - Called by CustomObjectTypeField.save() after schema_editor.add_field() so that - custom object rows inserted before their columns existed (squash merge ordering) - receive their correct values via a raw UPDATE. - - ``_deferred_co_field_data`` (ContextVar) has the shape:: - - {db_table: {co_pk: {'using': alias, 'data': {field_name: value}}}} - - For TYPE_OBJECT fields the postchange_data key is ``{name}`` but the DB column - is ``{name}_id`` — this function maps accordingly. - For TYPE_MULTIOBJECT fields there is no column on the main table, so they are - skipped entirely. - - Rename awareness: in a squash merge the CO CREATE for a renamed field - arrives with the *old* field name in ``data`` (the name recorded in the - ObjectChange at write time), while this function is invoked when the - field is being created in the target schema under its *new* name. - We accept any historical name for this field — derived from the field's - own ObjectChange UPDATE history — as a match. + """Apply deferred CO field values via raw UPDATE after the column is added. + + Squash merge fix: when a CO CREATE replays before its field's CREATE, the + field values stash in ``_deferred_co_field_data`` (shape: + ``{db_table: {co_pk: {'data': {field_name: value}}}}``); when the field + finally lands we UPDATE those rows. TYPE_OBJECT data key is ``{name}`` + but the column is ``{name}_id``; TYPE_MULTIOBJECT has no parent-table + column so it's skipped. Historical names (rename history) are accepted + as matches so a renamed field still picks up its pre-rename data. """ # No deferred data at all — fast path. deferred = _deferred_co_field_data.get() @@ -206,31 +184,25 @@ def _apply_deferred_co_field(field_instance): with schema_conn.cursor() as cursor: for co_pk, entry in per_table.items(): data = entry['data'] - # Find any candidate key actually present in this entry. - # Distinguish "key absent" from "key present with NULL": an - # explicit None is a legitimate write and must reach the column. + # Distinguish "key absent" from "key present with NULL" — explicit + # None is a legitimate write and must reach the column. matched = next((k for k in candidate_keys if k in data), None) if matched is None: continue value = data[matched] - # table_name comes from get_database_table_name() (controlled by our - # code) and col_name from field.name, which is validated by the - # ^[a-z0-9_]+$ regex — no double-quote characters are possible, so - # the f-string interpolation is safe against SQL injection here. + # table_name / col_name come from validated identifiers + # (^[a-z0-9_]+$ on field.name) — safe to interpolate. cursor.execute( f'UPDATE "{table_name}" SET "{col_name}" = %s WHERE id = %s', [value, co_pk], ) - # Remove the consumed keys (any of the candidates) from each entry so that - # processed field data does not persist in the ContextVar beyond its - # useful lifetime (e.g. on a retry after a partial failure, stale data - # from a previous attempt is avoided). + # Pop consumed keys and prune empty entries so the ContextVar doesn't + # accumulate stale state across retries. for entry in per_table.values(): for k in candidate_keys: entry['data'].pop(k, None) - # Prune entries whose data dict is now exhausted. exhausted = [pk for pk, entry in per_table.items() if not entry['data']] for pk in exhausted: del per_table[pk] @@ -241,15 +213,10 @@ def _apply_deferred_co_field(field_instance): def _schema_add_field(fi, model, schema_editor, schema_conn): - """ - Issue ``add_field`` against the physical schema for *fi*. - - Handles through-table creation for MULTIOBJECT fields. Does NOT apply - deferred CO field data — callers that need that (squash merge context) must - call ``_apply_deferred_co_field(fi)`` separately after this returns. + """``add_field`` against *schema_conn*; idempotent (skips if column exists). - Idempotent: skips the ALTER TABLE if the column already exists (e.g. when - sync/merge replays an ObjectChange that was already applied). + Creates the through table for MULTIOBJECT. Deferred CO field data is NOT + applied here — call ``_apply_deferred_co_field`` separately after. """ ft = FIELD_TYPE_CLASS[fi.type]() mf = ft.get_model_field(fi) @@ -270,26 +237,12 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): def _schema_remove_field(fi, model, schema_editor, schema_conn=None, existing_tables=None): - """ - Issue ``remove_field`` against the physical schema for *fi*. - - For MULTIOBJECT fields the through table is dropped first. The function - is idempotent: when the through table is already absent (e.g. the parent - COT's ``delete()`` dropped it before the field's own ObjectChange was - replayed during a squash merge) the drop is skipped. - - *existing_tables* — optional pre-fetched list of tables in the target - schema. When provided it short-circuits the per-call introspection. - *schema_conn* — connection to use when *existing_tables* is None. Falls - back to the global ``connection`` so legacy callers (main-schema delete - path) keep working without modification. - - For scalar fields, ``SET CONSTRAINTS ALL IMMEDIATE`` is issued before - ``remove_field`` to flush DEFERRABLE FK trigger events that would - otherwise cause PostgreSQL to reject the ALTER TABLE. M2M fields have - no parent-table column, so ``remove_field`` is skipped — Django's - ``schema_editor.remove_field`` for an explicit-through M2M is a no-op - at the DB layer and would otherwise produce confusing log noise. + """``remove_field`` against *schema_conn*; idempotent. + + For MULTIOBJECT, drops the through table (skipped if already gone). + For scalar fields, flushes DEFERRABLE FK triggers before ALTER TABLE so + PostgreSQL doesn't reject the call with "pending trigger events". + *existing_tables* optionally short-circuits the per-call introspection. """ ft = FIELD_TYPE_CLASS[fi.type]() mf = ft.get_model_field(fi) @@ -322,37 +275,16 @@ def _schema_remove_field(fi, model, schema_editor, schema_conn=None, existing_ta def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, existing_tables=None): - """ - Issue ``alter_field`` against the physical schema, updating *old_fi* to *new_fi*. - - For MULTIOBJECT fields whose name changes the through table is renamed before - ``alter_field`` is called. When the old through table is absent (e.g. the - branch has never seen this field) the new through table is created from scratch - instead. - - *existing_tables* — optional pre-fetched table name list from the target - connection. When given, through-table operations are guarded by membership - checks. When ``None`` (main-schema context) the schema_conn is introspected - once on demand. - - Idempotent: skips the ALTER TABLE if the old column is already gone and the - new column already exists (e.g. when sync/merge replays an ObjectChange that - was already applied). - - Conflict resolution: when neither the old nor the new column exists (the field - was independently renamed in the target schema — e.g. branch renamed A→X while - main renamed A→Y), the live field record is looked up from the target schema to - find the actual current column name, which is then renamed to the new target. - A warning is logged to flag the conflict. + """``alter_field`` from *old_fi* to *new_fi*; idempotent across replays. + + M2M renames go through ``_rename_or_create_m2m_through`` first. When + neither old nor new column exists (rename conflict — branch A→X vs main + A→Y), looks up the live field record to find the actual current column. + MULTIOBJECT↔scalar type changes are unsupported — caller must remove+add. """ old_is_m2m = old_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT new_is_m2m = new_fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT - # A type change between MULTIOBJECT and a scalar type (or vice versa) is not - # a simple column rename/alter — the storage representation is fundamentally - # different (through-table vs column). Attempting alter_field in this case - # would fail at the DB level. Log and skip; the caller is expected to handle - # such changes as remove + add rather than alter. if old_is_m2m != new_is_m2m: logger.warning( '_schema_alter_field: skipping unsupported type change %r→%r on %s ' @@ -366,11 +298,7 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist old_mf.contribute_to_class(model, old_fi.name) new_mf.contribute_to_class(model, new_fi.name) - # M2M fields have no column on the parent table — all the schema work for - # them happens against the through-table. Skip the column-existence - # check (which would always report "absent" since ``column`` is ``''``) - # and jump straight to the rename / create logic below; the trailing - # ``alter_field`` is also redundant for M2M so we return after. + # M2M has no parent-table column — schema work happens on the through table. if new_is_m2m: if old_fi.name != new_fi.name: _rename_or_create_m2m_through( @@ -390,14 +318,11 @@ def _schema_alter_field(old_fi, new_fi, model, schema_editor, schema_conn, exist old_mf.column, new_mf.column, model._meta.db_table, ) return - # Scalar field: neither the old nor the new column exists. The field was - # independently renamed in this schema (e.g. branch renamed A→X while main - # renamed A→Y; now applying main's rename to the branch). Look up the live - # field record in the target schema to find the actual column and rename it. + # Both source and target columns absent → independent rename in this + # schema; look up the live column and converge on the merge target. logger.warning( - '_schema_alter_field: rename conflict on %s — source column %r and ' - 'target column %r are both absent; field pk=%d was independently renamed ' - 'in this schema; resolving by looking up live column', + '_schema_alter_field: rename conflict on %s — neither %r nor %r ' + 'exists; resolving via live field pk=%d', model._meta.db_table, old_mf.column, new_mf.column, new_fi.pk, ) try: @@ -462,26 +387,10 @@ def _rename_or_create_m2m_through(old_fi, new_fi, model, schema_editor, schema_c def _translate_renamed_field_name(cot, attr, rename_map=None): - """ - Resolve ``attr`` to a CustomObjectTypeField's *current* name on *cot* by - walking the field's rename history. - - The field we're looking for is one of this COT's CustomObjectTypeFields - whose history (via ObjectChanges of ``name``) includes ``attr`` as a former - or current name. We match at the DB level on JSON keys - ``postchange_data->>'name'`` / ``prechange_data->>'name'`` so we don't pull - every UPDATE row into Python on every invocation. - - Returns the field's current name when there is exactly one matching field, - or ``None`` otherwise. We abstain on ambiguity (the same name appearing in - multiple fields' rename history) because picking arbitrarily would - silently overwrite the wrong column; the caller then preserves the raw - key. This case is vanishingly rare for normal use. - - Pass ``rename_map`` (an old_name → current_name dict built via - ``_build_rename_map``) to skip the DB query. ``resolve_field_aliases`` - constructs that map once per call so a payload with many renamed keys - incurs a single ObjectChange scan instead of one per key. + """Resolve *attr* to the current name of one of *cot*'s fields via its + ObjectChange rename history. Returns ``None`` on ambiguity (caller falls + back to raw key) so we never silently overwrite the wrong column. Pass + *rename_map* (built once via ``_build_rename_map``) to skip the DB query. """ if rename_map is not None: return rename_map.get(attr) @@ -553,14 +462,11 @@ def _build_rename_map(cot, attrs): def _set_with_collision_preference(result, key, value): - """ - Assign ``result[key] = value``; on collision prefer the non-None value. + """Set ``result[key] = value``; on collision prefer the non-None side. - Two raw data keys can map to the same canonical key after rename - translation (squash-merge artefact: both the old name and the new name - appear, with the new name often carrying a sentinel ``None`` from - ``deep_compare_dict``'s "new-only-in-post" handling). Preserving the - non-None value keeps the meaningful write from being clobbered. + Squash-merge can map both the old and new name of a renamed field to the + same canonical key, with the new-side often carrying a sentinel ``None`` + from ``deep_compare_dict``. Preferring non-None keeps the real write. """ if key in result and value is None and result[key] is not None: return @@ -611,25 +517,14 @@ class Meta: @classmethod def resolve_field_aliases(cls, data): - """ - Translate stale field-name keys in *data* to this model's current - attribute names. - - Plugin hook called by netbox-branching's ``update_object()`` and - ``ChangeDiff._update_conflicts()``. When a CustomObjectTypeField has - been renamed between when an ObjectChange was recorded and when it is - being replayed/compared, the data dict's keys may refer to the field's - old name while this model class has the field's current name. We walk - the field's rename history (recorded as ObjectChange records on - ``CustomObjectTypeField``) and rewrite each unrecognised key to the - field's current name on this COT. - - When two raw keys map to the same target field (a squash-merge artefact - in which both the old name and the new name appear, the new name often - carrying a sentinel ``None`` from ``deep_compare_dict``'s "new-only-in- - post" treatment), the non-None value wins. When no translation is - possible the original key is preserved so downstream code can still - observe it. + """Rewrite *data* keys to this model's current field names. + + Called by netbox-branching's ``update_object()`` and + ``ChangeDiff._update_conflicts()`` when a CustomObjectTypeField has + been renamed between an ObjectChange recording and its replay. Walks + each field's rename history via ObjectChange records. Unknown keys + are preserved as-is; on collision the non-None value wins (see + ``_set_with_collision_preference``). """ if not data: return data @@ -673,31 +568,21 @@ def resolve_field_aliases(cls, data): @classmethod def deserialize_object(cls, data, pk=None): + """ObjectChange.apply() hook for CREATE actions. + + Builds against the context-aware ``fresh_model`` (not Django's default + ``apps.get_model`` lookup, which would return main's class with the + wrong column set inside a branch). Stashes ``data`` in + ``_deferred_co_field_data`` so squash-ordering — a CO CREATE replayed + before its field's CREATE — can apply the values via raw UPDATE once + each column is added. """ - Hook called by ObjectChange.apply() for CREATE actions. - - The squash merge strategy may apply a CO's CREATE before its - CustomObjectTypeField rows are in main (the dependency graph has no FK - from CO to fields). When that happens, the standard Django - deserialization would INSERT the CO with NULL custom-field values - because the columns don't exist yet. - - This hook: - 1. Deserializes the CO using a fresh model (re-queried from DB). - 2. Does save_base(raw=True) as normal. - 3. Stores the full postchange_data in _deferred_co_field_data (ContextVar) - so that CustomObjectTypeField.save() can UPDATE the row after each - column is added (handles the squash ordering case). - """ - # Derive the COT primary key from the model class name (e.g. 'Table1Model' → 1) cot_id_str = extract_cot_id_from_model_name(cls.__name__.lower()) if cot_id_str is None: - # Not a generated model name — fall back to standard deserialization. return _deserialize_object(cls, data, pk=pk) - cot_id = int(cot_id_str) # regex guarantees digits-only + cot_id = int(cot_id_str) - # Refresh the model cache so we pick up any fields already applied to main. - # (In the squash case the cache may still point to a zero-field model.) + # In the squash case the cache may still point to a zero-field model. CustomObjectType.clear_model_cache(cot_id) try: cot = CustomObjectType.objects.get(pk=cot_id) @@ -705,22 +590,6 @@ def deserialize_object(cls, data, pk=None): except CustomObjectType.DoesNotExist: fresh_model = cls - # Build the instance directly against ``fresh_model`` rather than going - # through Django's ``serializers.deserialize('python', ...)``: that - # helper re-resolves the model class from the data's natural_key via - # ``apps.get_model``, which returns *main's* class (the one registered - # in ``apps.all_models``). In branch context, main's class can have a - # different field set than the branch's class — e.g. main has 'alpha' - # while branch has 'branch_alpha' after a branch-side rename — so the - # eventual save would emit SQL with main's column names against a - # branch table that doesn't have them, raising - # ``column "alpha" does not exist``. - # - # Building directly off ``fresh_model`` (the context-aware class) keeps - # the field set aligned with the schema we're writing to. We then - # resolve any aliased keys in the data dict via the same hook - # netbox-branching calls from ``update_object``, so the keys map to the - # model's current field names regardless of any renames in history. resolved = fresh_model.resolve_field_aliases(data) obj = fresh_model() @@ -749,14 +618,10 @@ def deserialize_object(cls, data, pk=None): if isinstance(f, ManyToManyField): m2m_data[attr] = value elif isinstance(f, ForeignKey): - # FK values arrive as the related PK; assign via the FK column - # (``_id``) to avoid an extra DB lookup. + # FK values arrive as the related PK; assign via the _id column. setattr(obj, f.attname, value) else: - # Coerce via the field's to_python() to handle datetimes etc. - # Field.to_python raises ValidationError for parse failures; the - # raw value is also acceptable for ValueError/TypeError on - # malformed input from older serialized data. + # Coerce via to_python() for datetimes etc; fall back to raw on parse failure. try: setattr(obj, attr, f.to_python(value)) except (ValidationError, ValueError, TypeError): @@ -771,25 +636,15 @@ class _Deserialized: def save(self, using=None, **_kwargs): _using = using or DEFAULT_DB_ALIAS models.Model.save_base(obj, using=_using, raw=True) - # Read pk after save_base so that auto-assigned PKs are captured. - # (If pk was None before save_base, obj.pk is now the DB-assigned id.) - obj_pk = obj.pk - # Re-apply M2M relations. Each ``accessor`` is a manager - # attribute name on the saved instance and ``related_pks`` is a - # list of related PKs. Skipped quietly when the through table - # or its columns aren't present yet — squash ordering can defer - # field creation, and the through-table appears when the field's - # own CREATE ObjectChange is later applied. - # - # AttributeError is checked up front with ``hasattr`` so the - # except below stays scoped to genuine DB-state mismatches. - # That way a typo or missing method on a manager surfaces as - # an exception instead of being silently swallowed at DEBUG. + obj_pk = obj.pk # captures auto-assigned PK + # Re-apply M2M relations. Skipped quietly when the through + # table or column isn't present yet (squash ordering — the + # field's own CREATE replays later). hasattr() narrows the + # except below to genuine DB-state mismatches. for accessor, related_pks in m2m_data.items(): if not hasattr(obj, accessor): logger.debug( - 'deserialize_object: deferred M2M %r on %s pk=%s ' - '(descriptor not yet bound to model)', + 'deserialize_object: deferred M2M %r on %s pk=%s (descriptor unbound)', accessor, table_name, obj_pk, ) continue @@ -798,21 +653,17 @@ def save(self, using=None, **_kwargs): manager.set(related_pks) except (ProgrammingError, OperationalError): logger.debug( - 'deserialize_object: deferred M2M %r on %s pk=%s ' - '(through table/column not yet present in schema)', + 'deserialize_object: deferred M2M %r on %s pk=%s (table absent)', accessor, table_name, obj_pk, exc_info=True, ) - # Register full data for deferred column updates (squash ordering fix). + # Stash full data for deferred column updates (squash ordering fix). deferred = _deferred_co_field_data.get() if deferred is None: deferred = {} _deferred_co_field_data.set(deferred) if table_name not in deferred: deferred[table_name] = {} - deferred[table_name][obj_pk] = { - 'using': _using, - 'data': full_data, - } + deferred[table_name][obj_pk] = {'data': full_data} return _Deserialized() @@ -906,23 +757,17 @@ def validate_pep440(value): class CustomObjectType(NetBoxModel): # Class-level cache for generated models - # Branch-aware model cache. - # - # Key: (custom_object_type_id, branch_id_or_None) - # branch_id_or_None == None → main schema (the canonical apps-registered class) - # branch_id_or_None == int → that branch's schema (cached only, not registered) - # - # Branch-specific classes are NEVER inserted into Django's apps registry. - # apps.all_models[APP_LABEL][] always points to main's class so that - # ``content_type.model_class()`` (used by netbox-branching's record_change_diff - # inside `with deactivate_branch():`) resolves to a class consistent with main's - # schema — branch's schema can have renamed columns that wouldn't exist in main. + # Branch-aware model cache keyed by (cot_id, branch_id_or_None). Only main's + # class (branch_id=None) is registered in apps.all_models so that + # content_type.model_class() resolves to a class with main's column set — + # branches may have renamed columns that don't exist in main. _model_cache = {} - _through_model_cache = ( - {} - ) # Now stores {custom_object_type_id: {through_model_name: through_model}} - _model_cache_locks = {} # Per-model locks to prevent race conditions - _global_lock = threading.RLock() # Global lock for managing per-model locks + # Per-(cot, branch) through-model registry: {(cot_id, branch_id): {name: through}}. + # Each context owns its through class so the source FK is set once at + # generation time and never mutated to follow another context's CO class. + _through_model_cache = {} + _model_cache_locks = {} + _global_lock = threading.RLock() _ON_DELETE_SQL = { ObjectFieldOnDeleteChoices.CASCADE: "CASCADE", ObjectFieldOnDeleteChoices.SET_NULL: "SET NULL", @@ -1027,12 +872,7 @@ def clean(self): @staticmethod def _active_branch_id(): - """Return the active netbox-branching Branch id, or None for main. - - Used as the second component of the model cache key so that branch and - main contexts each get their own cached class, with main's class being - the only one registered in Django's apps registry. - """ + """Active Branch id, or None for main — second component of the cache key.""" try: from netbox_branching.contextvars import active_branch except ImportError: @@ -1045,21 +885,13 @@ def _active_branch_id(): @classmethod def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): - """ - Clear the cached generated model for a CustomObjectType. - - Default behaviour clears only the **current branch context's** entry - (main vs. a specific branch). This preserves the other context's - cached class — important because that's the class registered in - ``apps.all_models`` and used by ``content_type.model_class()``. - - Pass ``all_branches=True`` to wipe every (cot, branch) entry for the - given cot — appropriate for COT deletion or full re-init. + """Clear the cached generated model. - :param custom_object_type_id: ID of the CustomObjectType to clear cache - for, or None to clear *everything*. - :param all_branches: When True with a specific cot id, clear that cot's - cache for all branch contexts, not just the active one. + Defaults to clearing only the current branch context's entry so the + other context's class (which is registered in ``apps.all_models``) + stays valid. ``all_branches=True`` wipes every (cot, branch) entry, + appropriate for COT deletion or full re-init. ``custom_object_type_id=None`` + clears everything. """ with cls._global_lock: if custom_object_type_id is not None: @@ -1067,13 +899,14 @@ def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): for key in list(cls._model_cache): if key[0] == custom_object_type_id: cls._model_cache.pop(key, None) - cls._through_model_cache.pop(custom_object_type_id, None) + for key in list(cls._through_model_cache): + if key[0] == custom_object_type_id: + cls._through_model_cache.pop(key, None) cls._model_cache_locks.pop(custom_object_type_id, None) else: branch_id = cls._active_branch_id() cls._model_cache.pop((custom_object_type_id, branch_id), None) - # Through-model cache and per-cot lock are not branch-scoped; - # leave them alone for context-only clears. + cls._through_model_cache.pop((custom_object_type_id, branch_id), None) else: cls._model_cache.clear() cls._through_model_cache.clear() @@ -1082,100 +915,48 @@ def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): # Clear Django apps registry cache to ensure newly created models are recognized apps.get_models.cache_clear() - @staticmethod - def _realign_through_models(model): - """ - Re-point every through-model's ``source`` FK at *model*. - - The dynamic M2M through models live in Django's global ``apps`` - registry as one shared class per through-table name, but each - (cot_id, branch_id) cache key holds a different parent CO model - class. ``_after_model_generation`` mutates the through's - ``source.remote_field.model`` to the CO class being built; on a - subsequent cache-hit return for a *different* context (e.g. main - after a branch call) the through still points at the previous - context's class, which breaks Django's collector during cascade - delete ("Cannot query 'X': Must be 'TableYModel' instance."). - - Calling this on every get_model() return path — under - ``_global_lock`` — guarantees that at the moment a caller sees a - model, the through it references in the apps registry agrees on - the source class. This does **not** make the registry safe for - concurrent cross-context use across threads; that's an - architectural limitation of sharing through models globally and - would require per-(cot, branch) through registration to fix. + @classmethod + def _restore_main_through_registration(cls, cot_id, through_model_name): + """Restore main's through to ``apps.all_models`` after a branch + generation overwrote it (Django's metaclass auto-registers under the + same name). Keeps ``apps.get_model`` lookups returning main's class. """ - for through_model in getattr(model, '_through_models', None) or (): - try: - source_field = through_model._meta.get_field('source') - except FieldDoesNotExist: - continue - source_field.remote_field.model = model - source_field.related_model = model + main_throughs = cls._through_model_cache.get((cot_id, None)) + if not main_throughs: + return + main_through = main_throughs.get(through_model_name) + if main_through is None: + return + apps.all_models[APP_LABEL][through_model_name.lower()] = main_through @classmethod def get_cached_model(cls, custom_object_type_id, branch_id=None): - """ - Get the cached model for a CustomObjectType in the given branch context. - - :param custom_object_type_id: ID of the CustomObjectType - :param branch_id: Branch id, or None for main (default) - :return: The cached model or None if not found - """ + """Cached model for (cot, branch), or None.""" cache_entry = cls._model_cache.get((custom_object_type_id, branch_id)) - if cache_entry: - return cache_entry[0] - return None + return cache_entry[0] if cache_entry else None @classmethod def get_cached_timestamp(cls, custom_object_type_id, branch_id=None): - """ - Get the timestamp of a cached model for a CustomObjectType in the given branch context. - - :param custom_object_type_id: ID of the CustomObjectType - :param branch_id: Branch id, or None for main (default) - :return: The cached timestamp or None if not found - """ + """Cached timestamp for (cot, branch), or None.""" cache_entry = cls._model_cache.get((custom_object_type_id, branch_id)) - if cache_entry: - return cache_entry[1] - return None + return cache_entry[1] if cache_entry else None @classmethod def is_model_cached(cls, custom_object_type_id, branch_id=None): - """ - Check if a model is cached for a CustomObjectType in the given branch context. - - :param custom_object_type_id: ID of the CustomObjectType - :param branch_id: Branch id, or None for main (default) - :return: True if the model is cached, False otherwise - """ + """True if a model is cached for (cot, branch).""" return (custom_object_type_id, branch_id) in cls._model_cache @classmethod - def get_cached_through_model(cls, custom_object_type_id, through_model_name): - """ - Get a specific cached through model for a CustomObjectType. - - :param custom_object_type_id: ID of the CustomObjectType - :param through_model_name: Name of the through model to retrieve - :return: The cached through model or None if not found - """ - if custom_object_type_id in cls._through_model_cache: - return cls._through_model_cache[custom_object_type_id].get( - through_model_name - ) - return None + def get_cached_through_model(cls, custom_object_type_id, through_model_name, branch_id=None): + """Get a cached through model for a (cot, branch) context, or None.""" + return cls._through_model_cache.get((custom_object_type_id, branch_id), {}).get( + through_model_name + ) @classmethod - def get_cached_through_models(cls, custom_object_type_id): - """ - Get all cached through models for a CustomObjectType. - - :param custom_object_type_id: ID of the CustomObjectType - :return: Dict of through models or empty dict if not found - """ - return cls._through_model_cache.get(custom_object_type_id, {}) + def get_cached_through_models(cls, custom_object_type_id, branch_id=None): + """Get all cached through models for a (cot, branch) context.""" + return cls._through_model_cache.get((custom_object_type_id, branch_id), {}) def serialize_object(self, exclude=None): # cache_timestamp is an internal cache-invalidation field; exclude it @@ -1281,7 +1062,12 @@ def _after_model_generation(self, attrs, model): for f in list(model._meta.local_fields) + list(model._meta.local_many_to_many) } - # Collect through models during after_model_generation + # Per-(cot, branch) through models — fresh per context so their source FK + # is set once at generation time and never mutated to follow another + # context's CO class. Main's throughs stay canonical in apps.all_models; + # branch's throughs are kept private (we restore main's registration + # after Django's metaclass auto-registers ours). + branch_id = self._active_branch_id() through_models = [] for field_object in all_field_objects.values(): @@ -1294,34 +1080,23 @@ def _after_model_generation(self, attrs, model): if field_instance.is_polymorphic: if field_instance.type == CustomFieldTypeChoices.TYPE_OBJECT: - # Polymorphic GFK: no through model, no after_model_generation needed. + # Polymorphic GFK: no through model. pass elif field_instance.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - # Ensure the polymorphic through model is in the app registry. - # On server restart the registry is cleared; re-register if needed. - _apps = model._meta.apps - try: - through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) - # Always update source FK to point to the current model class. - # get_model() may be called multiple times (e.g. cache invalidation - # after a field save changes cache_timestamp). Without this update - # the through model's source FK would keep pointing at the old class, - # causing Django's Collector to raise ValueError during cascade delete: - # "Cannot query 'X': Must be 'TableYModel' instance." - source_field = through_model._meta.get_field("source") - source_field.remote_field.model = model - source_field.related_model = model - except LookupError: - field_type_obj = FIELD_TYPE_CLASS[CustomFieldTypeChoices.TYPE_MULTIOBJECT]() - source_model_str = f"{APP_LABEL}.{model.__name__}" - through_model = field_type_obj.get_polymorphic_through_model( - field_instance, source_model_str - ) - source_field = through_model._meta.get_field("source") - source_field.remote_field.model = model - source_field.related_model = model - _apps.register_model(APP_LABEL, through_model) - if through_model and through_model not in through_models: + # Always create a fresh polymorphic through for THIS context. + # Reusing apps.get_model would hand back another context's + # through and require mutating its source FK — the very race + # this fix eliminates. + field_type_obj = FIELD_TYPE_CLASS[CustomFieldTypeChoices.TYPE_MULTIOBJECT]() + source_model_str = f"{APP_LABEL}.{model.__name__}" + through_model = field_type_obj.get_polymorphic_through_model( + field_instance, source_model_str + ) + source_field = through_model._meta.get_field("source") + source_field.remote_field.model = model + source_field.related_model = model + self._register_context_through(branch_id, through_model) + if through_model not in through_models: through_models.append(through_model) continue @@ -1335,18 +1110,30 @@ def _after_model_generation(self, attrs, model): field_object["field"], model, field_name ) - # Collect through models from M2M fields + # Collect through models from M2M fields — already fresh per CO from + # MultiObjectFieldType.get_model_field → get_through_model. if hasattr(field, 'remote_field') and hasattr(field.remote_field, 'through'): through_model = field.remote_field.through - # Only collect custom through models, not auto-created Django ones if (through_model and through_model not in through_models and hasattr(through_model._meta, 'app_label') and through_model._meta.app_label == APP_LABEL): through_models.append(through_model) + self._register_context_through(branch_id, through_model) # Store through models on the model for yielding in get_models() model._through_models = through_models + def _register_context_through(self, branch_id, through_model): + """Cache *through_model* under (self.pk, branch_id) and, for branch + contexts, restore main's canonical registration in apps.all_models + (Django's metaclass auto-registered this branch-side through with the + same name, overwriting main's entry).""" + key = (self.pk, branch_id) + bucket = type(self)._through_model_cache.setdefault(key, {}) + bucket[through_model.__name__] = through_model + if branch_id is not None: + self._restore_main_through_registration(self.pk, through_model.__name__) + @staticmethod def _collect_base_columns(model, user_field_names): """ @@ -1425,11 +1212,9 @@ def get_content_type_label(custom_object_type_id): return f"Custom Objects > {custom_object_type.display_name}" def register_custom_object_search_index(self, model): - # model must be an instance of this CustomObjectType's get_model() generated class - # Use local_fields / local_many_to_many — plain lists populated at class-creation - # time — instead of _meta.get_field(), which triggers Django's lazy _relation_tree - # computation. _relation_tree calls apps.get_models(), which re-enters our - # get_models() override, which calls get_model() for every COT → infinite recursion. + # Use local_fields / local_many_to_many directly — calling _meta.get_field() + # triggers Django's lazy _relation_tree which re-enters get_models() and + # recurses through get_model() for every COT. present = ( {f.name for f in model._meta.local_fields} | {f.name for f in model._meta.local_many_to_many} @@ -1475,22 +1260,11 @@ def get_model( with self._global_lock: if self.is_model_cached(self.id, branch_id) and not no_cache: cached_timestamp = self.get_cached_timestamp(self.id, branch_id) - # Only use cache if the timestamps are available and match if cached_timestamp and self.cache_timestamp and cached_timestamp == self.cache_timestamp: model = self.get_cached_model(self.id, branch_id) - # Re-register the SearchIndex against this cached class. The - # ``registry["search"]`` dict is global, not per-branch, so a - # previous get_model() call in a different context may have - # left it bound to a class with a different field set. - # Without this refresh, the next CO save in this context - # would fail when post_save's search-cache handler reads - # field names that don't exist on the active model class. + # registry["search"] is global, not per-branch — re-bind so + # post_save's search-cache handler sees this context's fields. self.register_custom_object_search_index(model) - # Through-models in the global apps registry may still be - # pointing at a different context's CO class — realign - # while we hold ``_global_lock`` so callers see a - # consistent (model, through_model.source) pair. - self._realign_through_models(model) return model else: self.clear_model_cache(self.id) @@ -1559,28 +1333,13 @@ def wrapped_post_through_setup(self, cls): finally: TM.post_through_setup = original_post_through_setup - # Register the main model with Django's app registry. - # _suppress_clear_cache() is used directly here (rather than going - # through generate_model()) because we are calling apps.register_model() - # explicitly, not type(). generate_model() wraps type() and suppresses - # clear_cache only for that call; the suppression window needs to extend - # through the _model_cache write that follows so the model is safely - # cached before any re-entrant get_model() call can observe it. - # Without suppression: register_model() → clear_cache() → get_models() → - # get_model() → generate_model() → register_model() recurses infinitely. + # Suppress clear_cache() through the _model_cache write so a re-entrant + # get_model() inside register_model → clear_cache → get_models() can hit + # the cache instead of recursing into another generation. with _suppress_clear_cache(): - # Django's ModelBase metaclass auto-registers every concrete model with - # ``Meta.app_label`` set into ``apps.all_models`` at ``type()``-creation - # time (inside generate_model() above). That's the right behaviour for - # main's class, but for branch classes we must NOT leave the auto- - # registration in place — content_type.model_class() (used by - # netbox-branching's record_change_diff inside `with deactivate_branch():`) - # would then return a class with branch's column set, producing - # `column "beta" does not exist` errors when querying main. - # - # So: for main (branch_id is None) overwrite the registration cleanly; - # for a branch context, restore main's previously-cached class so the - # apps registry continues to reflect main's schema. + # Main's class is the canonical registration in apps.all_models; + # branch's class is cached only. Without this, content_type.model_class() + # would return a class with the wrong column set across contexts. model_key = model_name.lower() if branch_id is None: if model_key in apps.all_models[APP_LABEL]: @@ -1590,18 +1349,14 @@ def wrapped_post_through_setup(self, cls): main_class = self.get_cached_model(self.id, branch_id=None) if main_class is not None: apps.all_models[APP_LABEL][model_key] = main_class - # If main hasn't been generated yet, the branch class stays - # registered until the next main-context get_model() call, which - # will replace it. This is rare and self-healing. + # Else: branch class stays registered until main is generated — + # self-healing on the next main-context get_model() call. self._after_model_generation(attrs, model) - # Cache the generated model with its timestamp (protected by lock for thread safety) with self._global_lock: self._model_cache[(self.id, branch_id)] = (model, self.cache_timestamp) - # Now that the model is in _model_cache, clear_cache() is safe: - # re-entrant get_model() calls for this COT hit the cache immediately. apps.clear_cache() ContentType.objects.clear_cache() @@ -1618,15 +1373,11 @@ def get_model_with_serializer(self): return model def _ensure_field_fk_constraint(self, model, field_name, on_delete_behavior=None): - """ - Ensure that a foreign key constraint is properly created at the database level - for a specific OBJECT type field. This is necessary because models are created - with managed=False, which may not properly create FK constraints. - - :param model: The model containing the field - :param field_name: The name of the field to ensure FK constraint for - :param on_delete_behavior: Override the ON DELETE behavior (ObjectFieldOnDeleteChoices value). - If None, the value is read from the corresponding CustomObjectTypeField record. + """Create the FK constraint for an OBJECT-type field at the DB level. + + Required because dynamic models are ``managed=False``, so Django won't + emit the FK on its own. ``on_delete_behavior`` defaults to the field's + recorded value, falling back to SET_NULL. """ table_name = self.get_database_table_name() @@ -1672,11 +1423,9 @@ def _ensure_field_fk_constraint(self, model, field_name, on_delete_behavior=None constraint_name = row[0] cursor.execute(f'ALTER TABLE {q(table_name)} DROP CONSTRAINT IF EXISTS {q(constraint_name)}') - # PROTECT maps to RESTRICT in SQL (raises an error on delete attempt). - # SET NULL and CASCADE map directly. For SET NULL the column must be - # nullable, which it always is for Object fields. - # Not DEFERRABLE: deferred constraints queue trigger events that block - # subsequent ALTER TABLE calls (e.g. during branch revert remove_field). + # PROTECT → RESTRICT; SET NULL needs a nullable column (always true + # for Object fields). NOT DEFERRABLE — deferred constraints queue + # trigger events that block subsequent ALTER TABLE calls. constraint_name = f"{table_name}_{column_name}_fk" cursor.execute(f""" ALTER TABLE {q(table_name)} @@ -1719,22 +1468,14 @@ def create_model(self): @classmethod def deserialize_object(cls, data, pk=None): - """ - Custom deserialization hook for netbox-branching's merge/revert engine. - - ``ObjectChange.apply()`` normally uses ``DeserializedObject.save()``, which - calls ``Model.save_base(raw=True)`` — bypassing our ``save()`` override and - all ``post_save`` signals. That means ``create_model()`` never runs and the - physical table is never created in the destination schema. - - By implementing this classmethod the apply engine calls our version instead, - returning a wrapper whose ``save()`` invokes the full ``CustomObjectType.save()`` - lifecycle (signals included) so that the table is created as a side effect of - replaying the ObjectChange. - - ``object_type`` is cleared before saving so the ``custom_object_type_post_save_handler`` - can re-create and link it correctly in the destination schema, avoiding any FK - mismatch between the branch and main ``ObjectType`` pks. + """Branching merge/revert hook — replays through the real ``save()``. + + The default ``DeserializedObject.save()`` does ``save_base(raw=True)``, + which skips signals and our ``save()`` override, so the dynamic table + never gets created in the destination schema. This wrapper runs the + full lifecycle. ``object_type`` is nulled only when the FK doesn't + resolve (mirrors ``clean_fields``) so normal merges don't churn the + ContentType/ObjectType pair. """ inner = _deserialize_object(cls, data, pk=pk) @@ -1747,10 +1488,15 @@ def save(self, using=None, **kwargs): # Snapshot before modifying so that diff()['pre'] records the # current state rather than showing all fields as None on revert. self.object.snapshot() - # Clear the ObjectType FK — it may not exist in main yet. - # custom_object_type_post_save_handler re-sets it after INSERT. - self.object.object_type = None - self.object.object_type_id = None + # Only null the ObjectType FK when it doesn't resolve in the + # destination schema; custom_object_type_post_save_handler will + # then rebuild the link via get_or_create after INSERT. + if ( + self.object.object_type_id is not None + and not ObjectType.objects.filter(pk=self.object.object_type_id).exists() + ): + self.object.object_type = None + self.object.object_type_id = None self.object.save() # Re-apply any M2M data (tags, etc.) that was stripped during deserialization. if self._inner.m2m_data: @@ -1761,29 +1507,14 @@ def save(self, using=None, **kwargs): return _SchemaAwareDeserialized(inner) def clean_fields(self, exclude=None): - """ - Tolerate a stale ``object_type`` FK on instances built by - netbox-branching's DELETE-undo path. - - ``CustomObjectType.delete()`` destroys the related - ``core_objecttype`` row to satisfy ChangeDiff's PROTECT FK. - When the same COT is later restored by a branch revert, the - standard ``DeserializedObject`` + ``full_clean`` path validates - ``object_type_id`` against a row that no longer exists and - raises ``ValidationError``. - - Nulling the dangling FK here lets validation pass. The - ``custom_object_type_post_save_handler`` then sees - ``created=True`` on the resulting INSERT and rebuilds the - ContentType/ObjectType pair via ``get_or_create``, restoring - the link with a fresh pk. This intentionally does *not* - preserve the original pk — any cross-branch audit data that - referenced the old pk was already invalidated when the COT - was deleted, so a fresh pk is the honest representation of - the restored state. - - Only acts when the FK genuinely doesn't resolve, so normal - lifecycle saves are unaffected. + """Tolerate a stale ``object_type`` FK on revert (DELETE-undo path). + + ``delete()`` destroys the core_objecttype row to satisfy ChangeDiff's + PROTECT FK; revert then re-inserts the COT, but full_clean would fail + validating the dangling FK. We null it here so validation passes; + ``custom_object_type_post_save_handler`` rebuilds the ContentType/ + ObjectType pair via get_or_create on the resulting INSERT. The new + pk is intentional — the old one's audit refs were already invalidated. """ if ( self.object_type_id is not None @@ -1805,8 +1536,7 @@ def save(self, *args, **kwargs): self.clear_model_cache(self.id) def delete(self, *args, **kwargs): - # Clear the model cache for this CustomObjectType (across all branches — - # the COT itself is going away, so every branch's cached class becomes stale). + # COT is going away — every branch's cached class is stale. self.clear_model_cache(self.id, all_branches=True) model = self.get_model() @@ -1852,21 +1582,16 @@ def delete(self, *args, **kwargs): pre_delete.connect(handle_deleted_object) with schema_conn.schema_editor() as schema_editor: - # Drop through tables before the main table (they have FKs pointing to it). - # Pass schema_conn so existence checks run against the active branch's - # schema, not main's — otherwise inside a branch we'd skip drops that - # need to happen (or attempt drops on tables that don't exist there). + # Drop through tables before the main table (FKs). Existence checks + # use schema_conn so they target the active branch's schema. for through_model in getattr(model, '_through_models', []): if _table_exists(through_model._meta.db_table, conn=schema_conn): schema_editor.delete_model(through_model) schema_editor.delete_model(model) - # Unregister the model and its through-models from Django's app registry so - # that subsequent ORM operations (e.g. deleting a related device) do not try - # to query the now-dropped table and receive a - # "relation 'custom_objects_' does not exist" error. - # Use _global_lock to prevent a concurrent get_model() call from racing - # against this de-registration and re-adding the model mid-cleanup. + # Unregister from apps.all_models so cascade-delete doesn't query the + # dropped table. _global_lock guards against a concurrent get_model() + # racing and re-registering mid-cleanup. with self._global_lock: model_name = model.__name__.lower() if model_name in apps.all_models.get(APP_LABEL, {}): @@ -1877,12 +1602,9 @@ def delete(self, *args, **kwargs): if through_name in apps.all_models.get(APP_LABEL, {}): del apps.all_models[APP_LABEL][through_name] - # Clear Django's internal relation/field caches so the removed model is no - # longer discovered during cascade-delete collector traversal. apps.clear_cache() - # Re-clear the model cache to remove re-cached model from get_model - # (across all branches — see comment at the top of this method). + # Re-clear in case anything re-cached during cleanup. self.clear_model_cache(self.id, all_branches=True) @@ -1895,30 +1617,20 @@ def custom_object_type_post_save_handler(sender, instance, created, **kwargs): app_label=APP_LABEL, model=content_type_name ) - # The snapshot here is for the *second* save (object_type assignment) - # below — not for the create that just fired this handler. Without it, - # the second save's ObjectChange would mark every field as changed - # instead of just the object_type FK. + # Snapshot for the second save below (the object_type assignment). + # Without this, its ObjectChange would mark every field as changed. instance.snapshot() instance.object_type = ct instance.save() def _rename_objectchange_field_key(fi, old_name, new_name): - """ - Rename a JSON key in all ObjectChange records for CustomObject instances of - this field's type, reflecting a field rename from *old_name* to *new_name*. - - Updates both ``prechange_data`` and ``postchange_data`` in the ObjectChange - table, and ``original``/``modified``/``current`` in netbox-branching's - ChangeDiff table when that plugin is installed. + """Rewrite *old_name* → *new_name* JSON keys in ObjectChange (and + ChangeDiff when netbox-branching is installed) for this field's COT. - Field names are validated with ``^[a-z0-9_]+$`` so string formatting of the - column names here is safe against SQL injection. - - This runs inside the same ``transaction.atomic()`` block as - ``CustomObjectTypeField.save()``, so it rolls back cleanly if the enclosing - transaction is aborted. + Runs inside ``CustomObjectTypeField.save()``'s atomic so it rolls back + cleanly. JSON column names are literals and field names are validated + against ``^[a-z0-9_]+$`` — safe to interpolate. """ cot = fi.custom_object_type model = cot.get_model() @@ -1937,9 +1649,8 @@ def _rename_objectchange_field_key(fi, old_name, new_name): 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' 'WHERE changed_object_type_id = %s AND {col} ? %s' ) - # Wrap in a savepoint so a ProgrammingError (e.g. core_objectchange - # unexpectedly missing from the active schema) doesn't poison the outer - # transaction. Mirrors the ChangeDiff guard below. + # Savepoint contains the failure cleanly, then we re-raise so the outer + # field save aborts — silently logging would leave audit data inconsistent. try: with transaction.atomic(using=conn.alias): with conn.cursor() as cursor: @@ -1949,30 +1660,25 @@ def _rename_objectchange_field_key(fi, old_name, new_name): [old_name, new_name, old_name, ct.id, old_name], ) except ProgrammingError: - logger.warning( - '_rename_objectchange_field_key: ObjectChange schema mismatch; ' - 'audit data for %r may not reflect rename to %r', + logger.error( + '_rename_objectchange_field_key: ObjectChange schema mismatch ' + 'rewriting %r -> %r; aborting rename', old_name, new_name, exc_info=True, ) + raise logger.debug('_rename_objectchange_field_key: %r -> %r for %s', old_name, new_name, ct) try: - from netbox_branching.models import ChangeDiff # noqa: F401 — presence check only + from netbox_branching.models import ChangeDiff # noqa: F401 except ImportError: - return # netbox-branching not installed; nothing more to update + return cd_sql = ( 'UPDATE netbox_branching_changediff ' 'SET {col} = ({col} - %s) || jsonb_build_object(%s, {col}->%s) ' 'WHERE object_type_id = %s AND {col} IS NOT NULL AND {col} ? %s' ) - # Wrap in a savepoint so a ProgrammingError (table/column missing due to - # netbox-branching schema mismatch) doesn't poison the outer merge - # transaction — every subsequent statement on the same connection would - # otherwise raise InFailedSqlTransaction. Other DatabaseError subclasses - # are allowed to propagate so a real bug aborts the rename instead of - # silently corrupting audit data. try: with transaction.atomic(using=conn.alias): with conn.cursor() as cursor: @@ -1982,11 +1688,12 @@ def _rename_objectchange_field_key(fi, old_name, new_name): [old_name, new_name, old_name, ct.id, old_name], ) except ProgrammingError: - logger.warning( - '_rename_objectchange_field_key: ChangeDiff schema mismatch; ' - 'audit data for %r may not reflect rename to %r', + logger.error( + '_rename_objectchange_field_key: ChangeDiff schema mismatch ' + 'rewriting %r -> %r; aborting rename', old_name, new_name, exc_info=True, ) + raise class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedModel): @@ -3078,15 +2785,11 @@ def through_model_name(self): @classmethod def deserialize_object(cls, data, pk=None): - """ - Custom deserialization hook for netbox-branching's merge/revert engine. + """Branching merge/revert hook — replays through the real ``save()``. - Same problem as ``CustomObjectType.deserialize_object``: the default - ``DeserializedObject.save(raw=True)`` bypasses ``CustomObjectTypeField.save()``, - so the physical column is never added to the custom object table. - - This wrapper calls the real ``save()`` so that ``add_field`` runs as a side - effect of replaying the CREATE ObjectChange during a merge. + Same shape as ``CustomObjectType.deserialize_object``: the default + ``DeserializedObject.save(raw=True)`` skips our ``save()``, so the + column never gets added. This wrapper runs the full lifecycle. """ inner = _deserialize_object(cls, data, pk=pk) @@ -3107,32 +2810,22 @@ def save(self, using=None, **kwargs): def save(self, *args, **kwargs): is_new = self._state.adding - # Use the branch connection when operating inside a branch so that schema - # editor operations target the branch schema rather than main. schema_conn = _get_schema_connection() - # Auto-assign schema_id for new fields that don't have one yet. - # Increments the monotonic counter on the parent CustomObjectType so that IDs are - # never reused, even after a field is deleted. The UniqueConstraint on - # (schema_id, custom_object_type) is the safety net against races; a concurrent - # writer would get an IntegrityError and must retry. - # Note: bulk_create() bypasses save() entirely, so auto-assignment will NOT fire for - # fields created via CustomObjectTypeField.objects.bulk_create(...). Always set - # schema_id explicitly when using bulk_create. + # Auto-assign schema_id from the parent's monotonic counter. IDs are + # never reused, even after a field is deleted; the + # UniqueConstraint(schema_id, custom_object_type) is the race safety net. + # bulk_create() bypasses save() — callers must set schema_id explicitly. if self._state.adding and self.schema_id is None: - # transaction.atomic() must target the same connection that the - # SELECT FOR UPDATE runs against; in a branch context the branching - # router routes CustomObjectType to the branch connection, so we - # pin both the atomic block and the queryset to schema_conn.alias. + # Atomic and queryset pinned to schema_conn — the branching router + # routes CustomObjectType writes to the branch connection. with transaction.atomic(using=schema_conn.alias): cot = CustomObjectType.objects.using(schema_conn.alias).select_for_update().get( pk=self.custom_object_type_id ) new_schema_id = cot.next_schema_id + 1 - # Use update() rather than save() to avoid dispatching post_save on - # CustomObjectType, which would clear the model cache prematurely. - # The model cache must remain valid until this field's own save() calls - # get_model() below (to contribute the new field and alter the DB table). + # update() avoids post_save → clear_model_cache; the cache must + # remain valid until this field's own get_model() below. CustomObjectType.objects.using(schema_conn.alias).filter( pk=self.custom_object_type_id ).update(next_schema_id=new_schema_id) @@ -3141,17 +2834,13 @@ def save(self, *args, **kwargs): field_type = FIELD_TYPE_CLASS[self.type]() model = self.custom_object_type.get_model() - # Wrap the schema mutation, audit-key rewrite, cache_timestamp bump, and - # parent save() in a single atomic so a failure between the DDL and the - # field row save can't leave audit data rewritten but the field record - # un-persisted (or vice versa). Nests inside the schema_id allocation - # block above as a savepoint. + # Schema mutation + audit-key rewrite + cache bump + parent save() share + # one atomic so a failure between DDL and row save can't leave audit + # data rewritten but the field record un-persisted (or vice versa). with transaction.atomic(using=schema_conn.alias): with schema_conn.schema_editor() as schema_editor: if self._state.adding: if self.is_polymorphic: - # Polymorphic Object: add content_type + object_id columns + index. - # Polymorphic MultiObject: create through table with content_type + object_id. if self.type == CustomFieldTypeChoices.TYPE_OBJECT: field_type.add_polymorphic_object_columns(self, model, schema_editor) elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: @@ -3160,10 +2849,8 @@ def save(self, *args, **kwargs): _schema_add_field(self, model, schema_editor, schema_conn) _apply_deferred_co_field(self) else: - # Polymorphic fields: renames and type changes are rejected by clean(). - # Non-schema attributes (label, description, …) may still change here. - # If clean() was bypassed and a rename slipped through, raise rather - # than silently leaving DB columns / through table out of sync. + # Polymorphic renames/type changes are rejected in clean(); + # raise here if one slipped through to avoid silent column drift. if self.is_polymorphic or self._original_is_polymorphic: if self.name != self._original_name: raise ValidationError( @@ -3172,12 +2859,8 @@ def save(self, *args, **kwargs): else: _schema_alter_field(self.original, self, model, schema_editor, schema_conn) - # When the field is renamed, update ObjectChange / ChangeDiff JSON keys so - # historical audit records and branch diffs stay consistent with the new - # name. Combined with the netbox-branching ObjectChange field migrator - # registered in branching.objectchange_field_migrator, this lets later - # replays (whether iterative undo or squash undo) resolve any data key — - # old or new name — to the field's current name on the model. + # Rewrite historical audit-data keys so any future replay can + # resolve old or new name to the current field name. if ( not self._state.adding and not self.is_polymorphic @@ -3185,9 +2868,7 @@ def save(self, *args, **kwargs): ): _rename_objectchange_field_key(self, self._original_name, self.name) - # Ensure FK constraints are properly created for OBJECT fields. Decide - # whether one is needed inside the atomic so any in-progress rollback - # also discards the decision. + # FK-constraint decision inside the atomic so a rollback discards it too. should_ensure_fk = False if self.type == CustomFieldTypeChoices.TYPE_OBJECT and not self.is_polymorphic: if self._state.adding: @@ -3207,18 +2888,16 @@ def save(self, *args, **kwargs): ) should_ensure_fk = type_changed_to_object or related_object_changed or on_delete_changed - # Clear and refresh the model cache for this CustomObjectType when a field is modified self.custom_object_type.clear_model_cache(self.custom_object_type.id) - # Update parent's cache_timestamp to invalidate cache across all workers. - # snapshot() must be called first so that change logging has a correct pre-state; - # without it, diff()['pre'] would set ALL fields to None during branch revert. + # Bump cache_timestamp to invalidate other workers. snapshot() first + # so change logging records a correct pre-state. self.custom_object_type.snapshot() self.custom_object_type.save(update_fields=['cache_timestamp']) super().save(*args, **kwargs) - # Ensure FK constraints AFTER the transaction commits to avoid "pending trigger events" errors + # FK constraint runs AFTER commit to avoid "pending trigger events". if should_ensure_fk: _on_delete = self.on_delete_behavior _field_name = self.name @@ -3236,23 +2915,11 @@ def ensure_constraint(): transaction.on_commit(ensure_constraint) - # On rename, ``_schema_alter_field`` calls contribute_to_class for both - # the old and new field names on the same model class, leaving the - # _meta in a corrupt state. Force a no_cache regeneration so the - # next access sees a clean model. This regeneration ALSO triggers - # apps.clear_cache() at the end of get_model(), which cascades into - # AppConfig.get_models() and re-caches every COT — but on rename - # that's tolerable because the rename doesn't affect related COTs' - # FK reverse relations. - # - # For non-rename changes (new field, attribute toggle, etc.) the - # cache_timestamp bump done above is sufficient: the next get_model() - # call detects the timestamp mismatch and regenerates lazily. We - # skip both the regeneration AND the search-index re-register here - # so that the cache invalidations performed by signal handlers — most - # importantly the eviction of related COTs in ``clear_cache_on_field_save`` - # for an OBJECT-type field — are not undone by the apps.clear_cache() - # cascade re-caching every COT in sight. + # On rename, _schema_alter_field calls contribute_to_class twice on the + # same class — force a no_cache regeneration so _meta is clean. Non- + # rename changes lean on cache_timestamp for lazy invalidation; we skip + # the apps.clear_cache() cascade so signal-driven cache evictions (e.g. + # clear_cache_on_field_save for OBJECT fields) survive. renamed = ( not self._state.adding and not self.is_polymorphic @@ -3274,7 +2941,6 @@ def ensure_constraint(): def delete(self, *args, **kwargs): field_type = FIELD_TYPE_CLASS[self.type]() model = self.custom_object_type.get_model() - # Use the branch connection when operating inside a branch. schema_conn = _get_schema_connection() with schema_conn.schema_editor() as schema_editor: @@ -3286,27 +2952,21 @@ def delete(self, *args, **kwargs): else: _schema_remove_field(self, model, schema_editor, schema_conn=schema_conn) - # Clear the model cache for this CustomObjectType when a field is deleted self.custom_object_type.clear_model_cache(self.custom_object_type.id) - # Update parent's cache_timestamp to invalidate cache across all workers. - # snapshot() must be called first so that change logging has a correct pre-state. + # snapshot() first so change logging records a correct pre-state. self.custom_object_type.snapshot() self.custom_object_type.save(update_fields=['cache_timestamp']) super().delete(*args, **kwargs) - # Regenerate and re-register the model so the app registry no longer includes - # the removed field. During squash revert the squash strategy may try to query - # CO rows (model.objects.get(pk=...)) after undoing this field but before undoing - # the CO itself. If the stale model class is still in the app registry it will - # include the now-absent column in its SELECT, causing ProgrammingError. + # Regenerate so the apps registry no longer holds a class with the + # removed column — squash revert may SELECT via the model class between + # field-undo and CO-undo, and a stale class would emit ProgrammingError. updated_model = self.custom_object_type.get_model() - # Reregister SearchIndex with new set of searchable fields self.custom_object_type.register_custom_object_search_index(updated_model) - # Reindex all objects of this type since a searchable field was removed if self.search_weight > 0: _cot_id = self.custom_object_type_id transaction.on_commit(lambda: ReindexCustomObjectTypeJob.enqueue(cot_id=_cot_id)) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index bd7f11ce..fdf419c4 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -49,8 +49,7 @@ def _make_request(user): return request -# Provisioning timeout for branch tests. Defaults to 30 s, which is generous -# for local laptops but can be tight on shared CI runners — override via the +# Provisioning timeout for branch tests. Override via the # ``NETBOX_CO_BRANCH_PROVISION_TIMEOUT`` env var (seconds) when CI flakes. BRANCH_PROVISION_TIMEOUT = float( os.environ.get('NETBOX_CO_BRANCH_PROVISION_TIMEOUT', '30') diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 48eb20c9..90c60221 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -894,6 +894,35 @@ def test_custom_validators_slug_key_case_insensitive(self): instance.full_clean() +class M2MSerializationRegressionTestCase(CustomObjectsTestCase, TestCase): + """Guards the ``through._meta.auto_created = model`` opt-in in + ``MultiObjectFieldType.after_model_generation`` — without it, Django's + JSON serializer skips the M2M and merge replay zeroes through-table rows. + """ + + def test_serialize_object_includes_m2m_values(self): + cot = self.create_custom_object_type(name="M2MSerialize", slug="m2m-serialize") + site_ct = self.get_site_object_type() + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, + ) + self.create_custom_object_type_field( + cot, name="sites", label="Sites", type="multiobject", + related_object_type=site_ct, + ) + model = cot.get_model() + site_model = site_ct.model_class() + site_a = site_model.objects.create(name="A", slug="a") + site_b = site_model.objects.create(name="B", slug="b") + + obj = model.objects.create(name="obj") + obj.sites.set([site_a, site_b]) + + data = obj.serialize_object() + self.assertIn("sites", data) + self.assertEqual(set(data["sites"]), {site_a.pk, site_b.pk}) + + class RelatedNameTestCase(CustomObjectsTestCase, TestCase): """Tests for the related_name field on Object and MultiObject fields.""" From 23b8e3346d310d59cf3e8a2926051cd0c7eda547 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 13:18:31 -0700 Subject: [PATCH 054/115] polymorphic fields --- netbox_custom_objects/__init__.py | 28 +- netbox_custom_objects/branching.py | 27 + netbox_custom_objects/field_types.py | 140 ++- netbox_custom_objects/models.py | 197 +++- netbox_custom_objects/tests/base.py | 6 +- netbox_custom_objects/tests/test_branching.py | 843 ++++++++++++++++++ 6 files changed, 1192 insertions(+), 49 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 488dec36..5de69a73 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -30,6 +30,11 @@ # access recomputes with the full set of COT models. _app_ready = False +# Guards ``super().ready()`` against the duplicate-registration check in +# NetBox's ``register_serializer_resolver`` (triggered by ``serializer_resolver`` +# on this PluginConfig). +_super_ready_called = False + def _migration_started(sender, **kwargs): """Signal handler for pre_migrate - sets the migration flag.""" @@ -246,6 +251,15 @@ def should_skip_dynamic_model_creation(): # Always clear the recursion flag _checking_migrations = False + def _call_super_ready_once(self): + """Call ``super().ready()`` once; subsequent calls are no-ops. + ``register_serializer_resolver`` rejects duplicates.""" + global _super_ready_called + if _super_ready_called: + return + super().ready() + _super_ready_called = True + def ready(self): # Install the thread-safe apps.clear_cache wrapper before any dynamic # model is registered (must happen exactly once, before get_model() runs). @@ -285,6 +299,14 @@ def ready(self): ) register_branching_resolver(supports_branching_resolver) register_objectchange_field_migrator(objectchange_field_migrator) + # Optional dependency resolver — skipped silently on older + # netbox-branching that doesn't expose the hook yet. + try: + from netbox_branching.utilities import register_dependency_resolver + from .branching import co_polymorphic_dependency_resolver + register_dependency_resolver(co_polymorphic_dependency_resolver) + except ImportError: + pass except ImportError: pass @@ -299,7 +321,7 @@ def ready(self): # Skip database calls if dynamic models can't be created yet if self.should_skip_dynamic_model_creation(): - super().ready() + self._call_super_ready_once() return try: @@ -311,7 +333,7 @@ def ready(self): except (ProgrammingError, OperationalError): # DB schema is incomplete (unapplied migrations). Skip dynamic # model registration — it will happen after migrations finish. - super().ready() + self._call_super_ready_once() return # Signal that ready() has fully completed. get_models() checks this flag @@ -325,7 +347,7 @@ def ready(self): from django.apps import apps as django_apps django_apps.clear_cache() - super().ready() + self._call_super_ready_once() def get_model(self, model_name, require_ready=True): self.apps.check_apps_ready() diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index b09d02af..0d480e12 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -35,3 +35,30 @@ def objectchange_field_migrator(model, data): if resolve is None: return None return resolve(data) + + +def co_polymorphic_dependency_resolver(model, data, changed_objects): + """Tell squash that a CO CREATE depends on its polymorphic-M2M field CREATEs. + + The polymorphic M2M lives on a ``PolymorphicM2MDescriptor`` (not a Django + field), so squash's default FK/GFK introspection sees no edge between the + CO's postchange_data and the ``CustomObjectTypeField`` rows. Without it + squash may apply the CO before the field — the through table doesn't + exist yet. The sidecar carries field PKs so we don't need to look them + up in main (they aren't there yet during a branch-only merge). + """ + from .constants import APP_LABEL + from .models import POLY_M2M_SIDECAR_KEY + + meta = getattr(model, '_meta', None) + if meta is None or meta.app_label != APP_LABEL: + return () + entries = data.get(POLY_M2M_SIDECAR_KEY) or () + if not entries: + return () + label = f'{APP_LABEL}.customobjecttypefield' + return [ + (label, entry['pk']) + for entry in entries + if isinstance(entry, dict) and entry.get('pk') is not None + ] diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index b67bcfe9..5fa6f20b 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -764,6 +764,11 @@ def add_polymorphic_object_columns(self, field_instance, model, schema_editor): on_delete=models.SET_NULL, related_name="+", db_column=f"{field_instance.name}_content_type_id", + # No DB-level FK to django_content_type — same reasoning as the + # polymorphic-MULTIOBJECT through (avoids the AccessExclusiveLock + # branching merge needs to release + lets test teardown TRUNCATE + # django_content_type without manual CASCADE). + db_constraint=False, ) ct_field.contribute_to_class(model, ct_field_name) schema_editor.add_field(model, ct_field) @@ -1461,10 +1466,10 @@ def get_polymorphic_through_model(self, field_instance, source_model_string): "app_label": APP_LABEL, "apps": apps, "managed": True, - "unique_together": (("source", "content_type", "object_id"),), + "unique_together": (("source", "content_type_id", "object_id"),), "indexes": [ models.Index( - fields=["content_type", "object_id"], + fields=["content_type_id", "object_id"], name=_safe_index_name( f"co_{field_instance.custom_object_type_id}" f"_{field_instance.name}_pgfk" @@ -1474,6 +1479,13 @@ def get_polymorphic_through_model(self, field_instance, source_model_string): }, ) + # content_type_id is a plain integer, not a ContentType FK. An ORM FK + # would (a) install a PG trigger needing AccessExclusiveLock on + # django_content_type during CREATE TABLE → deadlocks netbox-branching + # merge (default-conn DDL vs branch-conn changed_object_type lookups), + # and (b) make Django's collector traverse the through when an + # ObjectType is deleted, querying a table revert has already dropped. + # The descriptor tolerates orphan content_type_ids in _get_objects. attrs = { "__module__": "netbox_custom_objects.models", "Meta": meta, @@ -1484,12 +1496,7 @@ def get_polymorphic_through_model(self, field_instance, source_model_string): related_name="+", db_column="source_id", ), - "content_type": models.ForeignKey( - "contenttypes.ContentType", - on_delete=models.CASCADE, - related_name="+", - db_column="content_type_id", - ), + "content_type_id": models.PositiveBigIntegerField(db_column="content_type_id"), "object_id": models.PositiveBigIntegerField(db_column="object_id"), } @@ -1619,7 +1626,12 @@ def _get_objects(self): # Build a lookup map: (ct_id, obj_id) → object, preserving row order below. obj_map: dict[tuple, object] = {} for ct_id, obj_ids in by_ct.items(): - ct = ContentType.objects.get_for_id(ct_id) + try: + ct = ContentType.objects.get_for_id(ct_id) + except ContentType.DoesNotExist: + # Orphan row — content_type_id has no DB FK (see + # get_polymorphic_through_model). Skip. + continue model_class = ct.model_class() if model_class is None: continue @@ -1643,58 +1655,116 @@ def count(self): def exists(self): return self._get_through_model().objects.filter(source_id=self.instance.pk).exists() + def _fire_m2m_changed(self, action, pk_set): + """Emit ``m2m_changed`` so change-logging records the polymorphic write. + + Polymorphic targets span multiple model classes, so pass the through + as ``model`` (NetBox's handler only checks action + pk_set truthiness). + Postchange data comes from ``CustomObject.serialize_object``. + """ + if pk_set is not None and not pk_set: + return + through = self._get_through_model() + db = router.db_for_write(through, instance=self.instance) + m2m_changed.send( + sender=through, + instance=self.instance, + action=action, + reverse=False, + model=through, + pk_set=pk_set, + using=db, + ) + def add(self, *objs): through = self._get_through_model() + candidate_keys = [] for obj in objs: ct = ContentType.objects.get_for_model(obj) - through.objects.get_or_create( + candidate_keys.append((ct.pk, obj.pk)) + if not candidate_keys: + return + self._fire_m2m_changed('pre_add', {pk for _, pk in candidate_keys}) + added = set() + for ct_pk, obj_pk in candidate_keys: + _, created = through.objects.get_or_create( source_id=self.instance.pk, - content_type_id=ct.pk, - object_id=obj.pk, + content_type_id=ct_pk, + object_id=obj_pk, ) + if created: + added.add(obj_pk) + if added: + self._fire_m2m_changed('post_add', added) def remove(self, *objs): through = self._get_through_model() + candidate_keys = [] for obj in objs: ct = ContentType.objects.get_for_model(obj) - through.objects.filter( + candidate_keys.append((ct.pk, obj.pk)) + if not candidate_keys: + return + self._fire_m2m_changed('pre_remove', {pk for _, pk in candidate_keys}) + removed = set() + for ct_pk, obj_pk in candidate_keys: + n, _ = through.objects.filter( source_id=self.instance.pk, - content_type_id=ct.pk, - object_id=obj.pk, + content_type_id=ct_pk, + object_id=obj_pk, ).delete() + if n: + removed.add(obj_pk) + if removed: + self._fire_m2m_changed('post_remove', removed) def clear(self): - self._get_through_model().objects.filter(source_id=self.instance.pk).delete() + through = self._get_through_model() + existing = set( + through.objects.filter(source_id=self.instance.pk) + .values_list('object_id', flat=True) + ) + if not existing: + return + # Django's m2m_changed contract: pre_clear with pk_set=None. + self._fire_m2m_changed('pre_clear', None) + through.objects.filter(source_id=self.instance.pk).delete() + self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): if clear: self.clear() self.add(*objs) - else: - # Diff-based replacement: add new, remove old. Matches Django's - # standard ManyRelatedManager.set(clear=False) behaviour. - objs = tuple(objs) - through = self._get_through_model() - existing = { - (ct_id, obj_id) - for ct_id, obj_id in through.objects.filter(source_id=self.instance.pk) - .values_list("content_type_id", "object_id") - } - # Pre-compute (ct_id, obj_pk) once per object to avoid duplicate CT lookups. - new_items = [ - (ContentType.objects.get_for_model(obj).pk, obj.pk, obj) for obj in objs - ] - new_keys = {(ct_id, obj_pk) for ct_id, obj_pk, _ in new_items} - to_add = [obj for ct_id, obj_pk, obj in new_items if (ct_id, obj_pk) not in existing] - to_remove = existing - new_keys - if to_add: - self.add(*to_add) + return + + # Diff-based replacement: add new, remove old. Matches Django's + # standard ManyRelatedManager.set(clear=False) behaviour. + objs = tuple(objs) + through = self._get_through_model() + existing = { + (ct_id, obj_id) + for ct_id, obj_id in through.objects.filter(source_id=self.instance.pk) + .values_list("content_type_id", "object_id") + } + # Pre-compute (ct_id, obj_pk) once per object to avoid duplicate CT lookups. + new_items = [ + (ContentType.objects.get_for_model(obj).pk, obj.pk, obj) for obj in objs + ] + new_keys = {(ct_id, obj_pk) for ct_id, obj_pk, _ in new_items} + to_add = [obj for ct_id, obj_pk, obj in new_items if (ct_id, obj_pk) not in existing] + to_remove = existing - new_keys + if to_add: + self.add(*to_add) + if to_remove: + remove_pks = {obj_id for _, obj_id in to_remove} + self._fire_m2m_changed('pre_remove', remove_pks) for ct_id, obj_id in to_remove: through.objects.filter( source_id=self.instance.pk, content_type_id=ct_id, object_id=obj_id, ).delete() + self._fire_m2m_changed('post_remove', remove_pks) def __iter__(self): return iter(self.all()) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index dd899fdb..629cb32b 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -104,6 +104,45 @@ def _table_exists(table_name, conn=None): '_deferred_co_field_data', default=None ) +# Sidecar key listing polymorphic-M2M fields in a serialized CO dict, as +# ``[{'name': ..., 'pk': ...}, ...]``. ``pk`` lets the squash dependency +# resolver in ``branching.py`` produce a CO → field CREATE edge without +# looking the field up in main (it isn't there yet during a branch-only +# merge). ``name`` lets ``deserialize_object`` map rows to through tables. +POLY_M2M_SIDECAR_KEY = '__nco_poly_m2m_fields__' + + +def _apply_poly_m2m_rows(schema_conn, through_table, co_pk, rows): + """Insert polymorphic M2M *rows* into *through_table* on *schema_conn*, + set-style (clear existing for *co_pk* first). ContentType resolved by + natural key, also via *schema_conn*. + """ + alias = schema_conn.alias + with schema_conn.cursor() as cursor: + cursor.execute( + f'DELETE FROM "{through_table}" WHERE source_id = %s', [co_pk], + ) + for row in rows: + ct_label = row.get('content_type') + obj_id = row.get('object_id') + if not ct_label or obj_id is None: + continue + try: + app_label, model_name = ct_label.split('.', 1) + ct = ContentType.objects.using(alias).get( + app_label=app_label, model=model_name, + ) + except (ValueError, ContentType.DoesNotExist) as exc: + logger.debug( + 'poly M2M replay: ct %r unresolved (%s)', ct_label, exc, + ) + continue + cursor.execute( + f'INSERT INTO "{through_table}" ' + '(source_id, content_type_id, object_id) VALUES (%s, %s, %s)', + [co_pk, ct.pk, obj_id], + ) + def _get_schema_connection(): """Active branch's connection if any, else the default — so DDL targets the right schema.""" @@ -596,14 +635,29 @@ def deserialize_object(cls, data, pk=None): if pk is not None: obj.pk = pk m2m_data = {} + # Polymorphic M2M data: keys named by serialize_object's sidecar + # (POLY_M2M_SIDECAR_KEY) — explicit so we don't rely on _field_objects + # (empty when squash replays the CO CREATE before its field CREATE) + # or value-shape guessing. + poly_m2m_field_names = { + entry['name'] + for entry in (resolved.get(POLY_M2M_SIDECAR_KEY) or ()) + if isinstance(entry, dict) and entry.get('name') + } + poly_m2m_data = {} field_names = {f.name for f in fresh_model._meta.get_fields()} for attr, value in resolved.items(): + if attr == POLY_M2M_SIDECAR_KEY: + continue # Tags via the standard NetBox path (Tag rows are looked up by name). if attr == 'tags' and is_taggable(fresh_model): tag_model = apps.get_model('extras', 'Tag') m2m_data['tags'] = list(tag_model.objects.filter(name__in=value or [])) continue + if attr in poly_m2m_field_names: + poly_m2m_data[attr] = value or [] + continue if attr not in field_names: # Unknown attribute (likely a removed field) — preserve it as a # Python attribute so downstream code (e.g. _deferred_co_field_data) @@ -656,6 +710,15 @@ def save(self, using=None, **_kwargs): 'deserialize_object: deferred M2M %r on %s pk=%s (table absent)', accessor, table_name, obj_pk, exc_info=True, ) + # Replay polymorphic M2M directly via the through, pinned to + # _using. Bypasses manager.add() → m2m_changed → + # handle_changed_object, which can route across DB aliases. + schema_conn_local = connections[_using] + for field_name, rows in poly_m2m_data.items(): + through_table = ( + f'{USER_TABLE_DATABASE_NAME_PREFIX}{cot_id}_{field_name}' + ) + _apply_poly_m2m_rows(schema_conn_local, through_table, obj_pk, rows) # Stash full data for deferred column updates (squash ordering fix). deferred = _deferred_co_field_data.get() if deferred is None: @@ -691,11 +754,97 @@ def clean(self): if validators: run_validators(self, validators) + def serialize_object(self, exclude=None): + """Standard serialization plus polymorphic-field metadata. + + For polymorphic MULTIOBJECT fields, also appends + ``[{content_type, object_id}, ...]`` per field (Django's serializer + skips them — the descriptor isn't on ``_meta``). For both polymorphic + OBJECT and MULTIOBJECT, emits a sidecar of ``[{name, pk}, ...]`` so + the squash dependency resolver in ``branching.py`` can order the + field's CREATE before the CO's CREATE — without it, squash would + apply the CO before the columns/through exist. + """ + data = super().serialize_object(exclude=exclude) + field_objects = getattr(type(self), '_field_objects', None) or {} + poly_entries = [] + for fo in field_objects.values(): + field = fo['field'] + if not field.is_polymorphic: + continue + if field.type not in ( + CustomFieldTypeChoices.TYPE_OBJECT, + CustomFieldTypeChoices.TYPE_MULTIOBJECT, + ): + continue + if exclude and field.name in exclude: + continue + poly_entries.append({'name': field.name, 'pk': field.pk}) + if field.type != CustomFieldTypeChoices.TYPE_MULTIOBJECT: + continue + # MULTIOBJECT-only: append the through-table rows. + try: + manager = getattr(self, field.name) + through = manager._get_through_model() + except (AttributeError, LookupError): + continue + rows = list( + through.objects.filter(source_id=self.pk) + .values('content_type_id', 'object_id') + ) + if not rows: + data[field.name] = [] + continue + ct_ids = {r['content_type_id'] for r in rows} + ct_map = { + ct.id: f'{ct.app_label}.{ct.model}' + for ct in ContentType.objects.filter(id__in=ct_ids) + } + data[field.name] = [ + {'content_type': ct_map.get(r['content_type_id']), 'object_id': r['object_id']} + for r in rows + if r['content_type_id'] in ct_map + ] + if poly_entries: + data[POLY_M2M_SIDECAR_KEY] = poly_entries + return data + @property def _generated_table_model(self): # An indication that the model is a generated table model. return True + def delete(self, *args, **kwargs): + # Two prep steps before super() so the deletion collector doesn't + # raise traversing reverse FKs from through models: + # 1. Realign each through's ``source`` FK to ``type(self)`` — + # isinstance(instance, fk.related_model) otherwise sees two + # Table*Model classes and fails. + # 2. Temporarily unregister throughs whose physical table is gone + # (squash revert drops field-CREATEs before CO-CREATEs, so the + # CO's collector hits ``relation does not exist`` otherwise). + cls = type(self) + prefix = f'through_{cls._meta.db_table}' + registry = apps.all_models.get(APP_LABEL, {}) + existing_tables = connection.introspection.table_names() + hidden = {} + for name, through in list(registry.items()): + if not name.startswith(prefix): + continue + if through._meta.db_table not in existing_tables: + hidden[name] = registry.pop(name) + continue + try: + source_field = through._meta.get_field('source') + except FieldDoesNotExist: + continue + source_field.remote_field.model = cls + source_field.__dict__.pop('related_model', None) + try: + return super().delete(*args, **kwargs) + finally: + registry.update(hidden) + @property def clone_fields(self): """ @@ -1083,15 +1232,18 @@ def _after_model_generation(self, attrs, model): # Polymorphic GFK: no through model. pass elif field_instance.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: - # Always create a fresh polymorphic through for THIS context. - # Reusing apps.get_model would hand back another context's - # through and require mutating its source FK — the very race - # this fix eliminates. - field_type_obj = FIELD_TYPE_CLASS[CustomFieldTypeChoices.TYPE_MULTIOBJECT]() - source_model_str = f"{APP_LABEL}.{model.__name__}" - through_model = field_type_obj.get_polymorphic_through_model( - field_instance, source_model_str - ) + # Reuse the apps-registered through if present; otherwise + # create one. Avoids parallel through instances diverging + # from apps.all_models (collector class-identity mismatch). + _apps = model._meta.apps + try: + through_model = _apps.get_model(APP_LABEL, field_instance.through_model_name) + except LookupError: + field_type_obj = FIELD_TYPE_CLASS[CustomFieldTypeChoices.TYPE_MULTIOBJECT]() + source_model_str = f"{APP_LABEL}.{model.__name__}" + through_model = field_type_obj.get_polymorphic_through_model( + field_instance, source_model_str + ) source_field = through_model._meta.get_field("source") source_field.remote_field.model = model source_field.related_model = model @@ -1587,7 +1739,31 @@ def delete(self, *args, **kwargs): for through_model in getattr(model, '_through_models', []): if _table_exists(through_model._meta.db_table, conn=schema_conn): schema_editor.delete_model(through_model) - schema_editor.delete_model(model) + # Django's schema_editor.delete_model(parent) auto-recurses into + # M2M fields' through models when their _meta.auto_created is + # truthy — which we set deliberately in + # MultiObjectFieldType.after_model_generation so Django's JSON + # serializer includes M2M values. That recursion would attempt a + # DROP TABLE for through tables whose actual DB tables are absent + # (e.g. when this delete runs on a COT whose branch-side through + # was already dropped by a revert). Clear auto_created on those + # missing-table throughs for the duration of delete_model(parent) + # and restore it after. + cleared = [] + for field in model._meta.local_many_to_many: + through = field.remote_field.through + if through._meta.auto_created and not _table_exists( + through._meta.db_table, conn=schema_conn, + ): + cleared.append((through, through._meta.auto_created)) + through._meta.auto_created = False + try: + # Flush DEFERRABLE FK triggers so PG doesn't reject the DROP. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') + schema_editor.delete_model(model) + finally: + for through, original in cleared: + through._meta.auto_created = original # Unregister from apps.all_models so cascade-delete doesn't query the # dropped table. _global_lock guards against a concurrent get_model() @@ -3074,6 +3250,7 @@ def clear_cache_on_field_save(sender, instance, **kwargs): try: related_cot = CustomObjectType.objects.get(object_type_id=instance.related_object_type_id) CustomObjectType.clear_model_cache(related_cot.id) + related_cot.snapshot() related_cot.save(update_fields=['cache_timestamp']) except CustomObjectType.DoesNotExist: pass diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index 510a4376..f4e6cb5d 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -11,7 +11,11 @@ from utilities.testing import create_test_user from netbox_custom_objects.constants import APP_LABEL -from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField, _deferred_co_field_data +from netbox_custom_objects.models import ( + CustomObjectType, + CustomObjectTypeField, + _deferred_co_field_data, +) logger = logging.getLogger(__name__) diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index fdf419c4..b95da368 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -20,6 +20,7 @@ from core.models import ObjectType from dcim.models import Site from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType from django.db import connection as main_conn, connections from django.test import RequestFactory, TransactionTestCase from django.urls import reverse @@ -1062,6 +1063,389 @@ def test_multiobject_field_rename_merge_and_revert(self): 'M2M values must survive the round-trip rename → revert', ) + # ── single-field DELETE inside branch ───────────────────────────────── + + def test_single_field_delete_merge_and_revert(self): + """ + Delete one field from a COT in a branch (parent COT intact). + + Exercises ``_schema_remove_field`` via the merge replay engine + without the through-table-already-gone shortcut that + ``test_cot_deleted_in_branch_merge_and_revert`` triggers. Covers + the most direct CRUD gap: per-field delete is the common + production case but was previously only exercised transitively. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site = Site.objects.create(name='Field-Delete Site', slug='field-delete-site') + cot = CustomObjectType.objects.create(name='field_delete_cot', slug='field-delete-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='keep_me', label='Keep', type='text', + ) + scalar_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='drop_me', label='Drop', type='integer', + ) + multi_field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='drop_m2m', label='Drop M2M', + type='multiobject', related_object_type=site_ot, + ) + Model = cot.get_model() + co = Model.objects.create(keep_me='hello', drop_me=7) + co.drop_m2m.set([site]) + + cot_pk = cot.pk + co_pk = co.pk + scalar_field_pk = scalar_field.pk + multi_field_pk = multi_field.pk + through_table = multi_field.through_table_name + co_table = cot.get_database_table_name() + + # Sanity: column + through table exist in main before the merge. + with main_conn.cursor() as cursor: + cols_before = { + c.name for c in main_conn.introspection.get_table_description(cursor, co_table) + } + self.assertIn('drop_me', cols_before) + self.assertIn(through_table, main_conn.introspection.table_names()) + + branch = _provision_branch('Field Delete Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + CustomObjectTypeField.objects.get(pk=scalar_field_pk).delete() + CustomObjectTypeField.objects.get(pk=multi_field_pk).delete() + + # Before merge: main still has both fields and their schema. + self.assertTrue(CustomObjectTypeField.objects.filter(pk=scalar_field_pk).exists()) + self.assertTrue(CustomObjectTypeField.objects.filter(pk=multi_field_pk).exists()) + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + # Fields gone from ORM; physical column + through table gone too. + self.assertFalse(CustomObjectTypeField.objects.filter(pk=scalar_field_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=multi_field_pk).exists()) + with main_conn.cursor() as cursor: + cols_after = { + c.name for c in main_conn.introspection.get_table_description(cursor, co_table) + } + self.assertNotIn('drop_me', cols_after, 'drop_me column must be removed from main') + self.assertIn('keep_me', cols_after, 'keep_me column must remain in main') + self.assertNotIn( + through_table, main_conn.introspection.table_names(), + f'Through-table {through_table!r} must be dropped after merge', + ) + + # The retained field's data on the CO must still be readable. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.keep_me, 'hello') + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + + # Both fields are restored, column and through table re-created. + self.assertTrue(CustomObjectTypeField.objects.filter(pk=scalar_field_pk).exists()) + self.assertTrue(CustomObjectTypeField.objects.filter(pk=multi_field_pk).exists()) + with main_conn.cursor() as cursor: + cols_reverted = { + c.name for c in main_conn.introspection.get_table_description(cursor, co_table) + } + self.assertIn('drop_me', cols_reverted, 'drop_me column must be restored after revert') + self.assertIn( + through_table, main_conn.introspection.table_names(), + f'Through-table {through_table!r} must be restored after revert', + ) + + # ── polymorphic OBJECT field merge/revert ───────────────────────────── + + def test_polymorphic_object_field_merge_and_revert(self): + """ + Polymorphic OBJECT field (GenericForeignKey backed by content_type + + object_id columns). None of the existing branching tests cover this + path, and the per-(cot, branch) through-model refactor changed + through-model handling — polymorphic GFK fields have no through but + do create dedicated columns that need branch-aware DDL routing. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + + with event_tracking(self.request): + site = Site.objects.create(name='Poly Site', slug='poly-site') + + branch = _provision_branch('Poly OBJ Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='poly_obj_cot', slug='poly-obj-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='target', + label='Target', + type='object', + is_polymorphic=True, + ) + field.related_object_types.set([site_ot]) + Model = cot.get_model() + co = Model.objects.create( + target_content_type=ContentType.objects.get_for_model(Site), + target_object_id=site.pk, + ) + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.target_object_id, site.pk) + self.assertEqual( + co_main.target_content_type_id, + ContentType.objects.get_for_model(Site).pk, + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + + # ── polymorphic MULTIOBJECT field merge/revert ──────────────────────── + + def test_polymorphic_multiobject_field_merge_and_revert(self): + """ + Polymorphic MULTIOBJECT field — through table has + (source_id, content_type_id, object_id) columns. Asserts the + through-table schema lifecycle (create in branch / merge / revert) + AND the M2M data round-trip: rows added via + ``PolymorphicManyToManyManager`` in the branch must appear in main + after merge. + + Data preservation requires three pieces: + - ``PolymorphicManyToManyManager`` fires ``m2m_changed``, triggering + an UPDATE ObjectChange on the parent CO. + - ``CustomObject.serialize_object`` includes polymorphic M2M values + (Django's serializer skips them — the descriptor isn't a real + M2M field on ``_meta``). + - ``CustomObject.deserialize_object`` replays the polymorphic + values via the descriptor after the CO row saves. + """ + site_ot = ObjectType.objects.get(app_label='dcim', model='site') + site_ct = ContentType.objects.get_for_model(Site) + + with event_tracking(self.request): + site_a = Site.objects.create(name='Poly M2M A', slug='poly-m2m-a') + site_b = Site.objects.create(name='Poly M2M B', slug='poly-m2m-b') + + branch = _provision_branch('Poly M2M Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='poly_m2m_cot', slug='poly-m2m-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, + name='targets', + label='Targets', + type='multiobject', + is_polymorphic=True, + ) + field.related_object_types.set([site_ot]) + Model = cot.get_model() + co = Model.objects.create() + co.snapshot() + co.targets.add(site_a) + co.targets.add(site_b) + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + through_table = field.through_table_name + + # ── merge ───────────────────────────────────────────────────────── + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertIn( + through_table, main_conn.introspection.table_names(), + 'Polymorphic through-table must exist in main after merge', + ) + + # Read the through table directly: the polymorphic descriptor's + # query helpers are exercised elsewhere; here we just want to + # confirm the rows landed in main. + with main_conn.cursor() as cursor: + cursor.execute( + f'SELECT object_id FROM "{through_table}" ' + 'WHERE source_id = %s AND content_type_id = %s ORDER BY object_id', + [co_pk, site_ct.pk], + ) + rows = [r[0] for r in cursor.fetchall()] + self.assertEqual( + rows, sorted([site_a.pk, site_b.pk]), + 'Polymorphic M2M rows added in branch must land in main after merge', + ) + + # ── revert ──────────────────────────────────────────────────────── + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertFalse(CustomObjectTypeField.objects.filter(pk=field_pk).exists()) + self.assertNotIn( + through_table, main_conn.introspection.table_names(), + 'Polymorphic through-table must be dropped after revert', + ) + + # ── self-referential OBJECT field merge/revert ──────────────────────── + + def test_self_referential_object_field_merge_and_revert(self): + """ + Self-referential OBJECT field (FK to the same COT). The generated + model resolves the FK target back to itself in + ``after_model_generation``; we verify that path works across merge. + """ + branch = _provision_branch('Self Ref OBJ Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='self_ref_obj_cot', slug='self-ref-obj-cot') + self_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=cot.get_table_model_name(cot.id).lower(), + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='parent', label='Parent', + type='object', related_object_type=self_ot, + ) + Model = cot.get_model() + root = Model.objects.create(label='root') + child = Model.objects.create(label='child', parent_id=root.pk) + + cot_pk = cot.pk + root_pk, child_pk = root.pk, child.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + child_main = Model.objects.get(pk=child_pk) + self.assertEqual(child_main.parent_id, root_pk) + self.assertEqual(child_main.label, 'child') + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + # ── self-referential MULTIOBJECT field merge/revert ─────────────────── + + def test_self_referential_multiobject_field_merge_and_revert(self): + """ + Self-referential MULTIOBJECT — through table with both source and + target FKs pointing at the same dynamic CO model. Exercises the + ``_is_self_referential`` path in + ``MultiObjectFieldType.after_model_generation``. + """ + branch = _provision_branch('Self Ref M2M Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='self_ref_m2m_cot', slug='self-ref-m2m-cot') + self_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=cot.get_table_model_name(cot.id).lower(), + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='peers', label='Peers', + type='multiobject', related_object_type=self_ot, + ) + Model = cot.get_model() + a = Model.objects.create(label='a') + b = Model.objects.create(label='b') + c = Model.objects.create(label='c') + a.peers.set([b, c]) + + cot_pk = cot.pk + a_pk, b_pk, c_pk = a.pk, b.pk, c.pk + through_table = field.through_table_name + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + a_main = Model.objects.get(pk=a_pk) + self.assertEqual( + set(a_main.peers.values_list('pk', flat=True)), + {b_pk, c_pk}, + 'Self-referential M2M values must survive merge', + ) + self.assertIn(through_table, main_conn.introspection.table_names()) + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + self.assertNotIn(through_table, main_conn.introspection.table_names()) + + # ── cross-COT FK created entirely inside branch ─────────────────────── + + def test_cross_cot_fk_branch_creates_both_merge_and_revert(self): + """ + Branch creates COT B, then COT A with a FK pointing at B, then a CO + of type A referencing a CO of type B. Merge must apply the COT + creates and field create in an order that respects the FK + dependency, and the data must survive. + """ + branch = _provision_branch('Cross COT Branch', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot_b = CustomObjectType.objects.create(name='cross_b_cot', slug='cross-b-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot_b, name='b_label', label='B Label', type='text', + ) + b_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=cot_b.get_table_model_name(cot_b.id).lower(), + ) + + cot_a = CustomObjectType.objects.create(name='cross_a_cot', slug='cross-a-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot_a, name='a_label', label='A Label', type='text', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot_a, name='b_ref', label='B Ref', + type='object', related_object_type=b_ot, + ) + + BModel = cot_b.get_model() + b_inst = BModel.objects.create(b_label='hello B') + AModel = cot_a.get_model() + a_inst = AModel.objects.create(a_label='hello A', b_ref_id=b_inst.pk) + + cot_a_pk, cot_b_pk = cot_a.pk, cot_b.pk + a_pk, b_pk = a_inst.pk, b_inst.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + AModel = CustomObjectType.objects.get(pk=cot_a_pk).get_model() + BModel = CustomObjectType.objects.get(pk=cot_b_pk).get_model() + self.assertEqual(BModel.objects.get(pk=b_pk).b_label, 'hello B') + a_main = AModel.objects.get(pk=a_pk) + self.assertEqual(a_main.b_ref_id, b_pk) + self.assertEqual(a_main.a_label, 'hello A') + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_a_pk).exists()) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_b_pk).exists()) + # ── Concrete test classes (one per merge strategy) ──────────────────────────── @@ -1948,3 +2332,462 @@ class SequentialRenameSquashTestCase(SequentialRenameTestCase, TransactionTestCa def test_sequential_renames_alpha_beta_gamma_merge(self): self._run_sequential_rename_merge('seq_squash_cot', 'seq-squash-cot') + + +# ── Missing field-type coverage (iterative only) ────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class MissingFieldTypesTestCase(BranchingTestBase, TransactionTestCase): + """ + Field types that ``test_comprehensive_merge_and_revert`` doesn't cover: + longtext, date (separate from datetime), URL, JSON, multiselect. + + Iterative only — strategy-specific bugs in these field types would still + surface in the comprehensive squash test once they're added there. This + standalone class keeps the round-trip times manageable while filling out + the field-type matrix. + """ + + def test_merge_and_revert_for_extra_field_types(self): + with event_tracking(self.request): + choice_set = CustomFieldChoiceSet.objects.create( + name='Multi Statuses', + extra_choices=[['a', 'A'], ['b', 'B'], ['c', 'C']], + ) + + branch = _provision_branch('Extra Types Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='extra_types_cot', slug='extra-types-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='longtext_field', label='Long Text', type='longtext', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='date_field', label='Date', type='date', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='url_field', label='URL', type='url', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='json_field', label='JSON', type='json', + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='multiselect_field', label='Multi', + type='multiselect', choice_set=choice_set, + ) + Model = cot.get_model() + co = Model.objects.create( + longtext_field='line1\nline2', + date_field=datetime.date(2026, 5, 21), + url_field='https://example.com/path', + json_field={'k': 'v', 'n': 1, 'list': [1, 2, 3]}, + multiselect_field=['a', 'c'], + ) + + cot_pk, co_pk = cot.pk, co.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_main = Model.objects.get(pk=co_pk) + self.assertEqual(co_main.longtext_field, 'line1\nline2') + self.assertEqual(co_main.date_field, datetime.date(2026, 5, 21)) + self.assertEqual(co_main.url_field, 'https://example.com/path') + self.assertEqual(co_main.json_field, {'k': 'v', 'n': 1, 'list': [1, 2, 3]}) + self.assertEqual(sorted(co_main.multiselect_field), ['a', 'c']) + + branch.revert(user=self.user, commit=True) + self.assertFalse(CustomObjectType.objects.filter(pk=cot_pk).exists()) + + +# ── Field attribute changes & COT update (iterative only) ───────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class FieldAttributeChangesTestCase(BranchingTestBase, TransactionTestCase): + """ + Application-layer field attribute changes that the existing tests don't + cover individually: COT-level updates, field type change, primary swap, + and required toggle. Iterative only — these don't hit strategy-specific + code paths. + """ + + def test_cot_metadata_update_merge(self): + """COT name / verbose_name / description / version edited inside branch.""" + with event_tracking(self.request): + cot = CustomObjectType.objects.create( + name='meta_cot', slug='meta-cot', + verbose_name='Original Name', description='original', + version='1.0.0', + ) + + cot_pk = cot.pk + branch = _provision_branch('Meta Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + branch_cot = CustomObjectType.objects.get(pk=cot_pk) + branch_cot.snapshot() + branch_cot.verbose_name = 'Updated Name' + branch_cot.description = 'updated description' + branch_cot.version = '1.1.0' + branch_cot.save() + + # Main hasn't seen the change yet. + cot.refresh_from_db() + self.assertEqual(cot.verbose_name, 'Original Name') + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cot.refresh_from_db() + self.assertEqual(cot.verbose_name, 'Updated Name') + self.assertEqual(cot.description, 'updated description') + self.assertEqual(cot.version, '1.1.0') + + def test_field_type_change_text_to_integer_merge(self): + """text → integer field type change across merge. + + The CO has a value that's parseable as both; the column type + change replaces ``VARCHAR`` with ``INTEGER`` via ``alter_field``. + """ + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='retype_cot', slug='retype-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='value', label='Value', type='text', + ) + Model = cot.get_model() + # Use a value that parses as both text and integer so the + # USING cast applied by alter_field can succeed. + co = Model.objects.create(value='42') + + cot_pk, field_pk, co_pk = cot.pk, field.pk, co.pk + branch = _provision_branch('Retype Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.type = 'integer' + f.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + field_main = CustomObjectTypeField.objects.get(pk=field_pk) + self.assertEqual(field_main.type, 'integer') + + # PostgreSQL column type must be integer. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_table = cot_main.get_database_table_name() + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT data_type FROM information_schema.columns ' + 'WHERE table_name = %s AND column_name = %s', + [co_table, 'value'], + ) + data_type = cursor.fetchone()[0] + self.assertEqual(data_type, 'integer') + + # CO value survived the cast. + co_main = cot_main.get_model().objects.get(pk=co_pk) + self.assertEqual(co_main.value, 42) + + def test_primary_field_swap_merge(self): + """Switch which field is ``primary``; __str__ must follow.""" + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='primary_cot', slug='primary-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='code', label='Code', type='text', primary=True, + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='title', label='Title', type='text', primary=False, + ) + Model = cot.get_model() + co = Model.objects.create(code='ABC', title='Widget') + self.assertEqual(str(co), 'ABC') + + cot_pk = cot.pk + co_pk = co.pk + branch = _provision_branch('Primary Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f_code = CustomObjectTypeField.objects.get(custom_object_type=cot, name='code') + f_code.snapshot() + f_code.primary = False + f_code.save() + f_title = CustomObjectTypeField.objects.get(custom_object_type=cot, name='title') + f_title.snapshot() + f_title.primary = True + f_title.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_main = cot_main.get_model().objects.get(pk=co_pk) + # __str__ now follows the newly-primary 'title' field. + self.assertEqual(str(co_main), 'Widget') + + def test_field_required_toggle_merge(self): + """Toggle a field's required flag from False to True across merge. + + ``required`` is a form-layer attribute in this plugin — every field + constructor in ``field_types.py`` hardcodes ``null=True, blank=True`` + on the model field, so the DB column stays nullable regardless of + ``required``. This test pins both halves of that contract: + ``required=True`` does survive the merge as an ORM attribute (form + validation will reject empty values), but the underlying column does + NOT become NOT NULL. + """ + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='required_cot', slug='required-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='note', label='Note', type='text', + required=False, + ) + + cot_pk, field_pk = cot.pk, field.pk + branch = _provision_branch('Required Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.required = True + f.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertTrue( + CustomObjectTypeField.objects.get(pk=field_pk).required, + 'required=True ORM flag must survive the merge', + ) + + # DB column stays nullable — required is enforced at the form layer only. + cot_main = CustomObjectType.objects.get(pk=cot_pk) + co_table = cot_main.get_database_table_name() + with main_conn.cursor() as cursor: + cursor.execute( + 'SELECT is_nullable FROM information_schema.columns ' + 'WHERE table_name = %s AND column_name = %s', + [co_table, 'note'], + ) + is_nullable = cursor.fetchone()[0] + self.assertEqual( + is_nullable, 'YES', + 'required=True must NOT produce NOT NULL — this plugin enforces ' + 'required at the form layer only (field_types.py hardcodes null=True).', + ) + + +# ── Tags + journal entries survive merge ────────────────────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class TagsAndJournalTestCase(BranchingTestBase, TransactionTestCase): + """ + Tags use a separate code path in ``CustomObject.deserialize_object`` via + the ``is_taggable`` branch. Journal entries are NetBox infrastructure + used by NetBoxModel subclasses. Neither is exercised by the rest of the + branching suite. + """ + + def test_co_with_tags_survives_merge(self): + from extras.models import Tag + + with event_tracking(self.request): + tag_a = Tag.objects.create(name='Branch Tag A', slug='branch-tag-a') + tag_b = Tag.objects.create(name='Branch Tag B', slug='branch-tag-b') + cot = CustomObjectType.objects.create(name='tagged_cot', slug='tagged-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + + cot_pk = cot.pk + branch = _provision_branch('Tagged Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + Model = cot.get_model() + co = Model.objects.create(label='tagged') + co.tags.set([tag_a, tag_b]) + + co_pk = co.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_main = Model.objects.get(pk=co_pk) + self.assertEqual( + set(co_main.tags.values_list('name', flat=True)), + {'Branch Tag A', 'Branch Tag B'}, + 'Tags assigned in branch must appear on the merged CO', + ) + + def test_co_with_journal_entry_survives_merge(self): + """Journal entries created against a CO in branch arrive in main on merge.""" + from extras.models import JournalEntry + + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='journal_cot', slug='journal-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + + cot_pk = cot.pk + branch = _provision_branch('Journal Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + Model = cot.get_model() + co = Model.objects.create(label='journaled') + JournalEntry.objects.create( + assigned_object=co, + created_by=self.user, + kind='info', + comments='Note added in branch', + ) + + co_pk = co.pk + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + Model = CustomObjectType.objects.get(pk=cot_pk).get_model() + co_ct = ContentType.objects.get_for_model(Model) + entries = JournalEntry.objects.filter( + assigned_object_type=co_ct, assigned_object_id=co_pk, + ) + self.assertEqual( + list(entries.values_list('comments', flat=True)), + ['Note added in branch'], + ) + + +# ── ChoiceSet lifecycle, search_weight, sync-then-merge ─────────────────────── + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class ChoiceSetSearchLifecycleTestCase(BranchingTestBase, TransactionTestCase): + """Misc lifecycle gaps: ChoiceSet mutation, search_weight changes, + sync→edit→merge chains.""" + + def test_choice_set_choices_mutated_in_branch_merge(self): + """Add a new choice to a ChoiceSet inside a branch; merge to main.""" + with event_tracking(self.request): + cs = CustomFieldChoiceSet.objects.create( + name='Mutable Choices', + extra_choices=[['x', 'X'], ['y', 'Y']], + ) + cot = CustomObjectType.objects.create(name='cs_cot', slug='cs-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='c', label='Choice', + type='select', choice_set=cs, + ) + + cs_pk = cs.pk + branch = _provision_branch('CS Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + cs_branch = CustomFieldChoiceSet.objects.get(pk=cs_pk) + cs_branch.snapshot() + cs_branch.extra_choices = [['x', 'X'], ['y', 'Y'], ['z', 'Z']] + cs_branch.save() + Model = cot.get_model() + Model.objects.create(c='z') # uses the new choice + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + cs_main = CustomFieldChoiceSet.objects.get(pk=cs_pk) + keys = {pair[0] for pair in cs_main.extra_choices} + self.assertIn('z', keys, 'New choice must be present in main after merge') + + def test_field_search_weight_change_merge(self): + """Changing ``search_weight`` is a non-DDL field update. It must + survive merge as an ORM-level change; reindexing happens via + ``ReindexCustomObjectTypeJob`` which runs out-of-band.""" + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='sw_cot', slug='sw-cot') + field = CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', + type='text', search_weight=10, + ) + + field_pk = field.pk + branch = _provision_branch('SW Branch', 'iterative', self.user) + branch_request = _make_request(self.user) + + with activate_branch(branch), event_tracking(branch_request): + f = CustomObjectTypeField.objects.get(pk=field_pk) + f.snapshot() + f.search_weight = 100 + f.save() + + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + self.assertEqual( + CustomObjectTypeField.objects.get(pk=field_pk).search_weight, 100, + ) + + def test_sync_then_branch_edit_then_merge_lifecycle(self): + """ + Full lifecycle: main creates state → sync pulls it into branch → + branch edits a CO → merge brings the edit back. No standalone + test covers state persistence across sync and merge in one branch. + """ + branch = _provision_branch('Lifecycle Branch', 'iterative', self.user) + + # Main creates a COT + CO *after* the branch was provisioned. + with event_tracking(self.request): + cot = CustomObjectType.objects.create(name='lifecycle_cot', slug='lifecycle-cot') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='label', label='Label', type='text', + ) + Model = cot.get_model() + co = Model.objects.create(label='initial') + + co_pk = co.pk + cot_pk = cot.pk + + # Sync the branch so it sees the COT+CO. + branch.sync(user=self.user, commit=True) + branch.refresh_from_db() + + # Branch edits the CO. + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + BranchModel = CustomObjectType.objects.get(pk=cot_pk).get_model() + branch_co = BranchModel.objects.get(pk=co_pk) + branch_co.snapshot() + branch_co.label = 'edited after sync' + branch_co.save() + + # Main hasn't seen the edit yet. + co.refresh_from_db() + self.assertEqual(co.label, 'initial') + + # Merge. + branch.merge(user=self.user, commit=True) + branch.refresh_from_db() + self.assertEqual(branch.status, BranchStatusChoices.MERGED) + + co.refresh_from_db() + self.assertEqual( + co.label, 'edited after sync', + 'Edit made after sync must propagate to main on merge', + ) From b59097dfd047f7de9966a60a4d50e17f05b85d79 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 14:06:03 -0700 Subject: [PATCH 055/115] use signal handler for branching --- netbox_custom_objects/__init__.py | 17 +++-- netbox_custom_objects/branching.py | 105 +++++++++++++++++++++++------ netbox_custom_objects/models.py | 16 +++-- 3 files changed, 107 insertions(+), 31 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 5de69a73..fb945bb7 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -299,12 +299,19 @@ def ready(self): ) register_branching_resolver(supports_branching_resolver) register_objectchange_field_migrator(objectchange_field_migrator) - # Optional dependency resolver — skipped silently on older - # netbox-branching that doesn't expose the hook yet. + # Subscribe to the squash dependency-graph signal so CO-specific + # edges (M2M targets, polymorphic-M2M sidecar) get added before + # topological ordering. Skipped silently on older + # netbox-branching that doesn't expose the signal yet. try: - from netbox_branching.utilities import register_dependency_resolver - from .branching import co_polymorphic_dependency_resolver - register_dependency_resolver(co_polymorphic_dependency_resolver) + from netbox_branching.merge_strategies.squash import ( + squash_dependency_graph_built, + ) + from .branching import add_custom_object_dependencies + squash_dependency_graph_built.connect( + add_custom_object_dependencies, + weak=False, + ) except ImportError: pass except ImportError: diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index 0d480e12..c9de8601 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -3,7 +3,6 @@ Registered from ``__init__.ready()`` only when netbox-branching is installed. """ - def supports_branching_resolver(model): """Mark CustomObject M2M through models as branchable. @@ -37,28 +36,94 @@ def objectchange_field_migrator(model, data): return resolve(data) -def co_polymorphic_dependency_resolver(model, data, changed_objects): - """Tell squash that a CO CREATE depends on its polymorphic-M2M field CREATEs. +def _collect_co_refs(model_class, data): + """Return ``(app.model, pk)`` refs from CO-specific shapes in *data*. - The polymorphic M2M lives on a ``PolymorphicM2MDescriptor`` (not a Django - field), so squash's default FK/GFK introspection sees no edge between the - CO's postchange_data and the ``CustomObjectTypeField`` rows. Without it - squash may apply the CO before the field — the through table doesn't - exist yet. The sidecar carries field PKs so we don't need to look them - up in main (they aren't there yet during a branch-only merge). + Covers: + * Local M2M target lists (squash's default ``_get_fk_references`` only + walks FK / GFK fields, so M2M targets — including self-referential + ones — are invisible to it). + * Every ``CustomObjectTypeField`` on the CO's model. A CO INSERT needs + the field's column (scalar) or through table (M2M) to exist first; + without these edges squash may apply the CO CREATE before the field + CREATEs. Pulled from the model class's ``_field_objects`` plus the + polymorphic ``POLY_M2M_SIDECAR_KEY`` (which carries field PKs in the + ObjectChange payload even when ``_field_objects`` isn't available). """ from .constants import APP_LABEL from .models import POLY_M2M_SIDECAR_KEY - meta = getattr(model, '_meta', None) - if meta is None or meta.app_label != APP_LABEL: - return () + refs = set() + if not data: + return refs + + for field in model_class._meta.local_many_to_many: + values = data.get(field.name) + if not values: + continue + rel_meta = field.related_model._meta + label = f'{rel_meta.app_label}.{rel_meta.model_name}' + for pk in values: + if isinstance(pk, int): + refs.add((label, pk)) + + field_label = f'{APP_LABEL}.customobjecttypefield' + for fo in (getattr(model_class, '_field_objects', None) or {}).values(): + cotf = fo.get('field') if isinstance(fo, dict) else None + if cotf is not None and getattr(cotf, 'pk', None) is not None: + refs.add((field_label, cotf.pk)) + entries = data.get(POLY_M2M_SIDECAR_KEY) or () - if not entries: - return () - label = f'{APP_LABEL}.customobjecttypefield' - return [ - (label, entry['pk']) - for entry in entries - if isinstance(entry, dict) and entry.get('pk') is not None - ] + for entry in entries: + if isinstance(entry, dict) and entry.get('pk') is not None: + refs.add((field_label, entry['pk'])) + + return refs + + +def add_custom_object_dependencies(sender, collapsed_changes, **kwargs): + """Extend squash's dependency graph with CO-specific edges. + + Walks every collapsed change for a CO model and mirrors squash's four + edge-direction rules (UPDATE→DELETE, UPDATE→CREATE, CREATE→CREATE, + DELETE→DELETE) using ``_collect_co_refs`` instead of the FK/GFK walker. + """ + from .constants import APP_LABEL + + deletes_map = {} + updates_map = {} + creates_map = {} + for key, cc in collapsed_changes.items(): + action = cc.final_action.value if cc.final_action else None + if action == 'create': + creates_map[key] = cc + elif action == 'update': + updates_map[key] = cc + elif action == 'delete': + deletes_map[key] = cc + + for cc in collapsed_changes.values(): + meta = getattr(cc.model_class, '_meta', None) + if meta is None or meta.app_label != APP_LABEL: + continue + action = cc.final_action.value if cc.final_action else None + + if action == 'update': + for ref in _collect_co_refs(cc.model_class, cc.prechange_data): + if ref in deletes_map: + deletes_map[ref].depends_on.add(cc.key) + cc.depended_by.add(ref) + for ref in _collect_co_refs(cc.model_class, cc.postchange_data): + if ref in creates_map: + cc.depends_on.add(ref) + creates_map[ref].depended_by.add(cc.key) + elif action == 'create': + for ref in _collect_co_refs(cc.model_class, cc.postchange_data): + if ref != cc.key and ref in creates_map: + cc.depends_on.add(ref) + creates_map[ref].depended_by.add(cc.key) + elif action == 'delete': + for ref in _collect_co_refs(cc.model_class, cc.prechange_data): + if ref != cc.key and ref in deletes_map: + deletes_map[ref].depends_on.add(cc.key) + cc.depended_by.add(ref) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 629cb32b..d9f6a879 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -834,12 +834,16 @@ def delete(self, *args, **kwargs): if through._meta.db_table not in existing_tables: hidden[name] = registry.pop(name) continue - try: - source_field = through._meta.get_field('source') - except FieldDoesNotExist: - continue - source_field.remote_field.model = cls - source_field.__dict__.pop('related_model', None) + for fk_name in ('source', 'target'): + try: + field = through._meta.get_field(fk_name) + except FieldDoesNotExist: + continue + remote_meta = getattr(field.remote_field.model, '_meta', None) + if remote_meta is None or remote_meta.label != cls._meta.label: + continue + field.remote_field.model = cls + field.__dict__.pop('related_model', None) try: return super().delete(*args, **kwargs) finally: From e7af7b4f1598a0742e6a73f90afa83c1970a3887 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 14:27:25 -0700 Subject: [PATCH 056/115] cleanup --- netbox_custom_objects/branching.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index c9de8601..d19dced3 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -3,6 +3,7 @@ Registered from ``__init__.ready()`` only when netbox-branching is installed. """ + def supports_branching_resolver(model): """Mark CustomObject M2M through models as branchable. From c164269a09764af5ddc38cd5dfc59d94780944d1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 15:08:24 -0700 Subject: [PATCH 057/115] cleanup --- AGENTS.md | 4 ++-- docs/branching.md | 19 ------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6e8d9377..9171d3d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ Defer all version pins to `pyproject.toml` and `netbox_custom_objects/__init__.p │ ├── tables.py — django-tables2 tables for list views. │ ├── template_content.py — PluginTemplateExtension registrations. │ ├── urls.py — Web UI URL routing (80+ routes). -│ ├── utilities.py — AppsProxy, generate_model(), get_viewname(), is_in_branch(). +│ ├── utilities.py — AppsProxy, generate_model(), get_viewname(). │ ├── views.py — All UI views. │ ├── api/ │ │ ├── serializers.py — get_serializer_class() + static serializers. @@ -134,7 +134,7 @@ Multi-object fields create a separate through table (`custom_objects__= 4.6.2 @@ -16,18 +12,3 @@ These requirements are only enforced when `netbox_branching` is present in `PLUG > [!TIP] > If you have any questions the best place to start is on the GitHub [discussions](https://github.com/netboxlabs/netbox-custom-objects/discussions). If you are a NetBox Labs customer, you can also contact support. - -## Custom Object Types and Custom Object Type Fields - -Custom Object Types and Custom Object Type fields can be created, updated and deleted on branches, however the changes made on branches will be applied in main. This allows Custom Objects and Branching to be used safely alongside each other, but users should be aware of what this means. - -- When you are in an activated branch any creates, updates and deletes you perform on Custom Object Types and Custom Object Type Fields will not show up in the Diff or Changes Ahead views -- Although you're in an activated branch, these changes will be made directly to main -- Typically it will be NetBox admins who are altering Custom Object Types and Custom Object Type Fields - we recommend that you experiment in your staging instance until you are satisfied with the modelling and then move them into prod - -## Custom Objects - -Changes to Custom Objects on branches are disallowed. - -- When in an activated branch, users will still be able to see the available Custom Object Types and any Custom Objects that were brought into the branch upon branch creation, but will not be able to interact with them to affect changes on the branch. -- This approach was chosen to make sure that users can safely use both Custom Objects and Branching together, while we are working on fuller support. \ No newline at end of file From d82308b036b1835af48d55a2ffe32008359b096d Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 15:16:06 -0700 Subject: [PATCH 058/115] remove check framework --- docs/branching.md | 2 +- netbox_custom_objects/__init__.py | 3 -- netbox_custom_objects/checks.py | 54 ------------------------------- pyproject.toml | 5 --- 4 files changed, 1 insertion(+), 63 deletions(-) delete mode 100644 netbox_custom_objects/checks.py diff --git a/docs/branching.md b/docs/branching.md index 1254914e..61362a6d 100644 --- a/docs/branching.md +++ b/docs/branching.md @@ -5,7 +5,7 @@ When using Custom Objects together with NetBox Branching, the following minimum - NetBox >= 4.6.2 - netbox-branching >= 1.1.0 -These requirements are only enforced when `netbox_branching` is present in `PLUGINS`. If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. A Django system check will fail at startup if the combination is misconfigured. +If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. > [!NOTE] > We are working towards full support for Custom Objects on branches. Keep an eye on the GitHub issues for updates ahead of future releases. diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index fb945bb7..88c6fd44 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -265,9 +265,6 @@ def ready(self): # model is registered (must happen exactly once, before get_model() runs). install_clear_cache_suppressor() - # Register Django system checks (import triggers @register). - from . import checks # noqa: F401 - from .models import CustomObjectType from netbox_custom_objects.api.serializers import get_serializer_class diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py deleted file mode 100644 index e5d049d7..00000000 --- a/netbox_custom_objects/checks.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Django system checks. - -Enforces conditional version floors that PluginConfig's static -``min_version``/``max_version`` can't express: when netbox-branching is -installed, NetBox and netbox-branching versions are pinned tighter. -""" - -from importlib.metadata import PackageNotFoundError, version as _pkg_version - -from django.apps import apps -from django.conf import settings -from django.core.checks import Error, register -from packaging.version import InvalidVersion, Version - - -# Version floors enforced only when netbox-branching is installed. -REQUIRED_NETBOX_VERSION = '4.6.2' -REQUIRED_BRANCHING_VERSION = '1.1.0' - - -@register() -def check_branching_compatibility(app_configs, **kwargs): - """Enforce branching-only version floors; no-op without netbox-branching.""" - if not apps.is_installed('netbox_branching'): - return [] - - errors = [] - - try: - netbox_version = Version(settings.RELEASE.version) - if netbox_version < Version(REQUIRED_NETBOX_VERSION): - errors.append(Error( - f'netbox-custom-objects requires NetBox >= {REQUIRED_NETBOX_VERSION} ' - f'when netbox-branching is installed (detected {netbox_version}).', - hint='Upgrade NetBox, or remove netbox-branching from PLUGINS ' - 'if you do not need branching support for custom objects.', - id='netbox_custom_objects.E001', - )) - except (AttributeError, InvalidVersion): - pass # settings.RELEASE missing/unparseable — other checks surface it - - try: - branching_version = Version(_pkg_version('netbox-branching')) - if branching_version < Version(REQUIRED_BRANCHING_VERSION): - errors.append(Error( - f'netbox-custom-objects requires netbox-branching >= ' - f'{REQUIRED_BRANCHING_VERSION} (detected {branching_version}).', - hint=f'Upgrade with: pip install "netbox-branching>={REQUIRED_BRANCHING_VERSION}"', - id='netbox_custom_objects.E002', - )) - except (PackageNotFoundError, InvalidVersion): - pass # editable install without dist-info — skip rather than warn - - return errors diff --git a/pyproject.toml b/pyproject.toml index ed662679..0b23b65b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,11 +32,6 @@ dependencies = [ [project.optional-dependencies] dev = ["check-manifest", "mkdocs", "mkdocs-material", "ruff"] test = ["coverage", "pytest", "pytest-cov"] -# Install with `pip install "netboxlabs-netbox-custom-objects[branching]"` when -# pairing this plugin with netbox-branching. Note: this extra also implies a -# stricter NetBox version requirement (>= 4.6.2), which is enforced at startup -# by netbox_custom_objects.checks rather than declared here. -branching = ["netbox-branching>=1.1.0"] [project.urls] "Homepage" = "https://netboxlabs.com/" From e4120022d0c722833db04554fb59f53c43b6f956 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 15:46:09 -0700 Subject: [PATCH 059/115] cleanup --- netbox_custom_objects/__init__.py | 13 +++--- netbox_custom_objects/field_types.py | 4 -- netbox_custom_objects/models.py | 64 +++++++++++++++++++--------- 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 88c6fd44..688fa772 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -96,12 +96,6 @@ def _heal_mixin_columns(sender, **kwargs): if any(cmd in sys.argv for cmd in ("makemigrations", "collectstatic")): return - # Set the flag *before* running so that subsequent post_migrate firings - # (one per installed app) are no-ops even if the first attempt raises. - # A failure here will not be retried in the same process; operators can - # run 'manage.py upgrade_custom_objects' manually if needed. - _heal_ran = True - try: from netbox_custom_objects.mixin_migration import heal_all_cots # noqa: PLC0415 heal_all_cots(verbosity=kwargs.get("verbosity", 1)) @@ -110,6 +104,13 @@ def _heal_mixin_columns(sender, **kwargs): logging.getLogger(__name__).exception( "upgrade_custom_objects: unexpected error during mixin drift check" ) + # Leave _heal_ran False so a subsequent post_migrate firing (or a + # manual 'manage.py upgrade_custom_objects') gets another attempt. + return + + # Only mark complete on success so a transient failure can be retried by + # the next post_migrate firing in this process. + _heal_ran = True def _patch_object_selector_view(): diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 5fa6f20b..434d99e2 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -1037,10 +1037,6 @@ def __get__(self, instance, cls=None): return CustomManyToManyManager(instance=instance, field_name=self.field.name) - def get_prefetch_queryset(self, instances, queryset=None): - manager = CustomManyToManyManager(instances[0], self.field.name) - return manager.get_prefetch_queryset(instances, queryset) - def is_cached(self, instance): """ Returns True if the field's value has been cached for the given instance. diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index d9f6a879..988fd61b 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -111,11 +111,22 @@ def _table_exists(table_name, conn=None): # merge). ``name`` lets ``deserialize_object`` map rows to through tables. POLY_M2M_SIDECAR_KEY = '__nco_poly_m2m_fields__' +# Serializes the save/restore of TM.post_through_setup in get_model() across +# threads — without this, concurrent generations (e.g. main + branch context) +# can interleave their save/restore and run the original (unpatched) setup +# inside the other call. +_taggable_manager_patch_lock = threading.Lock() + def _apply_poly_m2m_rows(schema_conn, through_table, co_pk, rows): """Insert polymorphic M2M *rows* into *through_table* on *schema_conn*, set-style (clear existing for *co_pk* first). ContentType resolved by natural key, also via *schema_conn*. + + *through_table* originates from ``CustomObjectTypeField.through_table_name``, + which is derived from validated identifiers (the COT id and a field name + matching ``^[a-z0-9]+(_[a-z0-9]+)*$``) — safe to interpolate directly + into SQL. """ alias = schema_conn.alias with schema_conn.cursor() as cursor: @@ -826,7 +837,9 @@ def delete(self, *args, **kwargs): cls = type(self) prefix = f'through_{cls._meta.db_table}' registry = apps.all_models.get(APP_LABEL, {}) - existing_tables = connection.introspection.table_names() + # Branch contexts may have through tables only in the branch schema, so + # introspect via the active schema's connection, not the main one. + existing_tables = _get_schema_connection().introspection.table_names() hidden = {} for name, through in list(registry.items()): if not name.startswith(prefix): @@ -1470,24 +1483,27 @@ def get_model( # Wrap the existing post_through_setup method to handle ValueError exceptions from taggit.managers import TaggableManager as TM - original_post_through_setup = TM.post_through_setup + # TM.post_through_setup is class-level state; serialize concurrent + # generations so save/restore can't interleave across threads. + with _taggable_manager_patch_lock: + original_post_through_setup = TM.post_through_setup - def wrapped_post_through_setup(self, cls): - try: - return original_post_through_setup(self, cls) - except ValueError: - pass + def wrapped_post_through_setup(self, cls): + try: + return original_post_through_setup(self, cls) + except ValueError: + pass - TM.post_through_setup = wrapped_post_through_setup + TM.post_through_setup = wrapped_post_through_setup - try: - model = generate_model( - str(model_name), - (CustomObject, models.Model), - attrs, - ) - finally: - TM.post_through_setup = original_post_through_setup + try: + model = generate_model( + str(model_name), + (CustomObject, models.Model), + attrs, + ) + finally: + TM.post_through_setup = original_post_through_setup # Suppress clear_cache() through the _model_cache write so a re-entrant # get_model() inside register_model → clear_cache → get_models() can hit @@ -2158,6 +2174,13 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # Two distinct "original" mechanisms exist on this model: + # * ``_original_name`` / ``_original_type`` / ``_original_*`` (here) are + # scalar snapshots set on every instance (including freshly constructed + # ones) for cheap before/after comparisons in save(). + # * ``_original`` (set in ``from_db``) is a full instance clone from the + # DB row, used when more than a single attribute is needed; only + # DB-loaded objects have it — see ``original`` property. self._name = self.__dict__.get("name") self._original_name = self.name self._original_type = self.type @@ -2312,7 +2335,8 @@ def clean(self): ) # Check if uniqueness constraint can be applied when changing from non-unique to unique. - # Skip when _original is absent (e.g. during deserialization in branch merge/revert). + # _original is set by from_db only; deserialized objects (branch merge/revert) + # never load from DB and won't have it — guard before touching self.original. if ( self.pk and self.unique @@ -2933,7 +2957,6 @@ def from_db(cls, db, field_names, values): @property def original(self): return self._original - # return self.__class__(**self._loaded_values) @property def through_table_name(self): @@ -3109,7 +3132,10 @@ def ensure_constraint(): updated_model = self.custom_object_type.get_model(no_cache=True) self.custom_object_type.register_custom_object_search_index(updated_model) - # Reindex all objects of this type if search indexing was affected + # Reindex all objects of this type if search indexing was affected. + # self.original (backed by _original) is only set by from_db; the + # `not is_new` branch implies _state.adding is False, which implies + # the row came from the DB, so _original is guaranteed to exist. if is_new: needs_reindex = self.search_weight > 0 else: From 3cc11bd3f78febb3784a0d9ae0324481e10005c0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 16:07:52 -0700 Subject: [PATCH 060/115] cleanup --- netbox_custom_objects/field_types.py | 8 ++++++++ netbox_custom_objects/models.py | 21 ++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 434d99e2..ca999cd1 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -991,6 +991,10 @@ def clear(self): # Django's contract: pre_clear with pk_set=None. self._fire_m2m_changed('pre_clear', None) self.through.objects.filter(source_id=self.instance.pk).delete() + # Deliberate deviation from Django's contract (which fires post_clear + # with pk_set=None): netbox-branching's change-capture needs the actual + # cleared PKs to log the removal. Receivers that follow Django's spec + # and assume pk_set is None on post_clear may treat this as a remove. self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): @@ -1725,6 +1729,10 @@ def clear(self): # Django's m2m_changed contract: pre_clear with pk_set=None. self._fire_m2m_changed('pre_clear', None) through.objects.filter(source_id=self.instance.pk).delete() + # Deliberate deviation from Django's contract (which fires post_clear + # with pk_set=None): netbox-branching's change-capture needs the actual + # cleared PKs to log the removal. Receivers that follow Django's spec + # and assume pk_set is None on post_clear may treat this as a remove. self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 988fd61b..855c38fe 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -575,6 +575,12 @@ def resolve_field_aliases(cls, data): each field's rename history via ObjectChange records. Unknown keys are preserved as-is; on collision the non-None value wins (see ``_set_with_collision_preference``). + + Only top-level keys are rewritten. All current field-name carriers + (scalar columns, FK ``_id`` keys, M2M target lists, polymorphic + ``{content_type, object_id}`` payloads) appear at the top level, so + this is sufficient today. A future field type that stores user-field + names as nested-dict keys would need a recursive walk. """ if not data: return data @@ -932,7 +938,6 @@ class CustomObjectType(NetBoxModel): # Each context owns its through class so the source FK is set once at # generation time and never mutated to follow another context's CO class. _through_model_cache = {} - _model_cache_locks = {} _global_lock = threading.RLock() _ON_DELETE_SQL = { ObjectFieldOnDeleteChoices.CASCADE: "CASCADE", @@ -1068,7 +1073,6 @@ def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): for key in list(cls._through_model_cache): if key[0] == custom_object_type_id: cls._through_model_cache.pop(key, None) - cls._model_cache_locks.pop(custom_object_type_id, None) else: branch_id = cls._active_branch_id() cls._model_cache.pop((custom_object_type_id, branch_id), None) @@ -1076,7 +1080,6 @@ def clear_model_cache(cls, custom_object_type_id=None, *, all_branches=False): else: cls._model_cache.clear() cls._through_model_cache.clear() - cls._model_cache_locks.clear() # Clear Django apps registry cache to ensure newly created models are recognized apps.get_models.cache_clear() @@ -1436,6 +1439,11 @@ def get_model( self.register_custom_object_search_index(model) return model else: + # Only clear the current (cot_id, branch_id) entry. Lazy + # invalidation: each branch context detects its own stale + # timestamp on next access — main's COT save propagates the + # bumped cache_timestamp to branches via change-capture, so + # they'll re-evaluate against their own row independently. self.clear_model_cache(self.id) # Generate the model outside the lock to avoid holding it during expensive operations @@ -1524,9 +1532,12 @@ def wrapped_post_through_setup(self, cls): # Else: branch class stays registered until main is generated — # self-healing on the next main-context get_model() call. - self._after_model_generation(attrs, model) - + # _after_model_generation registers through models in + # _through_model_cache and mutates apps.all_models; hold the lock so + # concurrent get_model() calls for the same (cot_id, branch_id) can't + # interleave their through-model registrations. with self._global_lock: + self._after_model_generation(attrs, model) self._model_cache[(self.id, branch_id)] = (model, self.cache_timestamp) apps.clear_cache() From 4f32f5f7777b1fb48b454418b1a02ff13549a1ca Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 16:25:14 -0700 Subject: [PATCH 061/115] cleanup --- netbox_custom_objects/field_types.py | 27 ++++++++++------ netbox_custom_objects/models.py | 47 +++++++++++++++++++++------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index ca999cd1..566fc162 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -852,6 +852,14 @@ def remove_polymorphic_object_columns(self, field_instance, model, schema_editor # fields, this is the first place to check. The Django source to compare against # is django/db/models/fields/related_managers.py :: ManyRelatedManager. class CustomManyToManyManager(Manager): + """ + Many-related manager for custom M2M fields. + + m2m_changed deviation: ``post_clear`` fires with the cleared PKs instead + of ``None`` so netbox-branching's change-capture can log the removal. + Receivers that branch on ``pk_set is None`` will misread it as removes. + """ + def __init__(self, instance=None, field_name=None): super().__init__() self.instance = instance @@ -988,13 +996,10 @@ def clear(self): ) if not existing: return - # Django's contract: pre_clear with pk_set=None. + # pre_clear follows Django's contract; post_clear deliberately does + # not — see the class docstring. self._fire_m2m_changed('pre_clear', None) self.through.objects.filter(source_id=self.instance.pk).delete() - # Deliberate deviation from Django's contract (which fires post_clear - # with pk_set=None): netbox-branching's change-capture needs the actual - # cleared PKs to log the removal. Receivers that follow Django's spec - # and assume pk_set is None on post_clear may treat this as a remove. self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): @@ -1599,6 +1604,11 @@ class PolymorphicManyToManyManager: Manager for polymorphic many-to-many relationships. Handles objects from multiple model types via a through table with (source_id, content_type_id, object_id) columns. + + m2m_changed deviation: ``post_clear`` fires with the cleared object PKs + instead of ``None`` so netbox-branching's change-capture can log the + removal. Receivers that branch on ``pk_set is None`` will misread it as + removes. """ def __init__(self, instance, field_name, through_model_name): @@ -1726,13 +1736,10 @@ def clear(self): ) if not existing: return - # Django's m2m_changed contract: pre_clear with pk_set=None. + # pre_clear follows Django's contract; post_clear deliberately does + # not — see the class docstring. self._fire_m2m_changed('pre_clear', None) through.objects.filter(source_id=self.instance.pk).delete() - # Deliberate deviation from Django's contract (which fires post_clear - # with pk_set=None): netbox-branching's change-capture needs the actual - # cleared PKs to log the removal. Receivers that follow Django's spec - # and assume pk_set is None on post_clear may treat this as a remove. self._fire_m2m_changed('post_clear', existing) def set(self, objs, clear=False): diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 855c38fe..eb5e9be9 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -111,10 +111,10 @@ def _table_exists(table_name, conn=None): # merge). ``name`` lets ``deserialize_object`` map rows to through tables. POLY_M2M_SIDECAR_KEY = '__nco_poly_m2m_fields__' -# Serializes the save/restore of TM.post_through_setup in get_model() across -# threads — without this, concurrent generations (e.g. main + branch context) -# can interleave their save/restore and run the original (unpatched) setup -# inside the other call. +# Serializes the save/restore of TM.post_through_setup in get_model(). The +# patch needs to be in place during type() class creation, so the lock has to +# span the whole generate_model() — that serialises unrelated COT generations. +# Acceptable for now: the bottleneck is bounded to startup and squash merges. _taggable_manager_patch_lock = threading.Lock() @@ -129,6 +129,8 @@ def _apply_poly_m2m_rows(schema_conn, through_table, co_pk, rows): into SQL. """ alias = schema_conn.alias + inserted = 0 + dropped = 0 with schema_conn.cursor() as cursor: cursor.execute( f'DELETE FROM "{through_table}" WHERE source_id = %s', [co_pk], @@ -137,6 +139,7 @@ def _apply_poly_m2m_rows(schema_conn, through_table, co_pk, rows): ct_label = row.get('content_type') obj_id = row.get('object_id') if not ct_label or obj_id is None: + dropped += 1 continue try: app_label, model_name = ct_label.split('.', 1) @@ -144,15 +147,25 @@ def _apply_poly_m2m_rows(schema_conn, through_table, co_pk, rows): app_label=app_label, model=model_name, ) except (ValueError, ContentType.DoesNotExist) as exc: - logger.debug( - 'poly M2M replay: ct %r unresolved (%s)', ct_label, exc, + logger.warning( + 'poly M2M replay: ct %r unresolved (%s) for %s pk=%s', + ct_label, exc, through_table, co_pk, ) + dropped += 1 continue cursor.execute( f'INSERT INTO "{through_table}" ' '(source_id, content_type_id, object_id) VALUES (%s, %s, %s)', [co_pk, ct.pk, obj_id], ) + inserted += 1 + # All rows dropped — flag it. A CO with poly-M2M data should land with at + # least one row; zero inserts means the replay silently lost data. + if rows and inserted == 0 and dropped > 0: + logger.warning( + 'poly M2M replay: all %d row(s) dropped for %s pk=%s — ' + 'CO will land with empty %s', dropped, through_table, co_pk, through_table, + ) def _get_schema_connection(): @@ -414,8 +427,9 @@ def _rename_or_create_m2m_through(old_fi, new_fi, model, schema_editor, schema_c 'Meta', (), {'db_table': old_through, 'app_label': APP_LABEL, 'managed': True}, ) + temp_name = f'_TempOldThrough_{old_through}' old_through_model = generate_model( - f'_TempOldThrough_{old_through}', + temp_name, (models.Model,), { '__module__': 'netbox_custom_objects.models', @@ -429,7 +443,12 @@ def _rename_or_create_m2m_through(old_fi, new_fi, model, schema_editor, schema_c ), }, ) - schema_editor.alter_db_table(old_through_model, old_through, new_through) + try: + schema_editor.alter_db_table(old_through_model, old_through, new_through) + finally: + # generate_model() registered the temp class in apps.all_models; + # drop it so repeated renames don't leak entries. + apps.all_models.get(APP_LABEL, {}).pop(temp_name.lower(), None) else: # Old through table absent — create the new one from scratch ft = FIELD_TYPE_CLASS[new_fi.type]() @@ -1048,10 +1067,7 @@ def _active_branch_id(): from netbox_branching.contextvars import active_branch except ImportError: return None - try: - branch = active_branch.get() - except LookupError: - return None + branch = active_branch.get() return branch.id if branch is not None else None @classmethod @@ -1429,6 +1445,10 @@ def get_model( branch_id = self._active_branch_id() + # Lock guards the cache check, not the miss → re-cache window. Two + # threads can regenerate the same (cot_id, branch_id) in parallel; + # both produce equivalent classes, so the duplication is wasteful but + # not incorrect. Worth it to avoid serialising all generation. with self._global_lock: if self.is_model_cached(self.id, branch_id) and not no_cache: cached_timestamp = self.get_cached_timestamp(self.id, branch_id) @@ -1722,6 +1742,9 @@ def delete(self, *args, **kwargs): # COT is going away — every branch's cached class is stale. self.clear_model_cache(self.id, all_branches=True) + # Regenerate against the current context so the model used for the + # DDL drop below reflects this branch's column set, not whatever + # stale class an earlier context cached. model = self.get_model() schema_conn = _get_schema_connection() in_branch = schema_conn is not connection From dc8aac7cdd6c7e4e5aa581c0da41bb5872c8cbba Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 22 May 2026 16:46:25 -0700 Subject: [PATCH 062/115] cleanup --- netbox_custom_objects/__init__.py | 109 ++++++++++++++++++------------ netbox_custom_objects/models.py | 21 +++--- 2 files changed, 77 insertions(+), 53 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 688fa772..205dfc4d 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -48,13 +48,34 @@ def _migration_finished(sender, **kwargs): _migrations_checked = None -def _connect_deferred_data_reset_signals(): - """Reset ``_deferred_co_field_data`` at every merge/sync/revert boundary. +# Guards the netbox-branching hook setup against duplicate registration if +# ``ready()`` runs more than once (e.g. test isolation paths that reset the +# app registry). Covers signal connects and the netbox-branching +# ``register_*`` functions, which do not all dedupe internally. +_branching_hooks_registered = False - Connect both pre- and post- so the reset runs even when the operation - raises (post-signals only fire on success). ``weak=False`` keeps the - receiver alive past the end of ``ready()``. + +def _reset_deferred_co_field_data(sender, **kwargs): + """Module-level receiver so Django's ``Signal.connect`` dedupes it across + repeat ``ready()`` invocations (a closure would have a fresh id each call). """ + from netbox_custom_objects.models import _deferred_co_field_data + _deferred_co_field_data.set(None) + + +def _register_branching_hooks_once(): + """Register netbox-branching integration hooks at most once per process. + + Wraps the branching-resolver, objectchange-field-migrator, deferred-data + reset receivers, and squash-dependency-graph receiver. Connect both pre- + and post- merge/sync/revert so the deferred-data reset runs even when the + operation raises (post-signals only fire on success). ``weak=False`` keeps + the receivers alive past the end of ``ready()``. + """ + global _branching_hooks_registered + if _branching_hooks_registered: + return + try: from netbox_branching.signals import ( pre_merge, post_merge, @@ -64,12 +85,39 @@ def _connect_deferred_data_reset_signals(): except ImportError: return - def _reset(sender, **kwargs): - from netbox_custom_objects.models import _deferred_co_field_data - _deferred_co_field_data.set(None) - for sig in (pre_merge, post_merge, pre_sync, post_sync, pre_revert, post_revert): - sig.connect(_reset, weak=False) + sig.connect(_reset_deferred_co_field_data, weak=False) + + try: + from netbox_branching.utilities import ( + register_branching_resolver, + register_objectchange_field_migrator, + ) + from .branching import ( + objectchange_field_migrator, + supports_branching_resolver, + ) + register_branching_resolver(supports_branching_resolver) + register_objectchange_field_migrator(objectchange_field_migrator) + # Subscribe to the squash dependency-graph signal so CO-specific + # edges (M2M targets, polymorphic-M2M sidecar) get added before + # topological ordering. Skipped silently on older netbox-branching + # that doesn't expose the signal yet. + try: + from netbox_branching.merge_strategies.squash import ( + squash_dependency_graph_built, + ) + from .branching import add_custom_object_dependencies + squash_dependency_graph_built.connect( + add_custom_object_dependencies, + weak=False, + ) + except ImportError: + pass + except ImportError: + pass + + _branching_hooks_registered = True # Module-level flag so the heal runs at most once per process invocation even @@ -279,41 +327,12 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() - # Clear deferred CO data at every merge/sync/revert boundary so - # leftover entries from a failed op don't leak forward. - _connect_deferred_data_reset_signals() - - # Register netbox-branching hooks — branchable resolver for our through - # models and ObjectChange field-name migrator for runtime renames. - # Guarded so the plugin still works without netbox-branching. - try: - from netbox_branching.utilities import ( - register_branching_resolver, - register_objectchange_field_migrator, - ) - from .branching import ( - objectchange_field_migrator, - supports_branching_resolver, - ) - register_branching_resolver(supports_branching_resolver) - register_objectchange_field_migrator(objectchange_field_migrator) - # Subscribe to the squash dependency-graph signal so CO-specific - # edges (M2M targets, polymorphic-M2M sidecar) get added before - # topological ordering. Skipped silently on older - # netbox-branching that doesn't expose the signal yet. - try: - from netbox_branching.merge_strategies.squash import ( - squash_dependency_graph_built, - ) - from .branching import add_custom_object_dependencies - squash_dependency_graph_built.connect( - add_custom_object_dependencies, - weak=False, - ) - except ImportError: - pass - except ImportError: - pass + # Register netbox-branching integration hooks (deferred-data reset + # receivers, branchable resolver, ObjectChange field-name migrator, + # squash dependency-graph receiver). Guarded so the plugin still + # works without netbox-branching, and so repeat ready() invocations + # don't accumulate duplicate handlers. + _register_branching_hooks_once() # Suppress warnings about database calls during app initialization with warnings.catch_warnings(): diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index eb5e9be9..24d67a83 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -259,12 +259,11 @@ def _apply_deferred_co_field(field_instance): f'UPDATE "{table_name}" SET "{col_name}" = %s WHERE id = %s', [value, co_pk], ) - - # Pop consumed keys and prune empty entries so the ContextVar doesn't - # accumulate stale state across retries. - for entry in per_table.values(): - for k in candidate_keys: - entry['data'].pop(k, None) + # Pop consumed keys immediately so a mid-loop failure leaves + # un-applied rows intact for retry but doesn't re-apply + # rows that already succeeded. + for k in candidate_keys: + data.pop(k, None) exhausted = [pk for pk, entry in per_table.items() if not entry['data']] for pk in exhausted: @@ -322,12 +321,18 @@ def _schema_remove_field(fi, model, schema_editor, schema_conn=None, existing_ta 'Meta', (), {'db_table': through_table, 'app_label': APP_LABEL, 'managed': True}, ) + temp_name = f'_TempThrough_{through_table}' through_model = type( - f'_TempThrough_{through_table}', + temp_name, (models.Model,), {'Meta': through_meta, '__module__': 'netbox_custom_objects.models'}, ) - schema_editor.delete_model(through_model) + try: + schema_editor.delete_model(through_model) + finally: + # ModelBase.__new__ registered the temp class in apps.all_models; + # drop it so repeated remove/re-add cycles don't leak entries. + apps.all_models.get(APP_LABEL, {}).pop(temp_name.lower(), None) # M2M has no column on the parent table — nothing further to remove. return From 4265f5c97962c48ac780a0182ff5871d243dca61 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 27 May 2026 15:47:03 -0700 Subject: [PATCH 063/115] fix from branching changes --- netbox_custom_objects/__init__.py | 2 +- netbox_custom_objects/branching.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 205dfc4d..450bc65d 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -104,7 +104,7 @@ def _register_branching_hooks_once(): # topological ordering. Skipped silently on older netbox-branching # that doesn't expose the signal yet. try: - from netbox_branching.merge_strategies.squash import ( + from netbox_branching.signals import ( squash_dependency_graph_built, ) from .branching import add_custom_object_dependencies diff --git a/netbox_custom_objects/branching.py b/netbox_custom_objects/branching.py index d19dced3..667dd1b1 100644 --- a/netbox_custom_objects/branching.py +++ b/netbox_custom_objects/branching.py @@ -88,6 +88,11 @@ def add_custom_object_dependencies(sender, collapsed_changes, **kwargs): Walks every collapsed change for a CO model and mirrors squash's four edge-direction rules (UPDATE→DELETE, UPDATE→CREATE, CREATE→CREATE, DELETE→DELETE) using ``_collect_co_refs`` instead of the FK/GFK walker. + + The signal's ``operation`` kwarg ('merge' or 'revert') is intentionally + ignored: these edges express physical "must exist before" relationships, + and revert reverses the topological order so the same edges produce the + correct undo sequence. """ from .constants import APP_LABEL From 2f46e36ca22070945c67e2b7840b0d954d61025e Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 27 May 2026 16:34:59 -0700 Subject: [PATCH 064/115] merge feature/main --- netbox_custom_objects/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 5a0cc225..6f16d0d1 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -74,6 +74,8 @@ generate_model, ) +logger = logging.getLogger(__name__) + class UniquenessConstraintTestError(Exception): """Custom exception used to signal successful uniqueness constraint test.""" From 5528df53996b05384142f7ea7bc7ec4f2ca2cd84 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 29 May 2026 14:24:00 -0700 Subject: [PATCH 065/115] test fix --- netbox_custom_objects/field_types.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index abd9d826..6428a95e 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -127,11 +127,15 @@ def _resolve_model(self, model): try: from netbox_custom_objects.models import CustomObjectType cot = CustomObjectType.objects.get(pk=int(cot_id_str)) + branch_id = CustomObjectType._active_branch_id() # Only seed the cache if nothing is cached yet; don't overwrite # a full model that was generated by a previous get_model() call. - if not CustomObjectType.is_model_cached(cot.id): + if not CustomObjectType.is_model_cached(cot.id, branch_id): with CustomObjectType._global_lock: - CustomObjectType._model_cache[cot.id] = (actual_model, cot.cache_timestamp) + CustomObjectType._model_cache[(cot.id, branch_id)] = ( + actual_model, + cot.cache_timestamp, + ) except (CustomObjectType.DoesNotExist, OperationalError, ProgrammingError): pass From 40a789f26effda9de2292ed0f8f947fb202947d0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 29 May 2026 15:31:19 -0700 Subject: [PATCH 066/115] merge feature --- netbox_custom_objects/api/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index 961d4a83..97407361 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -5,7 +5,6 @@ from core.models import ObjectType from django.contrib.contenttypes.models import ContentType from django.db import transaction -from django.db.utils import OperationalError, ProgrammingError from django.urls import NoReverseMatch from django.utils.translation import gettext_lazy as _ from extras.choices import CustomFieldTypeChoices From ffcd97285c9e6c78c845fd6982bf621eb0db320f Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 29 May 2026 15:54:52 -0700 Subject: [PATCH 067/115] fix tests --- netbox_custom_objects/tests/test_models.py | 34 +++++++++++++--------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 9cd75d77..9e1df4ed 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -2145,10 +2145,11 @@ class LazySerializerRegistrationTestCase(CustomObjectsTestCase, TestCase): 1. get_serializer_class(model, skip_object_fields=True) used to overwrite the full module-level serializer with a partial one, causing subsequent requests to fail or return incomplete data. - 2. import_string("netbox_custom_objects.api.serializers.Table{N}ModelSerializer") - raised SerializerNotFound in workers whose ready() ran before the COT existed. - The module-level __getattr__ hook (PEP 562) fixes this by generating the - serializer on demand. + 2. get_serializer_for_model() raised SerializerNotFound for a Table{N}Model in + workers whose ready() ran before the COT existed. + The serializer_resolver() hook (registered via PluginConfig.serializer_resolver) + fixes this by generating the serializer on demand, ahead of any import-path + lookup. """ @classmethod @@ -2175,8 +2176,8 @@ def setUpTestData(cls): def _clear_serializer_registrations(self): """Remove any previously registered serializers for parent and child COTs - from the module so that __getattr__ and get_serializer_class() behave as - they would in a fresh worker process.""" + from the module so that serializer_resolver() and get_serializer_class() + behave as they would in a fresh worker process.""" import netbox_custom_objects.api.serializers as ser_module parent_model = self.parent_cot.get_model() child_model = self.child_cot.get_model() @@ -2184,10 +2185,14 @@ def _clear_serializer_registrations(self): attr = f"{model._meta.object_name}Serializer" ser_module.__dict__.pop(attr, None) - def test_module_getattr_generates_serializer_on_demand(self): - """__getattr__ must generate and return a serializer for any Table{N}Model - whose serializer was never pre-registered (simulates a worker that started - before the COT was created — issue #370 Scenario A).""" + def test_resolver_generates_serializer_on_demand(self): + """serializer_resolver() must generate and return a serializer for any + Table{N}Model whose serializer was never pre-registered (simulates a worker + that started before the COT was created — issue #370 Scenario A). + + NetBox calls this resolver from get_serializer_for_model() before falling + back to an import-path lookup, so a missing startup-time registration can + never raise SerializerNotFound.""" import netbox_custom_objects.api.serializers as ser_module self._clear_serializer_registrations() @@ -2198,14 +2203,15 @@ def test_module_getattr_generates_serializer_on_demand(self): self.assertNotIn(serializer_name, ser_module.__dict__, "precondition: serializer must not be pre-registered") - # getattr() must trigger __getattr__ and return a valid class - serializer_cls = getattr(ser_module, serializer_name) + # The resolver must generate and return a valid serializer class on demand + serializer_cls = ser_module.serializer_resolver(parent_model) self.assertIsNotNone(serializer_cls) self.assertIn("title", serializer_cls.Meta.fields) - # After __getattr__ fires, the serializer is cached in __dict__ + # After the resolver fires, the full serializer is cached in __dict__ + # (get_serializer_class registers it via setattr) self.assertIn(serializer_name, ser_module.__dict__, - "serializer must be cached in module __dict__ after first access") + "serializer must be cached in module __dict__ after first resolution") def test_skip_object_fields_does_not_overwrite_full_serializer(self): """get_serializer_class(model, skip_object_fields=True) must not overwrite From 0a6ff7363bad24b84c25745b786e8b7bd718dc84 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 29 May 2026 16:23:55 -0700 Subject: [PATCH 068/115] update query counts --- netbox_custom_objects/tests/query_counts.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/netbox_custom_objects/tests/query_counts.json b/netbox_custom_objects/tests/query_counts.json index 14ca20ec..7ab9d0a3 100644 --- a/netbox_custom_objects/tests/query_counts.json +++ b/netbox_custom_objects/tests/query_counts.json @@ -1,6 +1,6 @@ { - "customobject-complex:list_objects_with_permission": 41, - "customobject-objectfields:list_objects_with_permission": 50, - "customobject-simple:list_objects_with_permission": 33, - "customobjecttype:list_objects_with_permission": 32 -} \ No newline at end of file + "customobject-complex:list_objects_with_permission": 40, + "customobject-objectfields:list_objects_with_permission": 49, + "customobject-simple:list_objects_with_permission": 32, + "customobjecttype:list_objects_with_permission": 30 +} From d581e95d9d0f35dc7ea60d6f39ee60a6bbc6c8d7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 3 Jun 2026 10:44:14 -0700 Subject: [PATCH 069/115] fix connection for poly models --- netbox_custom_objects/field_types.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 6428a95e..52d1d800 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -10,7 +10,7 @@ from django.contrib.postgres.fields import ArrayField from django.core.exceptions import FieldDoesNotExist from django.core.validators import RegexValidator -from django.db import connection, models, router +from django.db import models, router from django.db.utils import OperationalError, ProgrammingError from django.db.models.fields.related import ForeignKey, ManyToManyDescriptor from django.db.models.manager import Manager @@ -1603,9 +1603,14 @@ def create_polymorphic_m2m_table(self, field_instance, model, schema_editor): source_field.remote_field.model = model source_field.related_model = model + # Probe the same schema the DDL will target. schema_editor is branch-aware + # (opened via _get_schema_connection() by the caller), whereas the module-level + # ``connection`` always points at the main schema — using it here would let the + # idempotency guard diverge from where create_model() actually writes. + conn = schema_editor.connection table_name = through._meta.db_table - with connection.cursor() as cursor: - existing_tables = connection.introspection.table_names(cursor) + with conn.cursor() as cursor: + existing_tables = conn.introspection.table_names(cursor) if table_name not in existing_tables: schema_editor.create_model(through) From ec6f9f7de601203d36427b339f3adfc02da3ed54 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 8 Jun 2026 14:26:36 -0700 Subject: [PATCH 070/115] #30 - add graphql support for custom objects --- docs/graphql.md | 115 ++++++++++ mkdocs.yml | 1 + netbox_custom_objects/__init__.py | 6 + netbox_custom_objects/graphql/__init__.py | 0 netbox_custom_objects/graphql/schema.py | 104 +++++++++ netbox_custom_objects/graphql/types.py | 195 +++++++++++++++++ netbox_custom_objects/tests/test_graphql.py | 222 ++++++++++++++++++++ 7 files changed, 643 insertions(+) create mode 100644 docs/graphql.md create mode 100644 netbox_custom_objects/graphql/__init__.py create mode 100644 netbox_custom_objects/graphql/schema.py create mode 100644 netbox_custom_objects/graphql/types.py create mode 100644 netbox_custom_objects/tests/test_graphql.py diff --git a/docs/graphql.md b/docs/graphql.md new file mode 100644 index 00000000..ffd4d234 --- /dev/null +++ b/docs/graphql.md @@ -0,0 +1,115 @@ +# GraphQL API + +The NetBox Custom Objects plugin exposes custom objects through NetBox's GraphQL +API at the standard `/graphql/` endpoint, alongside NetBox's built-in models. For +each Custom Object Type you have defined, two root query fields are generated: + +- `` — fetch a single custom object by `id`. +- `_list` — fetch a list of custom objects (paginated). + +The `` is derived from the Custom Object Type's **slug**, with any +characters that are not valid in a GraphQL name replaced by underscores (for +example a type with slug `dhcp-scope` becomes `dhcp_scope` and `dhcp_scope_list`). + +!!! note "New types require a restart" + Unlike the REST API — which resolves each request dynamically — NetBox builds + its GraphQL schema **once at startup**. A Custom Object Type created (or + deleted) while NetBox is running will not appear in (or disappear from) the + GraphQL schema until NetBox is restarted. This mirrors how adding a new Django + model requires a restart. Restart both the web service and the RQ workers. + +## Authentication + +GraphQL requests use the same authentication as the REST API. Pass a token in the +`Authorization` header: + +``` +Authorization: Token +``` + +Object-level permissions are enforced: queries only return custom objects the +authenticated user has permission to view. + +## Querying a list + +```graphql +query { + dhcp_scope_list { + id + display + name + description + } +} +``` + +## Querying a single object + +```graphql +query { + dhcp_scope(id: 42) { + id + display + name + } +} +``` + +## Available fields + +Each generated type exposes: + +| Field | Description | +|-------|-------------| +| `id` | The object's primary key. | +| `display` | The object's display string (its primary field value). | +| `created` / `last_updated` | Change-logging timestamps. | +| `tags` | The object's tags. | +| One field per custom field | Named exactly as the field's `name`. | + +### Scalar field types + +Scalar custom fields map to their natural GraphQL types: + +| Custom field type | GraphQL type | +|-------------------|--------------| +| text, long text, URL, select | `String` | +| integer | `Int` | +| decimal | `Decimal` | +| boolean | `Boolean` | +| date | `Date` | +| datetime | `DateTime` | +| JSON | `JSON` | +| multi-select | `[String]` | + +### Relationship field types + +Object and multi-object fields (including polymorphic ones) resolve to a uniform +`CustomObjectRelatedObjectType`, because a relationship may point at any NetBox +model or another custom object: + +```graphql +query { + server_list { + name + primary_site { # an object (single) field + id + object_type # e.g. "dcim.site" + display + url + } + interfaces { # a multi-object (list) field + id + object_type + display + } + } +} +``` + +| Field | Description | +|-------|-------------| +| `id` | The referenced object's primary key. | +| `object_type` | The referenced object's type, as `.`. | +| `display` | The referenced object's display string. | +| `url` | The referenced object's absolute URL, if resolvable. | diff --git a/mkdocs.yml b/mkdocs.yml index 7e127c75..c9c1dd2b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,5 +42,6 @@ nav: - Field Attributes: 'field-attributes.md' - Branching Support: 'branching.md' - REST API: 'rest-api.md' + - GraphQL API: 'graphql.md' - Portable Schema: 'portable-schema.md' - Releases: 'releases.md' diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 09c09296..1a858817 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -217,6 +217,12 @@ class CustomObjectsPluginConfig(PluginConfig): # Resolves dynamic CO models (table{n}model) to on-the-fly serializers — # they have no importable path at the conventional location. serializer_resolver = "api.serializers.serializer_resolver" + # Contributes a GraphQL Query (one field pair per custom object type) to + # NetBox's schema. Built at startup from the existing types; see + # graphql/types.py for the restart-on-new-type caveat. The path resolves as + # module ``graphql.schema`` + attribute ``schema`` (NetBox loads resources + # via ``import_string``, which treats the final component as an attribute). + graphql_schema = "graphql.schema.schema" @staticmethod def should_skip_dynamic_model_creation(): diff --git a/netbox_custom_objects/graphql/__init__.py b/netbox_custom_objects/graphql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py new file mode 100644 index 00000000..a91fb72c --- /dev/null +++ b/netbox_custom_objects/graphql/schema.py @@ -0,0 +1,104 @@ +""" +GraphQL schema contribution for the custom-objects plugin. + +NetBox discovers this module via the plugin's ``graphql_schema`` resource path +and extends its global ``Query`` with every class in the exported ``schema`` +list. We build a single ``Query`` class exposing two fields per custom object +type — ```` (single object by ``id``) and ``_list`` (filtered, +paginated list) — mirroring NetBox's own per-model GraphQL fields. + +The schema is assembled once, at plugin startup, from the custom object types +that exist at that moment. See :mod:`netbox_custom_objects.graphql.types` for +why runtime-created types require a restart to appear. +""" + +import logging +import re +from typing import List + +import strawberry +import strawberry_django + +from .types import build_object_type + +logger = logging.getLogger("netbox_custom_objects.graphql") + + +def _query_field_name(custom_object_type, used_names): + """ + Derive a GraphQL-safe, unique field name from a custom object type's slug. + + GraphQL names must match ``[_A-Za-z][_0-9A-Za-z]*``; slugs may contain + hyphens. Collisions (after sanitisation) are disambiguated with the type id. + """ + base = re.sub(r"[^0-9a-zA-Z_]", "_", (custom_object_type.slug or "").lower()) + if not base or base[0].isdigit(): + base = f"_{base}" + name = base + if name in used_names: + name = f"{base}_{custom_object_type.id}" + used_names.add(name) + return name + + +def _build_query_classes(): + """ + Build the list of Strawberry query classes contributed to NetBox's schema. + + Returns an empty list when dynamic models are unavailable (during + migrations, tests, or before migrations have been applied) or when no custom + object types are defined — extending NetBox's Query with an empty/invalid + class is avoided. + """ + # Import lazily to avoid import-time side effects and circular imports. + # The skip-check must run *before* importing models: during migrations and + # the test run the app registry may not be ready, and importing models then + # raises "model isn't in an application in INSTALLED_APPS". + from netbox_custom_objects import CustomObjectsPluginConfig + + if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): + return [] + + from netbox_custom_objects.models import CustomObjectType + + try: + custom_object_types = list(CustomObjectType.objects.all()) + except Exception: # noqa: BLE001 - DB may be unavailable at import time + logger.debug("Could not load custom object types for GraphQL schema", exc_info=True) + return [] + + annotations = {} + attrs = {} + used_names = set() + + for cot in custom_object_types: + try: + gql_type = build_object_type(cot) + except Exception: # noqa: BLE001 - never break the whole schema for one type + logger.warning( + "Failed to build GraphQL type for custom object type %r (id=%s); skipping", + cot.name, + cot.id, + exc_info=True, + ) + continue + if gql_type is None: + continue + + field_name = _query_field_name(cot, used_names) + list_name = f"{field_name}_list" + + annotations[field_name] = gql_type + attrs[field_name] = strawberry_django.field() + annotations[list_name] = List[gql_type] + attrs[list_name] = strawberry_django.field() + + if not attrs: + return [] + + attrs["__annotations__"] = annotations + query_cls = strawberry.type(type("CustomObjectsQuery", (), attrs)) + return [query_cls] + + +schema = _build_query_classes() diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py new file mode 100644 index 00000000..7916d0bb --- /dev/null +++ b/netbox_custom_objects/graphql/types.py @@ -0,0 +1,195 @@ +""" +Dynamic GraphQL type generation for custom object models. + +Each ``CustomObjectType`` is backed by a real, runtime-generated Django model +(``TableModel``). This module builds a Strawberry GraphQL type for each of +those models so that custom objects can be queried through NetBox's GraphQL API +exactly like first-class NetBox models. + +Because the underlying models are generated at runtime, the GraphQL types are +also generated at runtime — at plugin startup (``ready()``), when NetBox +assembles its GraphQL schema. A custom object type created *after* startup will +not appear in the GraphQL schema until NetBox is restarted (the GraphQL schema, +like the Django model registry it mirrors, is built once at boot). This mirrors +how adding a new Django model requires a restart, and differs from the REST API, +which resolves serializers per-request. + +Scalar fields are mapped to their natural GraphQL scalar types. Object and +multi-object (relationship) fields — including polymorphic ones — are exposed +through a single shared ``CustomObjectRelatedObjectType`` so that references to +*any* NetBox model or other custom object resolve uniformly. +""" + +import datetime +import decimal +import logging +from typing import List, Optional + +import strawberry +import strawberry_django +from core.graphql.mixins import ChangelogMixin +from extras.choices import CustomFieldTypeChoices +from extras.graphql.mixins import TagsMixin +from netbox.graphql.types import BaseObjectType +from strawberry.scalars import JSON + +logger = logging.getLogger("netbox_custom_objects.graphql") + +__all__ = ( + "CustomObjectObjectType", + "CustomObjectRelatedObjectType", + "build_object_type", +) + + +# Mapping of custom-field scalar types to the Python annotation Strawberry should +# use. Relationship types (OBJECT / MULTIOBJECT) are handled separately via +# resolvers, so they are intentionally absent here. +SCALAR_TYPE_MAP = { + CustomFieldTypeChoices.TYPE_TEXT: str, + CustomFieldTypeChoices.TYPE_LONGTEXT: str, + CustomFieldTypeChoices.TYPE_URL: str, + CustomFieldTypeChoices.TYPE_SELECT: str, + CustomFieldTypeChoices.TYPE_INTEGER: int, + CustomFieldTypeChoices.TYPE_DECIMAL: decimal.Decimal, + CustomFieldTypeChoices.TYPE_BOOLEAN: bool, + CustomFieldTypeChoices.TYPE_DATE: datetime.date, + CustomFieldTypeChoices.TYPE_DATETIME: datetime.datetime, + CustomFieldTypeChoices.TYPE_JSON: JSON, + CustomFieldTypeChoices.TYPE_MULTISELECT: List[str], +} + +RELATIONSHIP_TYPES = ( + CustomFieldTypeChoices.TYPE_OBJECT, + CustomFieldTypeChoices.TYPE_MULTIOBJECT, +) + + +@strawberry.type +class CustomObjectRelatedObjectType: + """ + A lightweight, uniform representation of an object referenced by a custom + object's relationship field. + + Relationship fields can point at any NetBox model, another custom object, or + (for polymorphic fields) a mix of types, so a single concrete Strawberry type + per target is not feasible. This shared type exposes the information common + to every referenced object. + """ + + id: int + object_type: str + display: str + url: Optional[str] + + +@strawberry.type +class CustomObjectObjectType(ChangelogMixin, TagsMixin, BaseObjectType): + """ + Base GraphQL type for all custom object models. + + ``BaseObjectType`` provides ``display``/``class_type`` and, crucially, + ``get_queryset()`` which enforces NetBox object-level view permissions. + ``ChangelogMixin`` and ``TagsMixin`` add change-log and tag access — both are + supported by the ``CustomObject`` base model. Custom fields are added per + type by :func:`build_object_type`. + """ + + pass + + +def _related_repr(obj): + """Convert a referenced model instance into a ``CustomObjectRelatedObjectType``.""" + if obj is None: + return None + url = None + try: + url = obj.get_absolute_url() + except Exception: # noqa: BLE001 - URL resolution is best-effort + url = None + return CustomObjectRelatedObjectType( + id=obj.pk, + object_type=f"{obj._meta.app_label}.{obj._meta.model_name}", + display=str(obj), + url=url, + ) + + +def _make_object_resolver(field_name): + """Build a resolver returning the single related object for an OBJECT field.""" + + @strawberry_django.field(description=f"Related object referenced by '{field_name}'") + def resolver(self) -> Optional[CustomObjectRelatedObjectType]: + return _related_repr(getattr(self, field_name, None)) + + return resolver + + +def _make_multiobject_resolver(field_name): + """Build a resolver returning related objects for a MULTIOBJECT field.""" + + @strawberry_django.field(description=f"Related objects referenced by '{field_name}'") + def resolver(self) -> List[CustomObjectRelatedObjectType]: + manager = getattr(self, field_name, None) + if manager is None: + return [] + try: + related = list(manager.all()) + except Exception: # noqa: BLE001 - never let one field break the query + logger.warning( + "Failed to resolve multi-object GraphQL field %r", field_name, exc_info=True + ) + return [] + return [_related_repr(obj) for obj in related if obj is not None] + + return resolver + + +def build_object_type(custom_object_type): + """ + Build (or return ``None`` on failure) a Strawberry type for a single + ``CustomObjectType``. + + The returned class is a ``strawberry_django.type`` bound to the runtime + model, with one GraphQL field per custom field plus the inherited base + fields (id, display, tags, changelog, created, last_updated). + """ + model = custom_object_type.get_model() + if model is None: + return None + + type_name = f"{model.__name__}Type" + + namespace = { + "__doc__": f"Custom object type '{custom_object_type.name}'.", + "__annotations__": {}, + } + + for field in custom_object_type.fields.all(): + field_name = field.name + if field.type in RELATIONSHIP_TYPES: + if field.type == CustomFieldTypeChoices.TYPE_OBJECT: + namespace[field_name] = _make_object_resolver(field_name) + else: + namespace[field_name] = _make_multiobject_resolver(field_name) + continue + + annotation = SCALAR_TYPE_MAP.get(field.type) + if annotation is None: + logger.debug( + "Skipping custom field %r of unsupported GraphQL type %r", + field_name, + field.type, + ) + continue + # Every custom field is nullable at the database level. + namespace["__annotations__"][field_name] = Optional[annotation] + + cls = type(type_name, (CustomObjectObjectType,), namespace) + + return strawberry_django.type( + model, + name=type_name, + fields=["id", "created", "last_updated"], + pagination=True, + )(cls) diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py new file mode 100644 index 00000000..9d6fdf7b --- /dev/null +++ b/netbox_custom_objects/tests/test_graphql.py @@ -0,0 +1,222 @@ +""" +Tests for GraphQL support for custom objects. + +The plugin contributes its GraphQL schema at startup, which is intentionally +skipped during the test run (``should_skip_dynamic_model_creation()`` returns +True when ``test`` is on the command line — see ``__init__.py``). We therefore +exercise the schema-generation functions directly: build a Strawberry schema +from custom object types created in each test and execute queries against it +in-process, mirroring what NetBox does at boot. +""" + +from typing import List + +import strawberry +import strawberry_django +from django.test import TestCase +from strawberry.schema.config import StrawberryConfig + +from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Site + +from netbox_custom_objects.graphql import schema as schema_module +from netbox_custom_objects.graphql.schema import _build_query_classes, _query_field_name +from netbox_custom_objects.graphql.types import build_object_type + +from .base import CustomObjectsTestCase + + +class _Context: + """Minimal stand-in for Strawberry-Django's request context.""" + + def __init__(self, request): + self.request = request + + +def build_test_schema(custom_object_types): + """Assemble a Strawberry schema from the given COTs, bypassing the startup guard.""" + annotations = {} + attrs = {} + used_names = set() + for cot in custom_object_types: + gql_type = build_object_type(cot) + field_name = _query_field_name(cot, used_names) + list_name = f"{field_name}_list" + annotations[field_name] = gql_type + attrs[field_name] = strawberry_django.field() + annotations[list_name] = List[gql_type] + attrs[list_name] = strawberry_django.field() + attrs["__annotations__"] = annotations + # The GraphQL type name must be "Query" for strawberry-django to attach the + # single-object `id` lookup argument (it only does so on the root Query + # type). In production our class is mixed into NetBox's real Query, which is + # named "Query"; here we name it directly. + query_cls = strawberry.type(type("Query", (), attrs)) + return strawberry.Schema(query=query_cls, config=StrawberryConfig(auto_camel_case=False)) + + +class GraphQLSchemaGenerationTestCase(CustomObjectsTestCase, TestCase): + """Tests for the schema/query-class generation helpers.""" + + def test_query_field_name_sanitizes_slug(self): + used = set() + cot = self.create_custom_object_type(name="Widget", slug="my-widget") + name = _query_field_name(cot, used) + self.assertEqual(name, "my_widget") + + def test_query_field_name_dedupes_collisions(self): + used = set() + a = self.create_custom_object_type(name="A", slug="a-b") + b = self.create_custom_object_type(name="B", slug="a_b", verbose_name_plural="Bs") + first = _query_field_name(a, used) + second = _query_field_name(b, used) + self.assertEqual(first, "a_b") + self.assertEqual(second, f"a_b_{b.id}") + + def test_build_query_classes_skipped_during_tests(self): + # ``test`` is on sys.argv during the suite, so the startup builder + # short-circuits to an empty list rather than touching the DB. + self.create_simple_custom_object_type() + self.assertEqual(_build_query_classes(), []) + + def test_module_exposes_schema_list(self): + # The module-level ``schema`` attribute must always be a list so that + # NetBox's ``register_graphql_schema`` (which calls ``.extend``) works. + self.assertIsInstance(schema_module.schema, list) + + def test_real_builder_produces_assemblable_query(self): + # Exercise the actual production builder (normally skipped during tests) + # to confirm it yields a Query class that assembles into a valid schema + # exposing the expected per-type fields. + from unittest import mock + + self.create_simple_custom_object_type(name="Gadget", slug="gadget") + with mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ): + classes = _build_query_classes() + + self.assertEqual(len(classes), 1) + # Assemble exactly as NetBox does: the contributed class as a base of the + # root Query type. + query_cls = strawberry.type(type("Query", (classes[0],), {})) + built = strawberry.Schema( + query=query_cls, config=StrawberryConfig(auto_camel_case=False) + ) + sdl = str(built) + self.assertIn("gadget", sdl) + self.assertIn("gadget_list", sdl) + + +class GraphQLQueryTestCase(CustomObjectsTestCase, TestCase): + """End-to-end query execution against a generated schema.""" + + def setUp(self): + super().setUp() + # restrict() returns everything for a superuser, keeping these tests + # focused on schema generation rather than permission wiring. + self.user.is_superuser = True + self.user.save() + self.request = self._make_request() + + def _make_request(self): + from django.test import RequestFactory + + request = RequestFactory().post("/graphql/") + request.user = self.user + return request + + def _execute(self, schema, query): + result = schema.execute_sync(query, context_value=_Context(self.request)) + self.assertIsNone(result.errors, msg=str(result.errors)) + return result.data + + def _make_device(self): + manufacturer = Manufacturer.objects.create(name="Mfr", slug="mfr") + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model="Model", slug="model" + ) + role = DeviceRole.objects.create(name="Role", slug="role") + site = Site.objects.create(name="Site", slug="site") + return Device.objects.create( + name="dev1", device_type=device_type, role=role, site=site + ) + + def test_scalar_fields_query(self): + cot = self.create_complex_custom_object_type(name="Asset", slug="asset") + model = cot.get_model() + model.objects.create(name="First", count=7, active=True, status="choice1") + + schema = build_test_schema([cot]) + data = self._execute( + schema, + "{ asset_list { id name count active status } }", + ) + rows = data["asset_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "First") + self.assertEqual(rows[0]["count"], 7) + self.assertTrue(rows[0]["active"]) + self.assertEqual(rows[0]["status"], "choice1") + + def test_single_object_query_by_id(self): + cot = self.create_simple_custom_object_type(name="Note", slug="note") + model = cot.get_model() + instance = model.objects.create(name="Hello", description="world") + + schema = build_test_schema([cot]) + data = self._execute( + schema, + f'{{ note(id: {instance.pk}) {{ id name description }} }}', + ) + self.assertEqual(data["note"]["name"], "Hello") + self.assertEqual(data["note"]["description"], "world") + + def test_object_relationship_field(self): + device = self._make_device() + cot = self.create_complex_custom_object_type(name="Link", slug="link") + model = cot.get_model() + model.objects.create(name="L1", device=device) + + schema = build_test_schema([cot]) + data = self._execute( + schema, + "{ link_list { name device { id display object_type } } }", + ) + related = data["link_list"][0]["device"] + self.assertEqual(related["id"], device.pk) + self.assertEqual(related["object_type"], "dcim.device") + self.assertEqual(related["display"], str(device)) + + def test_multiobject_relationship_field(self): + device = self._make_device() + cot = self.create_multi_object_custom_object_type(name="Group", slug="group") + model = cot.get_model() + instance = model.objects.create(name="G1") + instance.devices.add(device) + + schema = build_test_schema([cot]) + data = self._execute( + schema, + "{ group_list { name devices { id object_type } } }", + ) + devices = data["group_list"][0]["devices"] + self.assertEqual(len(devices), 1) + self.assertEqual(devices[0]["id"], device.pk) + self.assertEqual(devices[0]["object_type"], "dcim.device") + + def test_tags_and_base_fields(self): + cot = self.create_simple_custom_object_type(name="Doc", slug="doc") + model = cot.get_model() + model.objects.create(name="D1") + + schema = build_test_schema([cot]) + data = self._execute( + schema, + "{ doc_list { id display created tags { name } } }", + ) + row = data["doc_list"][0] + self.assertEqual(row["display"], "D1") + self.assertIsNotNone(row["id"]) + self.assertEqual(row["tags"], []) From bcb6d90af7c0bdbf2200a83d710a323c9b0b8b72 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 8 Jun 2026 16:34:22 -0700 Subject: [PATCH 071/115] allow live reloading graphql --- docs/graphql.md | 7 -- netbox_custom_objects/__init__.py | 49 ++++++++ netbox_custom_objects/graphql/live.py | 124 ++++++++++++++++++++ netbox_custom_objects/graphql/schema.py | 12 +- netbox_custom_objects/tests/test_graphql.py | 79 ++++++++++++- 5 files changed, 257 insertions(+), 14 deletions(-) create mode 100644 netbox_custom_objects/graphql/live.py diff --git a/docs/graphql.md b/docs/graphql.md index ffd4d234..13c6bced 100644 --- a/docs/graphql.md +++ b/docs/graphql.md @@ -11,13 +11,6 @@ The `` is derived from the Custom Object Type's **slug**, with any characters that are not valid in a GraphQL name replaced by underscores (for example a type with slug `dhcp-scope` becomes `dhcp_scope` and `dhcp_scope_list`). -!!! note "New types require a restart" - Unlike the REST API — which resolves each request dynamically — NetBox builds - its GraphQL schema **once at startup**. A Custom Object Type created (or - deleted) while NetBox is running will not appear in (or disappear from) the - GraphQL schema until NetBox is restarted. This mirrors how adding a new Django - model requires a restart. Restart both the web service and the RQ workers. - ## Authentication GraphQL requests use the same authentication as the REST API. Pass a token in the diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 1a858817..28f0fb33 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -196,6 +196,51 @@ def _patched_get_filterset_class(self, model): ObjectSelectorView._get_filterset_class = _patched_get_filterset_class +_graphql_view_patched = False + + +def _patch_graphql_view(): + """ + Patch NetBox's GraphQL view so custom object types appear without a restart. + + NetBox builds its GraphQL schema once at startup and binds it to the view via + ``as_view(schema=...)``. Custom object types are created/deleted at runtime + and the app runs across multiple worker processes, so that single bound + schema goes stale. This patch swaps in a per-request, per-process schema + that is rebuilt only when the database changes (see + :mod:`netbox_custom_objects.graphql.live`). + + Like the ObjectSelectorView patch above, this monkey-patches NetBox because + the schema binding leaves no other extension point. The wrapper is + re-decorated with ``csrf_exempt`` because Django copies that marker from + ``dispatch.__dict__`` onto the view at ``as_view()`` time (which runs after + this patch), and the GraphQL endpoint must accept token-authenticated POSTs. + """ + global _graphql_view_patched + if _graphql_view_patched: + return + + from django.views.decorators.csrf import csrf_exempt + from netbox.graphql.views import NetBoxGraphQLView + + _original_dispatch = NetBoxGraphQLView.dispatch + + @csrf_exempt + def _patched_dispatch(self, request, *args, **kwargs): + from netbox_custom_objects.graphql.live import get_live_schema + + try: + live_schema = get_live_schema() + if live_schema is not None: + self.schema = live_schema + except Exception: # noqa: BLE001 - fall back to the static schema + logger.warning("Failed to load live GraphQL schema; using static schema", exc_info=True) + return _original_dispatch(self, request, *args, **kwargs) + + NetBoxGraphQLView.dispatch = _patched_dispatch + _graphql_view_patched = True + + # Plugin Configuration class CustomObjectsPluginConfig(PluginConfig): name = "netbox_custom_objects" @@ -333,6 +378,10 @@ def ready(self): # Patch ObjectSelectorView to support dynamically-generated custom object models _patch_object_selector_view() + # Patch the GraphQL view so custom object types added/removed at runtime + # are reflected in the schema without a NetBox restart. + _patch_graphql_view() + # Register netbox-branching integration hooks (deferred-data reset # receivers, branchable resolver, ObjectChange field-name migrator, # squash dependency-graph receiver). Guarded so the plugin still diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py new file mode 100644 index 00000000..b2760fbf --- /dev/null +++ b/netbox_custom_objects/graphql/live.py @@ -0,0 +1,124 @@ +""" +Live (restart-free) GraphQL schema for custom objects. + +NetBox builds its GraphQL schema once at startup and binds it to the GraphQL +view. Custom object types, however, are created and deleted at runtime, and the +plugin runs across multiple worker processes — so a schema built once at boot +goes stale and a signal-based rebuild in one process can't reach the others. + +This module makes the schema reflect the current database on every request, in +every process, without a restart: + +- :func:`schema_signature` computes a cheap fingerprint of the custom object + types and their fields (counts + most-recent change timestamps). One small + aggregate query pair per request. +- :func:`get_live_schema` caches the assembled schema per process and rebuilds + it only when the signature changes. Because each process checks the signature + independently, a type created by a request handled in one process becomes + visible to every process on its next request — no restart, no cross-process + messaging. + +The view patch in ``__init__.py`` calls :func:`get_live_schema` per request and +assigns the result to the view before it executes the operation. +""" + +import logging +import threading + +logger = logging.getLogger("netbox_custom_objects.graphql") + +_lock = threading.RLock() +# Per-process cache of the assembled schema and the signature it was built from. +_state = {"signature": None, "schema": None} + + +def schema_signature(): + """ + Return a cheap, comparable fingerprint of the custom object type schema. + + Captures both custom object types and their fields so that creating, + deleting, or editing either invalidates the cached GraphQL schema: + + - count detects additions and removals, + - max timestamp detects edits (``cache_timestamp`` / ``last_updated`` are + ``auto_now`` fields). + """ + from django.db.models import Count, Max + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + cot = CustomObjectType.objects.aggregate(n=Count("id"), t=Max("cache_timestamp")) + fields = CustomObjectTypeField.objects.aggregate(n=Count("id"), t=Max("last_updated")) + return (cot["n"], cot["t"], fields["n"], fields["t"]) + + +def build_full_schema(): + """ + Assemble a complete NetBox GraphQL schema with the current custom object types. + + Reuses NetBox's own ``Query`` base classes (all core apps plus every other + plugin) so the result is identical to NetBox's startup schema except that our + custom object query is rebuilt from the live database. Our previously + contributed query class is identified by the ``_nco_query`` marker and + replaced. + """ + import strawberry + from strawberry.schema.config import StrawberryConfig + + import netbox.graphql.schema as ngs + from netbox.graphql.scalars import BigInt, BigIntScalar + + from .schema import build_query_classes + + # Start from NetBox's canonical Query bases, dropping any stale custom-object + # query class we contributed previously, then graft a freshly built one on. + bases = tuple(b for b in ngs.Query.__bases__ if not getattr(b, "_nco_query", False)) + bases += tuple(build_query_classes()) + + query_cls = strawberry.type(type("Query", bases, {})) + return strawberry.Schema( + query=query_cls, + config=StrawberryConfig( + auto_camel_case=False, + scalar_map={BigInt: BigIntScalar}, + ), + extensions=ngs.get_schema_extensions(), + ) + + +def get_live_schema(): + """ + Return the schema for the current request, rebuilding it if the database has + changed since this process last built it. + + Returns ``None`` when dynamic models are unavailable (migrations/tests) or if + the very first build fails — the caller then falls back to NetBox's static + schema. + """ + from netbox_custom_objects import CustomObjectsPluginConfig + + if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): + return None + + try: + signature = schema_signature() + except Exception: # noqa: BLE001 - DB hiccup: serve whatever we already have + logger.debug("Could not compute GraphQL schema signature", exc_info=True) + return _state["schema"] + + with _lock: + if _state["schema"] is None or _state["signature"] != signature: + try: + _state["schema"] = build_full_schema() + _state["signature"] = signature + except Exception: # noqa: BLE001 - never break the endpoint + logger.exception("Failed to rebuild live GraphQL schema") + return _state["schema"] + return _state["schema"] + + +def reset_cache(): + """Clear the cached schema (used by tests).""" + with _lock: + _state["signature"] = None + _state["schema"] = None diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index a91fb72c..49573798 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -41,7 +41,7 @@ def _query_field_name(custom_object_type, used_names): return name -def _build_query_classes(): +def build_query_classes(): """ Build the list of Strawberry query classes contributed to NetBox's schema. @@ -49,6 +49,11 @@ def _build_query_classes(): migrations, tests, or before migrations have been applied) or when no custom object types are defined — extending NetBox's Query with an empty/invalid class is avoided. + + Used both at startup (the module-level ``schema`` below) and on every live + rebuild (:mod:`netbox_custom_objects.graphql.live`). The returned class + carries a ``_nco_query`` marker so live rebuilds can identify and replace a + previously contributed instance. """ # Import lazily to avoid import-time side effects and circular imports. # The skip-check must run *before* importing models: during migrations and @@ -98,7 +103,10 @@ class is avoided. attrs["__annotations__"] = annotations query_cls = strawberry.type(type("CustomObjectsQuery", (), attrs)) + # Marker so live schema rebuilds can find and replace a stale instance of + # this class among NetBox's Query bases. + query_cls._nco_query = True return [query_cls] -schema = _build_query_classes() +schema = build_query_classes() diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index 9d6fdf7b..f403ca22 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -10,6 +10,7 @@ """ from typing import List +from unittest import mock import strawberry import strawberry_django @@ -18,8 +19,9 @@ from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Site +from netbox_custom_objects.graphql import live as live_module from netbox_custom_objects.graphql import schema as schema_module -from netbox_custom_objects.graphql.schema import _build_query_classes, _query_field_name +from netbox_custom_objects.graphql.schema import build_query_classes, _query_field_name from netbox_custom_objects.graphql.types import build_object_type from .base import CustomObjectsTestCase @@ -76,7 +78,7 @@ def test_build_query_classes_skipped_during_tests(self): # ``test`` is on sys.argv during the suite, so the startup builder # short-circuits to an empty list rather than touching the DB. self.create_simple_custom_object_type() - self.assertEqual(_build_query_classes(), []) + self.assertEqual(build_query_classes(), []) def test_module_exposes_schema_list(self): # The module-level ``schema`` attribute must always be a list so that @@ -87,15 +89,13 @@ def test_real_builder_produces_assemblable_query(self): # Exercise the actual production builder (normally skipped during tests) # to confirm it yields a Query class that assembles into a valid schema # exposing the expected per-type fields. - from unittest import mock - self.create_simple_custom_object_type(name="Gadget", slug="gadget") with mock.patch( "netbox_custom_objects.CustomObjectsPluginConfig." "should_skip_dynamic_model_creation", return_value=False, ): - classes = _build_query_classes() + classes = build_query_classes() self.assertEqual(len(classes), 1) # Assemble exactly as NetBox does: the contributed class as a base of the @@ -109,6 +109,75 @@ def test_real_builder_produces_assemblable_query(self): self.assertIn("gadget_list", sdl) +class GraphQLLiveSchemaTestCase(CustomObjectsTestCase, TestCase): + """ + The schema must reflect custom object types created/deleted at runtime + without a NetBox restart (issue #30 follow-up). + + ``should_skip_dynamic_model_creation`` returns True during the test run, so + we patch it off for these tests and reset the per-process cache around each. + """ + + def setUp(self): + super().setUp() + live_module.reset_cache() + patcher = mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(live_module.reset_cache) + + def test_signature_changes_when_type_added(self): + sig_before = live_module.schema_signature() + self.create_simple_custom_object_type(name="Sig", slug="sig") + sig_after = live_module.schema_signature() + self.assertNotEqual(sig_before, sig_after) + + def test_signature_changes_when_field_added(self): + cot = self.create_simple_custom_object_type(name="Sig2", slug="sig2") + sig_before = live_module.schema_signature() + self.create_custom_object_type_field(cot, name="extra", label="Extra", type="text") + sig_after = live_module.schema_signature() + self.assertNotEqual(sig_before, sig_after) + + def test_get_live_schema_rebuilds_on_new_type(self): + # A type that does not exist yet must not be in the schema... + first = live_module.get_live_schema() + self.assertIsNotNone(first) + self.assertNotIn("runtime_thing", str(first)) + + # ...and must appear after creation, with no restart and without manually + # clearing the cache (the signature change drives the rebuild). + self.create_simple_custom_object_type(name="Runtime Thing", slug="runtime_thing") + second = live_module.get_live_schema() + self.assertIsNot(first, second) + sdl = str(second) + self.assertIn("runtime_thing", sdl) + self.assertIn("runtime_thing_list", sdl) + + def test_get_live_schema_cached_when_unchanged(self): + self.create_simple_custom_object_type(name="Stable", slug="stable") + a = live_module.get_live_schema() + b = live_module.get_live_schema() + # No DB change between calls → same cached object, no rebuild. + self.assertIs(a, b) + + def test_live_schema_drops_deleted_type(self): + from netbox_custom_objects.models import CustomObjectType + + cot = self.create_simple_custom_object_type(name="Temp", slug="temp_type") + self.assertIn("temp_type", str(live_module.get_live_schema())) + # Delete via the queryset rather than cot.delete(): the schema-drop + # behaviour only depends on the row being gone (which changes the + # signature and triggers a rebuild), and this avoids the unrelated COT + # teardown machinery. + CustomObjectType.objects.filter(pk=cot.pk).delete() + self.assertNotIn("temp_type", str(live_module.get_live_schema())) + + class GraphQLQueryTestCase(CustomObjectsTestCase, TestCase): """End-to-end query execution against a generated schema.""" From 83b247b474f679d85ac9eaa4007d4d2bd321bd16 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 8 Jun 2026 17:26:15 -0700 Subject: [PATCH 072/115] cache graphql framework --- netbox_custom_objects/graphql/live.py | 63 ++++++++--- netbox_custom_objects/graphql/schema.py | 39 ++++--- netbox_custom_objects/graphql/types.py | 110 +++++++++++++++++--- netbox_custom_objects/tests/test_graphql.py | 33 ++++++ 4 files changed, 201 insertions(+), 44 deletions(-) diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index b2760fbf..3ce34cf1 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -27,9 +27,14 @@ logger = logging.getLogger("netbox_custom_objects.graphql") -_lock = threading.RLock() -# Per-process cache of the assembled schema and the signature it was built from. -_state = {"signature": None, "schema": None} +# Single-flight rebuild lock: only one thread rebuilds at a time; concurrent +# requests serve the current (possibly slightly stale) schema instead of blocking. +_rebuild_lock = threading.Lock() +# Atomically-swapped (signature, schema) pair. Read without a lock on the hot +# path — a single reference read/assignment is atomic in CPython, and pairing the +# signature with the schema in one tuple means a reader can never see a schema +# that doesn't match its signature. +_current = (None, None) def schema_signature(): @@ -95,6 +100,8 @@ def get_live_schema(): the very first build fails — the caller then falls back to NetBox's static schema. """ + global _current + from netbox_custom_objects import CustomObjectsPluginConfig if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): @@ -104,21 +111,43 @@ def get_live_schema(): signature = schema_signature() except Exception: # noqa: BLE001 - DB hiccup: serve whatever we already have logger.debug("Could not compute GraphQL schema signature", exc_info=True) - return _state["schema"] + return _current[1] + + # Hot path: structure unchanged since this process last built — no lock, no + # rebuild, just return the cached schema. + sig, schema = _current + if schema is not None and sig == signature: + return schema + + # Structure changed (or first build). Single-flight: one thread rebuilds + # while concurrent requests keep serving the existing schema rather than + # blocking on the (potentially expensive) rebuild. + if not _rebuild_lock.acquire(blocking=False): + # Another thread is already rebuilding; serve the current schema — valid, + # just one signature behind — or None (→ static fallback) on first build. + return schema - with _lock: - if _state["schema"] is None or _state["signature"] != signature: - try: - _state["schema"] = build_full_schema() - _state["signature"] = signature - except Exception: # noqa: BLE001 - never break the endpoint - logger.exception("Failed to rebuild live GraphQL schema") - return _state["schema"] - return _state["schema"] + try: + # Re-check: a prior holder may have just published a matching schema. + sig, schema = _current + if schema is not None and sig == signature: + return schema + try: + new_schema = build_full_schema() + except Exception: # noqa: BLE001 - never break the endpoint + logger.exception("Failed to rebuild live GraphQL schema") + return schema + _current = (signature, new_schema) + return new_schema + finally: + _rebuild_lock.release() def reset_cache(): - """Clear the cached schema (used by tests).""" - with _lock: - _state["signature"] = None - _state["schema"] = None + """Clear the cached schema and per-type cache (used by tests).""" + global _current + _current = (None, None) + + from .types import clear_type_cache + + clear_type_cache() diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 49573798..5c6f1fb8 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -3,13 +3,17 @@ NetBox discovers this module via the plugin's ``graphql_schema`` resource path and extends its global ``Query`` with every class in the exported ``schema`` -list. We build a single ``Query`` class exposing two fields per custom object -type — ```` (single object by ``id``) and ``_list`` (filtered, -paginated list) — mirroring NetBox's own per-model GraphQL fields. - -The schema is assembled once, at plugin startup, from the custom object types -that exist at that moment. See :mod:`netbox_custom_objects.graphql.types` for -why runtime-created types require a restart to appear. +list. :func:`build_query_classes` builds a single ``Query`` class exposing two +fields per custom object type — ```` (single object by ``id``) and +``_list`` (filtered, paginated list) — mirroring NetBox's own per-model +GraphQL fields. + +The module-level ``schema`` export is intentionally empty: the live schema in +:mod:`netbox_custom_objects.graphql.live` (installed via the view patch in +``__init__.py``) rebuilds the custom-object query per request from the current +database, so runtime-created types appear without a restart. A statically built +query here would be immediately shadowed by that live rebuild. ``live`` reuses +:func:`build_query_classes`, so both paths share one implementation. """ import logging @@ -35,9 +39,15 @@ def _query_field_name(custom_object_type, used_names): if not base or base[0].isdigit(): base = f"_{base}" name = base - if name in used_names: + # Reserve the singular field name *and* its ``_list`` companion together. + # Checking/recording both prevents one type's list field from silently + # colliding with another type's singular field — e.g. slug 'foo' yields + # foo/foo_list while slug 'foo-list' sanitises to foo_list/foo_list_list, and + # the bare 'foo_list' would otherwise clobber the first type's list field. + if name in used_names or f"{name}_list" in used_names: name = f"{base}_{custom_object_type.id}" used_names.add(name) + used_names.add(f"{name}_list") return name @@ -50,10 +60,10 @@ def build_query_classes(): object types are defined — extending NetBox's Query with an empty/invalid class is avoided. - Used both at startup (the module-level ``schema`` below) and on every live - rebuild (:mod:`netbox_custom_objects.graphql.live`). The returned class - carries a ``_nco_query`` marker so live rebuilds can identify and replace a - previously contributed instance. + Called on every live rebuild (:mod:`netbox_custom_objects.graphql.live`); the + module-level ``schema`` export below deliberately does not call it (see the + module docstring). The returned class carries a ``_nco_query`` marker so live + rebuilds can identify and replace a previously contributed instance. """ # Import lazily to avoid import-time side effects and circular imports. # The skip-check must run *before* importing models: during migrations and @@ -109,4 +119,7 @@ class is avoided. return [query_cls] -schema = build_query_classes() +# Empty by design — the live schema (graphql/live.py) owns per-request assembly so +# runtime-created types appear without a restart. Must remain a list: NetBox calls +# ``.extend`` on it when registering plugin GraphQL schemas. +schema = [] diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index 7916d0bb..84bff0b8 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -7,12 +7,10 @@ exactly like first-class NetBox models. Because the underlying models are generated at runtime, the GraphQL types are -also generated at runtime — at plugin startup (``ready()``), when NetBox -assembles its GraphQL schema. A custom object type created *after* startup will -not appear in the GraphQL schema until NetBox is restarted (the GraphQL schema, -like the Django model registry it mirrors, is built once at boot). This mirrors -how adding a new Django model requires a restart, and differs from the REST API, -which resolves serializers per-request. +also generated at runtime. They are rebuilt per-request by +:mod:`netbox_custom_objects.graphql.live` (installed via the view patch in +``__init__.py``) whenever the set of custom object types or their fields changes, +so a type created *after* startup appears without a NetBox restart. Scalar fields are mapped to their natural GraphQL scalar types. Object and multi-object (relationship) fields — including polymorphic ones — are exposed @@ -23,6 +21,7 @@ import datetime import decimal import logging +import threading from typing import List, Optional import strawberry @@ -39,8 +38,25 @@ "CustomObjectObjectType", "CustomObjectRelatedObjectType", "build_object_type", + "clear_type_cache", ) +# Per-process cache of built GraphQL types, keyed by (cot id, cache_timestamp). +# A COT's cache_timestamp is bumped (auto_now, plus an explicit save() on every +# field add/edit/delete) whenever the type or any of its fields changes, so a +# cached entry can never go stale: a structural change changes the key and forces +# a rebuild. This lets a schema rebuild triggered by one COT reuse the +# already-built types of every other COT instead of re-running build_object_type +# (and its per-COT fields query) for all of them. +_type_cache = {} +_type_cache_lock = threading.RLock() + + +def clear_type_cache(): + """Drop all cached GraphQL types (used by tests).""" + with _type_cache_lock: + _type_cache.clear() + # Mapping of custom-field scalar types to the Python annotation Strawberry should # use. Relationship types (OBJECT / MULTIOBJECT) are handled separately via @@ -64,6 +80,36 @@ CustomFieldTypeChoices.TYPE_MULTIOBJECT, ) +_field_type_coverage_checked = False + + +def _warn_on_unmapped_field_types(): + """ + Warn once if a registered custom field type has no GraphQL mapping. + + SCALAR_TYPE_MAP and RELATIONSHIP_TYPES together re-enumerate the field types + owned by ``field_types.FIELD_TYPE_CLASS``. If a new field type is registered + there without a corresponding entry here, it would be silently omitted from + the GraphQL schema (see :func:`build_object_type`). Surface that drift loudly + rather than leaving the field quietly missing. + """ + global _field_type_coverage_checked + if _field_type_coverage_checked: + return + _field_type_coverage_checked = True + try: + from netbox_custom_objects.field_types import FIELD_TYPE_CLASS + except Exception: # noqa: BLE001 - never break schema build over a self-check + return + unmapped = set(FIELD_TYPE_CLASS) - set(SCALAR_TYPE_MAP) - set(RELATIONSHIP_TYPES) + if unmapped: + logger.warning( + "Custom field type(s) %s have no GraphQL mapping and will be omitted from " + "the GraphQL schema; add them to SCALAR_TYPE_MAP or RELATIONSHIP_TYPES in " + "netbox_custom_objects/graphql/types.py.", + sorted(unmapped), + ) + @strawberry.type class CustomObjectRelatedObjectType: @@ -115,20 +161,33 @@ def _related_repr(obj): ) -def _make_object_resolver(field_name): +def _make_object_resolver(field): """Build a resolver returning the single related object for an OBJECT field.""" - - @strawberry_django.field(description=f"Related object referenced by '{field_name}'") + field_name = field.name + # Optimize the parent list query so each row's related object is fetched in + # bulk rather than one query per row. NetBox's DjangoOptimizerExtension reads + # these hints off the field even though it has a custom resolver. A + # non-polymorphic OBJECT field is a plain ForeignKey (select_related); a + # polymorphic one is a GenericForeignKey, for which only prefetch_related works. + hint = {"prefetch_related": field_name} if field.is_polymorphic else {"select_related": field_name} + + @strawberry_django.field(description=f"Related object referenced by '{field_name}'", **hint) def resolver(self) -> Optional[CustomObjectRelatedObjectType]: return _related_repr(getattr(self, field_name, None)) return resolver -def _make_multiobject_resolver(field_name): +def _make_multiobject_resolver(field): """Build a resolver returning related objects for a MULTIOBJECT field.""" - - @strawberry_django.field(description=f"Related objects referenced by '{field_name}'") + field_name = field.name + # A non-polymorphic MULTIOBJECT field is a real ManyToManyField and can be + # prefetched to avoid one query per parent row. A polymorphic one is backed + # by a custom descriptor (PolymorphicM2MDescriptor), not a Django relation, so + # it can't be prefetched here and remains one query per row. + hint = {} if field.is_polymorphic else {"prefetch_related": field_name} + + @strawberry_django.field(description=f"Related objects referenced by '{field_name}'", **hint) def resolver(self) -> List[CustomObjectRelatedObjectType]: manager = getattr(self, field_name, None) if manager is None: @@ -154,10 +213,33 @@ def build_object_type(custom_object_type): model, with one GraphQL field per custom field plus the inherited base fields (id, display, tags, changelog, created, last_updated). """ + _warn_on_unmapped_field_types() + model = custom_object_type.get_model() if model is None: return None + # Reuse the already-built type when this COT's structure is unchanged (see + # _type_cache). cache_timestamp moves on every structural change, so this can + # never serve a type that no longer matches the model. + cache_key = (custom_object_type.id, custom_object_type.cache_timestamp) + with _type_cache_lock: + cached = _type_cache.get(cache_key) + if cached is not None: + return cached + + gql_type = _build_object_type(custom_object_type, model) + + with _type_cache_lock: + _type_cache[cache_key] = gql_type + # Bound growth: drop now-superseded entries for this COT. + for key in [k for k in _type_cache if k[0] == cache_key[0] and k != cache_key]: + del _type_cache[key] + return gql_type + + +def _build_object_type(custom_object_type, model): + """Construct a fresh Strawberry type for ``custom_object_type`` (uncached).""" type_name = f"{model.__name__}Type" namespace = { @@ -169,9 +251,9 @@ def build_object_type(custom_object_type): field_name = field.name if field.type in RELATIONSHIP_TYPES: if field.type == CustomFieldTypeChoices.TYPE_OBJECT: - namespace[field_name] = _make_object_resolver(field_name) + namespace[field_name] = _make_object_resolver(field) else: - namespace[field_name] = _make_multiobject_resolver(field_name) + namespace[field_name] = _make_multiobject_resolver(field) continue annotation = SCALAR_TYPE_MAP.get(field.type) diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index f403ca22..c9be53c3 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -74,6 +74,39 @@ def test_query_field_name_dedupes_collisions(self): self.assertEqual(first, "a_b") self.assertEqual(second, f"a_b_{b.id}") + def test_query_field_name_avoids_list_suffix_collision(self): + # A type whose slug sanitizes to '_list' must not claim the same + # GraphQL field as another type's auto-generated '_list' list field. + used = set() + a = self.create_custom_object_type(name="Foo", slug="foo") + b = self.create_custom_object_type( + name="Foo List", slug="foo-list", verbose_name_plural="Foo Lists" + ) + name_a = _query_field_name(a, used) + name_b = _query_field_name(b, used) + self.assertEqual(name_a, "foo") + # 'foo_list' is already reserved as A's list field, so B is disambiguated. + self.assertNotEqual(name_b, "foo_list") + # All four generated names (singular + list for each type) stay distinct. + all_fields = {name_a, f"{name_a}_list", name_b, f"{name_b}_list"} + self.assertEqual(len(all_fields), 4) + + def test_build_object_type_is_cached_per_structure(self): + # An unchanged COT reuses its built type; a structural change (which bumps + # cache_timestamp) forces a rebuild. + from netbox_custom_objects.graphql import types as types_module + + types_module.clear_type_cache() + self.addCleanup(types_module.clear_type_cache) + + cot = self.create_simple_custom_object_type(name="Cached", slug="cached") + first = build_object_type(cot) + self.assertIs(first, build_object_type(cot)) + + self.create_custom_object_type_field(cot, name="extra", label="Extra", type="text") + cot.refresh_from_db() + self.assertIsNot(first, build_object_type(cot)) + def test_build_query_classes_skipped_during_tests(self): # ``test`` is on sys.argv during the suite, so the startup builder # short-circuits to an empty list rather than touching the DB. From 94d0777b84e437340a0d053a125c8bcf38e1e9a3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 8 Jun 2026 19:27:46 -0700 Subject: [PATCH 073/115] update to real netbox graphql objects --- docs/graphql.md | 57 ++- netbox_custom_objects/field_types.py | 36 ++ netbox_custom_objects/graphql/schema.py | 7 +- netbox_custom_objects/graphql/types.py | 386 ++++++++++++++------ netbox_custom_objects/tests/test_graphql.py | 341 ++++++++++++----- 5 files changed, 608 insertions(+), 219 deletions(-) diff --git a/docs/graphql.md b/docs/graphql.md index 13c6bced..1c99125b 100644 --- a/docs/graphql.md +++ b/docs/graphql.md @@ -20,11 +20,17 @@ GraphQL requests use the same authentication as the REST API. Pass a token in th Authorization: Token ``` -Object-level permissions are enforced: queries only return custom objects the -authenticated user has permission to view. +Object-level permissions are enforced. The top-level `` / `_list` +queries only return custom objects the authenticated user has permission to view, +and objects reached through a relationship field are likewise filtered to those +the user may view. ## Querying a list +The fields you can select depend on how the Custom Object Type is defined. Every +type exposes `id` and `display`; the examples below also assume the type defines +custom fields named `name` and `description`. + ```graphql query { dhcp_scope_list { @@ -36,6 +42,10 @@ query { } ``` +> **Tip:** `display` is the only human-readable field guaranteed to exist on every +> type — it is the value of the type's primary field. `name` exists only if the +> type defines a custom field literally named `name`. + ## Querying a single object ```graphql @@ -43,7 +53,6 @@ query { dhcp_scope(id: 42) { id display - name } } ``` @@ -77,29 +86,51 @@ Scalar custom fields map to their natural GraphQL types: ### Relationship field types -Object and multi-object fields (including polymorphic ones) resolve to a uniform -`CustomObjectRelatedObjectType`, because a relationship may point at any NetBox -model or another custom object: +Object and multi-object fields resolve to the **native GraphQL type** of their +target, so a related object is fully traversable exactly as it would be when +queried directly. A field pointing at a Site resolves to NetBox's `SiteType`: ```graphql query { server_list { name - primary_site { # an object (single) field + primary_site { # an object (single) field → SiteType id - object_type # e.g. "dcim.site" - display - url + name + region { name } # traverse into the related object's own fields } - interfaces { # a multi-object (list) field + interfaces { # a multi-object (list) field → [InterfaceType] id - object_type - display + name } } } ``` +#### Polymorphic fields + +A polymorphic relationship field may point at several different model types. It +resolves to a GraphQL **union** of those types; select fields per type with inline +fragments: + +```graphql +query { + binding_list { + name + target { # polymorphic object field + ... on SiteType { id name } + ... on DeviceType { id name } + } + } +} +``` + +#### Fallback for targets without a GraphQL type + +A small number of NetBox models are not exposed in GraphQL. When a relationship +field's target has no native GraphQL type, that field falls back to a uniform +`CustomObjectRelatedObjectType` so it is never silently dropped: + | Field | Description | |-------|-------------| | `id` | The referenced object's primary key. | diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 52d1d800..4af2aff1 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -1,8 +1,12 @@ +import datetime +import decimal import hashlib import json import logging +from typing import List import django_tables2 as tables +from strawberry.scalars import JSON from django import forms from django.apps import apps from django.contrib.contenttypes.fields import GenericForeignKey @@ -170,6 +174,18 @@ def _make_lazy_cot_fk(cot, field, on_delete, **field_kwargs): class FieldType: + # The Python annotation Strawberry should use when exposing this field type + # as a scalar GraphQL field. ``None`` means the type is not exposed as a + # plain scalar — relationship types (OBJECT / MULTIOBJECT) are handled by + # dedicated resolvers in ``graphql/types.py`` instead. Co-locating the + # GraphQL mapping here (next to the model/serializer/form mappings) keeps the + # GraphQL schema in sync automatically: a new field type that needs a scalar + # annotation sets it on its own subclass. + graphql_annotation = None + + def get_graphql_annotation(self): + return self.graphql_annotation + def get_display_value(self, instance, field_name): """ This value is used as the object title in the Custom Object detail view. @@ -245,6 +261,7 @@ def create_m2m_table(self, instance, model, field_name, schema_conn=None): ... class TextFieldType(FieldType): + graphql_annotation = str def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) @@ -277,6 +294,8 @@ def get_filterform_field(self, field, **kwargs): class LongTextFieldType(FieldType): + graphql_annotation = str + def get_filterform_field(self, field, **kwargs): return forms.CharField( label=field, @@ -314,6 +333,7 @@ def render_table_column(self, value): class IntegerFieldType(FieldType): + graphql_annotation = int def get_model_field(self, field, **kwargs): # TODO: handle all args for IntegerField @@ -337,6 +357,8 @@ def get_form_field(self, field, **kwargs): class DecimalFieldType(FieldType): + graphql_annotation = decimal.Decimal + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -368,6 +390,8 @@ def get_filterform_field(self, field, **kwargs): class BooleanFieldType(FieldType): + graphql_annotation = bool + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -402,6 +426,8 @@ def get_table_column_field(self, field, **kwargs): class DateFieldType(FieldType): + graphql_annotation = datetime.date + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -421,6 +447,8 @@ def get_filterform_field(self, field, **kwargs): class DateTimeFieldType(FieldType): + graphql_annotation = datetime.datetime + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -440,6 +468,8 @@ def get_filterform_field(self, field, **kwargs): class URLFieldType(FieldType): + graphql_annotation = str + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -458,6 +488,8 @@ def get_filterform_field(self, field, **kwargs): class JSONFieldType(FieldType): + graphql_annotation = JSON + def get_model_field(self, field, **kwargs): field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) @@ -477,6 +509,8 @@ def get_filterform_field(self, field, **kwargs): class SelectFieldType(FieldType): + graphql_annotation = str + def get_display_value(self, instance, field_name): value = getattr(instance, field_name) if value is None: @@ -546,6 +580,8 @@ def get_form_field(self, field, for_csv_import=False, **kwargs): class MultiSelectFieldType(FieldType): + graphql_annotation = List[str] + def get_filterform_field(self, field, **kwargs): choices = field.choice_set.choices return DynamicMultipleChoiceField( diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 5c6f1fb8..54fb909e 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -112,7 +112,12 @@ class is avoided. return [] attrs["__annotations__"] = annotations - query_cls = strawberry.type(type("CustomObjectsQuery", (), attrs)) + # The GraphQL type name must be "Query": strawberry-django only attaches the + # single-object ``id`` lookup argument to fields whose origin type is named + # "Query" (see strawberry_django.filters: ``is_root_query``). NetBox names + # every per-app query class "Query" for the same reason; our contributed class + # is mixed into NetBox's real Query as a base, so it must do likewise. + query_cls = strawberry.type(type("CustomObjectsQuery", (), attrs), name="Query") # Marker so live schema rebuilds can find and replace a stale instance of # this class among NetBox's Query bases. query_cls._nco_query = True diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index 84bff0b8..ee22942a 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -12,17 +12,21 @@ ``__init__.py``) whenever the set of custom object types or their fields changes, so a type created *after* startup appears without a NetBox restart. -Scalar fields are mapped to their natural GraphQL scalar types. Object and -multi-object (relationship) fields — including polymorphic ones — are exposed -through a single shared ``CustomObjectRelatedObjectType`` so that references to -*any* NetBox model or other custom object resolve uniformly. +Scalar fields are mapped to their natural GraphQL scalar types (the mapping +lives on each ``FieldType`` in ``field_types.py``). Object and multi-object +(relationship) fields resolve to the *native* NetBox GraphQL type of their +target — a field pointing at a Site resolves to NetBox's ``SiteType`` and is +fully traversable. Polymorphic relationship fields, which may point at several +model types, resolve to a Strawberry union of those native types (mirroring how +NetBox exposes ``assigned_object``/cable terminations). When a target model has +no registered GraphQL type, the field falls back to a lightweight, uniform +``CustomObjectRelatedObjectType`` stub so the field is never silently dropped. """ -import datetime -import decimal import logging +import re import threading -from typing import List, Optional +from typing import Annotated, List, Optional, Union import strawberry import strawberry_django @@ -30,7 +34,10 @@ from extras.choices import CustomFieldTypeChoices from extras.graphql.mixins import TagsMixin from netbox.graphql.types import BaseObjectType -from strawberry.scalars import JSON +from strawberry.types import Info + +from netbox_custom_objects.constants import APP_LABEL +from netbox_custom_objects.utilities import extract_cot_id_from_model_name logger = logging.getLogger("netbox_custom_objects.graphql") @@ -51,6 +58,18 @@ _type_cache = {} _type_cache_lock = threading.RLock() +# Tracks the COT ids whose GraphQL type is being built on the current thread, so +# that a relationship between two custom objects (A -> B -> A) does not recurse +# forever: a back-reference to a type still under construction falls back to the +# flat stub instead of rebuilding it. +_building = threading.local() + +# Lazily-built map of Django model class -> its registered NetBox strawberry +# GraphQL type. The app-defined types are static for the life of the process, so +# this is computed once. +_model_type_registry = None +_registry_lock = threading.RLock() + def clear_type_cache(): """Drop all cached GraphQL types (used by tests).""" @@ -58,69 +77,21 @@ def clear_type_cache(): _type_cache.clear() -# Mapping of custom-field scalar types to the Python annotation Strawberry should -# use. Relationship types (OBJECT / MULTIOBJECT) are handled separately via -# resolvers, so they are intentionally absent here. -SCALAR_TYPE_MAP = { - CustomFieldTypeChoices.TYPE_TEXT: str, - CustomFieldTypeChoices.TYPE_LONGTEXT: str, - CustomFieldTypeChoices.TYPE_URL: str, - CustomFieldTypeChoices.TYPE_SELECT: str, - CustomFieldTypeChoices.TYPE_INTEGER: int, - CustomFieldTypeChoices.TYPE_DECIMAL: decimal.Decimal, - CustomFieldTypeChoices.TYPE_BOOLEAN: bool, - CustomFieldTypeChoices.TYPE_DATE: datetime.date, - CustomFieldTypeChoices.TYPE_DATETIME: datetime.datetime, - CustomFieldTypeChoices.TYPE_JSON: JSON, - CustomFieldTypeChoices.TYPE_MULTISELECT: List[str], -} - RELATIONSHIP_TYPES = ( CustomFieldTypeChoices.TYPE_OBJECT, CustomFieldTypeChoices.TYPE_MULTIOBJECT, ) -_field_type_coverage_checked = False - - -def _warn_on_unmapped_field_types(): - """ - Warn once if a registered custom field type has no GraphQL mapping. - - SCALAR_TYPE_MAP and RELATIONSHIP_TYPES together re-enumerate the field types - owned by ``field_types.FIELD_TYPE_CLASS``. If a new field type is registered - there without a corresponding entry here, it would be silently omitted from - the GraphQL schema (see :func:`build_object_type`). Surface that drift loudly - rather than leaving the field quietly missing. - """ - global _field_type_coverage_checked - if _field_type_coverage_checked: - return - _field_type_coverage_checked = True - try: - from netbox_custom_objects.field_types import FIELD_TYPE_CLASS - except Exception: # noqa: BLE001 - never break schema build over a self-check - return - unmapped = set(FIELD_TYPE_CLASS) - set(SCALAR_TYPE_MAP) - set(RELATIONSHIP_TYPES) - if unmapped: - logger.warning( - "Custom field type(s) %s have no GraphQL mapping and will be omitted from " - "the GraphQL schema; add them to SCALAR_TYPE_MAP or RELATIONSHIP_TYPES in " - "netbox_custom_objects/graphql/types.py.", - sorted(unmapped), - ) - @strawberry.type class CustomObjectRelatedObjectType: """ - A lightweight, uniform representation of an object referenced by a custom - object's relationship field. + Fallback representation of an object referenced by a relationship field whose + target model has no registered NetBox GraphQL type. - Relationship fields can point at any NetBox model, another custom object, or - (for polymorphic fields) a mix of types, so a single concrete Strawberry type - per target is not feasible. This shared type exposes the information common - to every referenced object. + Most relationship targets resolve to their native GraphQL type and are fully + traversable; this uniform stub is only used when no such type exists, so the + field still exposes the basics rather than disappearing from the schema. """ id: int @@ -144,64 +115,265 @@ class CustomObjectObjectType(ChangelogMixin, TagsMixin, BaseObjectType): pass +def _in_progress_set(): + ids = getattr(_building, "cot_ids", None) + if ids is None: + ids = set() + _building.cot_ids = ids + return ids + + +def _request_user(info): + """Best-effort extraction of the requesting user from the GraphQL info context.""" + request = getattr(getattr(info, "context", None), "request", None) + return getattr(request, "user", None) + + +def _user_can_view(user, obj): + """ + Return whether ``user`` has NetBox 'view' permission for ``obj``. + + The top-level query restricts the custom objects themselves, but the objects + reached through their relationship fields are *not* covered by that check, so + each one must be gated individually or the field would leak objects the user + cannot see. + """ + if obj is None or user is None: + return False + if getattr(user, "is_superuser", False): + return True + manager = getattr(type(obj), "_default_manager", None) + if manager is None or not hasattr(manager, "restrict"): + # Target model isn't permission-aware; nothing to enforce. + return True + return manager.restrict(user, "view").filter(pk=obj.pk).exists() + + def _related_repr(obj): """Convert a referenced model instance into a ``CustomObjectRelatedObjectType``.""" if obj is None: return None - url = None try: url = obj.get_absolute_url() except Exception: # noqa: BLE001 - URL resolution is best-effort url = None return CustomObjectRelatedObjectType( id=obj.pk, - object_type=f"{obj._meta.app_label}.{obj._meta.model_name}", + object_type=obj._meta.label_lower, display=str(obj), url=url, ) -def _make_object_resolver(field): - """Build a resolver returning the single related object for an OBJECT field.""" - field_name = field.name - # Optimize the parent list query so each row's related object is fetched in - # bulk rather than one query per row. NetBox's DjangoOptimizerExtension reads - # these hints off the field even though it has a custom resolver. A - # non-polymorphic OBJECT field is a plain ForeignKey (select_related); a - # polymorphic one is a GenericForeignKey, for which only prefetch_related works. - hint = {"prefetch_related": field_name} if field.is_polymorphic else {"select_related": field_name} +def _build_model_type_registry(): + """ + Map every Django model to its registered NetBox strawberry GraphQL type. - @strawberry_django.field(description=f"Related object referenced by '{field_name}'", **hint) - def resolver(self) -> Optional[CustomObjectRelatedObjectType]: - return _related_repr(getattr(self, field_name, None)) + NetBox (and other plugins) declare their per-model types in + ``.graphql.types``. Each such type carries a strawberry-django + definition naming its model, so importing those modules and indexing by model + gives a model -> type lookup that relationship fields use to resolve their + target to a native, traversable type. + """ + from importlib import import_module - return resolver + from django.apps import apps as django_apps + from strawberry_django.utils.typing import get_django_definition + registry = {} + for app_config in django_apps.get_app_configs(): + if app_config.label == APP_LABEL: + # Custom-object types are resolved dynamically, not from a module. + continue + module_name = f"{app_config.name}.graphql.types" + try: + module = import_module(module_name) + except ModuleNotFoundError: + continue + except Exception: # noqa: BLE001 - a broken app module must not break the schema + logger.debug("Could not import %s for the GraphQL type registry", module_name, exc_info=True) + continue + for value in vars(module).values(): + if not isinstance(value, type): + continue + definition = get_django_definition(value) + if definition is None or definition.model is None: + continue + # Only index GraphQL *output* object types. Filters and inputs also + # carry a django definition (and a model), but using one as a field's + # type would break the schema. + sb_definition = getattr(value, "__strawberry_definition__", None) + if sb_definition is None or sb_definition.is_input or sb_definition.is_interface: + continue + registry.setdefault(definition.model, value) + return registry + + +def _get_model_type_registry(): + global _model_type_registry + if _model_type_registry is not None: + return _model_type_registry + with _registry_lock: + if _model_type_registry is None: + _model_type_registry = _build_model_type_registry() + return _model_type_registry + + +def _custom_object_graphql_type(model_name): + """Resolve a custom-object target (``tablemodel``) to its GraphQL type.""" + from netbox_custom_objects.models import CustomObjectType -def _make_multiobject_resolver(field): - """Build a resolver returning related objects for a MULTIOBJECT field.""" + try: + cot_id = extract_cot_id_from_model_name(model_name) + except Exception: # noqa: BLE001 + return None + if cot_id is None or cot_id in _in_progress_set(): + # No id, or a back-reference to a type still being built — fall back to + # the flat stub for this edge to avoid infinite recursion. + return None + cot = CustomObjectType.objects.filter(pk=cot_id).first() + if cot is None: + return None + try: + return build_object_type(cot) + except Exception: # noqa: BLE001 + logger.warning( + "Failed to build related custom-object GraphQL type for %r", model_name, exc_info=True + ) + return None + + +def _graphql_type_for_content_type(content_type): + """Return the native strawberry GraphQL type for ``content_type``, or ``None``.""" + model = content_type.model_class() + if model is None: + return None + if content_type.app_label == APP_LABEL: + return _custom_object_graphql_type(content_type.model) + return _get_model_type_registry().get(model) + + +def _field_target_content_types(field): + """Return the ContentType(s) a relationship field may point at.""" + if field.is_polymorphic: + return list(field.related_object_types.all()) + if field.related_object_type_id: + return [field.related_object_type] + return [] + + +def _resolve_relationship_members(field): + """ + Return ``(members, native_models)`` for a relationship field. + + ``members`` is the list of GraphQL types the field can resolve to — the + native type of each target that has one, plus the flat stub when at least one + target has no native type (or there are no targets at all). ``native_models`` + is the set of Django model classes that resolve to a native type, used at + resolve time to decide whether to return the raw instance or wrap it. + """ + members = [] + native_models = set() + needs_stub = False + for content_type in _field_target_content_types(field): + gql_type = _graphql_type_for_content_type(content_type) + model = content_type.model_class() + if gql_type is not None and model is not None: + if gql_type not in members: + members.append(gql_type) + native_models.add(model) + else: + needs_stub = True + if needs_stub or not members: + if CustomObjectRelatedObjectType not in members: + members.append(CustomObjectRelatedObjectType) + return members, native_models + + +def _relationship_union_name(field): + """A schema-unique, GraphQL-safe name for a polymorphic field's union type.""" + base = re.sub(r"[^0-9a-zA-Z_]", "_", field.name) + return f"CustomObject{field.custom_object_type_id}_{base}_Related" + + +def _relationship_annotation(members, is_list, union_name): + """Build the resolver return annotation from the resolved member types.""" + if len(members) == 1: + inner = members[0] + else: + inner = Annotated[Union[tuple(members)], strawberry.union(union_name)] + return List[inner] if is_list else Optional[inner] + + +def _coerce_related(obj, native_models): + """Return ``obj`` itself if its model resolves natively, else the flat stub.""" + if type(obj) in native_models or any(isinstance(obj, model) for model in native_models): + return obj + return _related_repr(obj) + + +def _make_relationship_resolver(field): + """ + Build a resolver for an OBJECT or MULTIOBJECT relationship field. + + The resolver returns the referenced object(s) as their native GraphQL + type(s) (or the flat stub for targets without one), filtered to those the + requesting user may view. + """ field_name = field.name - # A non-polymorphic MULTIOBJECT field is a real ManyToManyField and can be - # prefetched to avoid one query per parent row. A polymorphic one is backed - # by a custom descriptor (PolymorphicM2MDescriptor), not a Django relation, so - # it can't be prefetched here and remains one query per row. - hint = {} if field.is_polymorphic else {"prefetch_related": field_name} - - @strawberry_django.field(description=f"Related objects referenced by '{field_name}'", **hint) - def resolver(self) -> List[CustomObjectRelatedObjectType]: - manager = getattr(self, field_name, None) - if manager is None: - return [] - try: - related = list(manager.all()) - except Exception: # noqa: BLE001 - never let one field break the query - logger.warning( - "Failed to resolve multi-object GraphQL field %r", field_name, exc_info=True - ) - return [] - return [_related_repr(obj) for obj in related if obj is not None] + is_list = field.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT - return resolver + members, native_models = _resolve_relationship_members(field) + if not members: + return None + annotation = _relationship_annotation(members, is_list, _relationship_union_name(field)) + + # Query-optimisation hints read by NetBox's DjangoOptimizerExtension. A + # non-polymorphic OBJECT field is a ForeignKey (select_related); a polymorphic + # one is a GenericForeignKey (prefetch_related only). A non-polymorphic + # MULTIOBJECT field is a real M2M (prefetch_related); a polymorphic one is a + # custom descriptor that can't be prefetched. + if is_list: + hint = {} if field.is_polymorphic else {"prefetch_related": field_name} + description = f"Related objects referenced by '{field_name}'" + else: + hint = {"prefetch_related": field_name} if field.is_polymorphic else {"select_related": field_name} + description = f"Related object referenced by '{field_name}'" + + def resolver(self, info: Info): + user = _request_user(info) + value = getattr(self, field_name, None) + if is_list: + if value is None: + return [] + try: + related = list(value.all()) + except Exception: # noqa: BLE001 - never let one field break the query + logger.warning( + "Failed to resolve multi-object GraphQL field %r", field_name, exc_info=True + ) + return [] + return [ + _coerce_related(obj, native_models) + for obj in related + if obj is not None and _user_can_view(user, obj) + ] + if value is None or not _user_can_view(user, value): + return None + return _coerce_related(value, native_models) + + resolver.__annotations__ = {"info": Info, "return": annotation} + return strawberry_django.field(description=description, **hint)(resolver) + + +def _scalar_annotation_for(field_type): + """Return the GraphQL scalar annotation for a field type, or ``None``.""" + from netbox_custom_objects.field_types import FIELD_TYPE_CLASS + + field_type_cls = FIELD_TYPE_CLASS.get(field_type) + if field_type_cls is None: + return None + return field_type_cls().get_graphql_annotation() def build_object_type(custom_object_type): @@ -213,8 +385,6 @@ def build_object_type(custom_object_type): model, with one GraphQL field per custom field plus the inherited base fields (id, display, tags, changelog, created, last_updated). """ - _warn_on_unmapped_field_types() - model = custom_object_type.get_model() if model is None: return None @@ -228,7 +398,12 @@ def build_object_type(custom_object_type): if cached is not None: return cached - gql_type = _build_object_type(custom_object_type, model) + in_progress = _in_progress_set() + in_progress.add(custom_object_type.id) + try: + gql_type = _build_object_type(custom_object_type, model) + finally: + in_progress.discard(custom_object_type.id) with _type_cache_lock: _type_cache[cache_key] = gql_type @@ -250,13 +425,12 @@ def _build_object_type(custom_object_type, model): for field in custom_object_type.fields.all(): field_name = field.name if field.type in RELATIONSHIP_TYPES: - if field.type == CustomFieldTypeChoices.TYPE_OBJECT: - namespace[field_name] = _make_object_resolver(field) - else: - namespace[field_name] = _make_multiobject_resolver(field) + resolver = _make_relationship_resolver(field) + if resolver is not None: + namespace[field_name] = resolver continue - annotation = SCALAR_TYPE_MAP.get(field.type) + annotation = _scalar_annotation_for(field.type) if annotation is None: logger.debug( "Skipping custom field %r of unsupported GraphQL type %r", diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index c9be53c3..de125215 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -1,23 +1,32 @@ """ Tests for GraphQL support for custom objects. -The plugin contributes its GraphQL schema at startup, which is intentionally -skipped during the test run (``should_skip_dynamic_model_creation()`` returns -True when ``test`` is on the command line — see ``__init__.py``). We therefore -exercise the schema-generation functions directly: build a Strawberry schema -from custom object types created in each test and execute queries against it -in-process, mirroring what NetBox does at boot. +Two layers are exercised: + +* Unit tests for the schema-generation helpers (query-field naming, per-structure + type caching, the live-schema signature/rebuild machinery). The plugin's + GraphQL schema contribution is intentionally skipped during the test run + (``should_skip_dynamic_model_creation()`` returns True when ``test`` is on the + command line), so these call the generation functions directly. + +* End-to-end tests that drive the real ``/graphql/`` HTTP endpoint with token + authentication, modelled on NetBox's own GraphQL test pattern. These patch the + startup guard off so the live schema (installed via the view patch in + ``__init__.py``) is built and bound per request, exactly as in production. """ -from typing import List +import json from unittest import mock import strawberry -import strawberry_django -from django.test import TestCase -from strawberry.schema.config import StrawberryConfig +from django.test import TestCase, override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APIClient -from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Site +from core.models import ObjectType +from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Region, Site +from users.models import ObjectPermission, Token from netbox_custom_objects.graphql import live as live_module from netbox_custom_objects.graphql import schema as schema_module @@ -27,33 +36,19 @@ from .base import CustomObjectsTestCase -class _Context: - """Minimal stand-in for Strawberry-Django's request context.""" - - def __init__(self, request): - self.request = request - - -def build_test_schema(custom_object_types): - """Assemble a Strawberry schema from the given COTs, bypassing the startup guard.""" - annotations = {} - attrs = {} - used_names = set() - for cot in custom_object_types: - gql_type = build_object_type(cot) - field_name = _query_field_name(cot, used_names) - list_name = f"{field_name}_list" - annotations[field_name] = gql_type - attrs[field_name] = strawberry_django.field() - annotations[list_name] = List[gql_type] - attrs[list_name] = strawberry_django.field() - attrs["__annotations__"] = annotations - # The GraphQL type name must be "Query" for strawberry-django to attach the - # single-object `id` lookup argument (it only does so on the root Query - # type). In production our class is mixed into NetBox's real Query, which is - # named "Query"; here we name it directly. - query_cls = strawberry.type(type("Query", (), attrs)) - return strawberry.Schema(query=query_cls, config=StrawberryConfig(auto_camel_case=False)) +def create_token(user): + """Create an API token, plaintext key, across NetBox token versions.""" + try: + # NetBox >= 4.5 + from users.choices import TokenVersionChoices + token = Token(version=TokenVersionChoices.V1, user=user) + token.save() + return token.token + except ImportError: + # NetBox < 4.5 + token = Token(user=user) + token.save() + return token.key class GraphQLSchemaGenerationTestCase(CustomObjectsTestCase, TestCase): @@ -122,6 +117,8 @@ def test_real_builder_produces_assemblable_query(self): # Exercise the actual production builder (normally skipped during tests) # to confirm it yields a Query class that assembles into a valid schema # exposing the expected per-type fields. + from strawberry.schema.config import StrawberryConfig + self.create_simple_custom_object_type(name="Gadget", slug="gadget") with mock.patch( "netbox_custom_objects.CustomObjectsPluginConfig." @@ -211,52 +208,85 @@ def test_live_schema_drops_deleted_type(self): self.assertNotIn("temp_type", str(live_module.get_live_schema())) -class GraphQLQueryTestCase(CustomObjectsTestCase, TestCase): - """End-to-end query execution against a generated schema.""" +@override_settings(LOGIN_REQUIRED=True) +class GraphQLEndpointTestCase(CustomObjectsTestCase, TestCase): + """ + End-to-end tests against the real ``/graphql/`` HTTP endpoint. + + Patches the startup guard off so the live schema is built and bound to the + (monkey-patched) GraphQL view per request, and authenticates with a token — + the same path a real client takes. The user is a superuser so these tests + focus on schema/resolution correctness rather than permission wiring + (permissions are covered separately below). + """ def setUp(self): super().setUp() - # restrict() returns everything for a superuser, keeping these tests - # focused on schema generation rather than permission wiring. + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + patcher = mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + self.user.is_superuser = True self.user.save() - self.request = self._make_request() - - def _make_request(self): - from django.test import RequestFactory - - request = RequestFactory().post("/graphql/") - request.user = self.user - return request - - def _execute(self, schema, query): - result = schema.execute_sync(query, context_value=_Context(self.request)) - self.assertIsNone(result.errors, msg=str(result.errors)) - return result.data + # A fresh APIClient using token auth (not the session login set up by the + # base class), mirroring how an API client reaches the endpoint. + self.client = APIClient() + token_key = create_token(self.user) + self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} + self.url = reverse("graphql") + + def _gql(self, query): + response = self.client.post( + self.url, data={"query": query}, format="json", **self.header + ) + self.assertEqual( + response.status_code, status.HTTP_200_OK, getattr(response, "content", response) + ) + payload = json.loads(response.content) + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + return payload["data"] - def _make_device(self): - manufacturer = Manufacturer.objects.create(name="Mfr", slug="mfr") - device_type = DeviceType.objects.create( + def _make_device(self, name="dev1"): + manufacturer, _ = Manufacturer.objects.get_or_create(name="Mfr", slug="mfr") + device_type, _ = DeviceType.objects.get_or_create( manufacturer=manufacturer, model="Model", slug="model" ) - role = DeviceRole.objects.create(name="Role", slug="role") - site = Site.objects.create(name="Site", slug="site") + role, _ = DeviceRole.objects.get_or_create(name="Role", slug="role") + site = self._make_site() return Device.objects.create( - name="dev1", device_type=device_type, role=role, site=site + name=name, device_type=device_type, role=role, site=site ) + def _make_site(self, name="Site", slug="site", region=None): + return Site.objects.create(name=name, slug=slug, region=region) + + def _site_object_field_type(self, name="Server", slug="server"): + """A COT with a primary text field and a single-object field → Site.""" + cot = self.create_custom_object_type(name=name, slug=slug) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + cot, name="site", label="Site", type="object", + related_object_type=self.get_site_object_type(), + ) + return cot + def test_scalar_fields_query(self): cot = self.create_complex_custom_object_type(name="Asset", slug="asset") model = cot.get_model() model.objects.create(name="First", count=7, active=True, status="choice1") - schema = build_test_schema([cot]) - data = self._execute( - schema, - "{ asset_list { id name count active status } }", - ) + data = self._gql("{ asset_list { id display name count active status } }") rows = data["asset_list"] self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["display"], "First") self.assertEqual(rows[0]["name"], "First") self.assertEqual(rows[0]["count"], 7) self.assertTrue(rows[0]["active"]) @@ -267,58 +297,171 @@ def test_single_object_query_by_id(self): model = cot.get_model() instance = model.objects.create(name="Hello", description="world") - schema = build_test_schema([cot]) - data = self._execute( - schema, - f'{{ note(id: {instance.pk}) {{ id name description }} }}', + data = self._gql( + f"{{ note(id: {instance.pk}) {{ id display name description }} }}" ) self.assertEqual(data["note"]["name"], "Hello") self.assertEqual(data["note"]["description"], "world") - def test_object_relationship_field(self): - device = self._make_device() - cot = self.create_complex_custom_object_type(name="Link", slug="link") + def test_object_relationship_resolves_to_native_site_type(self): + # A single-object field pointing at a Site must resolve to NetBox's + # SiteType and be fully traversable (including nested relations like + # region) — not a flat stub. + region = Region.objects.create(name="West", slug="west") + site = self._make_site(name="HQ", slug="hq", region=region) + cot = self._site_object_field_type() model = cot.get_model() - model.objects.create(name="L1", device=device) + model.objects.create(name="S1", site=site) - schema = build_test_schema([cot]) - data = self._execute( - schema, - "{ link_list { name device { id display object_type } } }", + data = self._gql( + "{ server_list { name site { id name slug region { name } } } }" ) - related = data["link_list"][0]["device"] - self.assertEqual(related["id"], device.pk) - self.assertEqual(related["object_type"], "dcim.device") - self.assertEqual(related["display"], str(device)) - - def test_multiobject_relationship_field(self): - device = self._make_device() - cot = self.create_multi_object_custom_object_type(name="Group", slug="group") + related = data["server_list"][0]["site"] + self.assertEqual(related["id"], str(site.pk)) + self.assertEqual(related["name"], "HQ") + self.assertEqual(related["slug"], "hq") + # Deep traversal into the related object's own relations proves it is the + # native SiteType, not the flat CustomObjectRelatedObjectType. + self.assertEqual(related["region"]["name"], "West") + + def test_multiobject_relationship_resolves_to_native_device_type(self): + device = self._make_device(name="dev-a") + # 'devgroup' rather than 'group': a slug of 'group' would collide with + # NetBox's own built-in `group_list` root query field (user groups). + cot = self.create_multi_object_custom_object_type(name="DevGroup", slug="devgroup") model = cot.get_model() instance = model.objects.create(name="G1") instance.devices.add(device) - schema = build_test_schema([cot]) - data = self._execute( - schema, - "{ group_list { name devices { id object_type } } }", - ) - devices = data["group_list"][0]["devices"] + data = self._gql("{ devgroup_list { name devices { id name role { name } } } }") + devices = data["devgroup_list"][0]["devices"] self.assertEqual(len(devices), 1) - self.assertEqual(devices[0]["id"], device.pk) - self.assertEqual(devices[0]["object_type"], "dcim.device") + self.assertEqual(devices[0]["id"], str(device.pk)) + # 'name'/'role' are Device fields → confirms native DeviceType. + self.assertEqual(devices[0]["name"], "dev-a") + self.assertEqual(devices[0]["role"]["name"], "Role") + + def test_polymorphic_object_field_resolves_to_union(self): + # A polymorphic single-object field exposes a union of its target types; + # an instance pointing at a Site resolves through the SiteType arm. + site = self._make_site(name="PolySite", slug="polysite") + cot = self.create_custom_object_type(name="Binding", slug="binding") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_polymorphic_field( + cot, + [self.get_site_object_type(), self.get_device_object_type()], + name="target", type="object", + ) + model = cot.get_model() + model.objects.create(name="B1", target=site) + + # Alias the per-type 'name' selections: Site.name is String! while + # Device.name is String, and GraphQL's same-response-shape rule forbids + # selecting both under one response key. + data = self._gql( + "{ binding_list { name target { " + "... on SiteType { id siteName: name } " + "... on DeviceType { id deviceName: name } } } }" + ) + target = data["binding_list"][0]["target"] + self.assertEqual(target["id"], str(site.pk)) + self.assertEqual(target["siteName"], "PolySite") + + def test_multiple_types_in_one_schema(self): + # Several custom object types must all be queryable from the same schema. + a = self.create_simple_custom_object_type(name="Alpha", slug="alpha") + b = self.create_simple_custom_object_type(name="Beta", slug="beta") + a.get_model().objects.create(name="a1") + b.get_model().objects.create(name="b1") + + data = self._gql("{ alpha_list { name } beta_list { name } }") + self.assertEqual(data["alpha_list"][0]["name"], "a1") + self.assertEqual(data["beta_list"][0]["name"], "b1") def test_tags_and_base_fields(self): cot = self.create_simple_custom_object_type(name="Doc", slug="doc") model = cot.get_model() model.objects.create(name="D1") - schema = build_test_schema([cot]) - data = self._execute( - schema, - "{ doc_list { id display created tags { name } } }", - ) + data = self._gql("{ doc_list { id display created tags { name } } }") row = data["doc_list"][0] self.assertEqual(row["display"], "D1") self.assertIsNotNone(row["id"]) self.assertEqual(row["tags"], []) + + +@override_settings(LOGIN_REQUIRED=True) +class GraphQLPermissionTestCase(CustomObjectsTestCase, TestCase): + """ + Object-level view permissions must be enforced for objects reached through a + custom object's relationship fields, not just for the top-level objects. + """ + + def setUp(self): + super().setUp() + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + patcher = mock.patch( + "netbox_custom_objects.CustomObjectsPluginConfig." + "should_skip_dynamic_model_creation", + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + # A non-superuser; permissions are granted explicitly per test. + self.client = APIClient() + token_key = create_token(self.user) + self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} + self.url = reverse("graphql") + + # COT 'Server' with a single-object field → Site, holding one Site. + self.site = Site.objects.create(name="Secret", slug="secret") + self.cot = self.create_custom_object_type(name="Server", slug="server") + self.create_custom_object_type_field( + self.cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + self.cot, name="site", label="Site", type="object", + related_object_type=self.get_site_object_type(), + ) + self.model = self.cot.get_model() + self.model.objects.create(name="srv", site=self.site) + + def _grant(self, model_class, name): + perm = ObjectPermission(name=name, actions=["view"]) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model_class)) + return perm + + def _post(self, query): + response = self.client.post( + self.url, data={"query": query}, format="json", **self.header + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.content) + return json.loads(response.content) + + def test_related_object_hidden_without_permission(self): + # View permission on the custom object, but NOT on Site → the related + # site must be withheld (null), while the object itself is returned. + self._grant(self.model, "view-co") + payload = self._post("{ server_list { name site { id name } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + rows = payload["data"]["server_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "srv") + self.assertIsNone(rows[0]["site"]) + + def test_related_object_visible_with_permission(self): + # Granting view on Site as well makes the related object appear, proving + # the previous test's null was permission enforcement, not a broken field. + self._grant(self.model, "view-co") + self._grant(Site, "view-site") + payload = self._post("{ server_list { name site { id name } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + related = payload["data"]["server_list"][0]["site"] + self.assertEqual(related["id"], str(self.site.pk)) + self.assertEqual(related["name"], "Secret") From 6d3308733e16d18102cba686ebab5602fb0aa6d0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 08:43:07 -0700 Subject: [PATCH 074/115] add namespace --- docs/graphql.md | 21 ++++-- netbox_custom_objects/graphql/schema.py | 20 +++-- netbox_custom_objects/tests/test_graphql.py | 82 +++++++++++---------- 3 files changed, 74 insertions(+), 49 deletions(-) diff --git a/docs/graphql.md b/docs/graphql.md index 1c99125b..eced5c8c 100644 --- a/docs/graphql.md +++ b/docs/graphql.md @@ -7,9 +7,16 @@ each Custom Object Type you have defined, two root query fields are generated: - `` — fetch a single custom object by `id`. - `_list` — fetch a list of custom objects (paginated). -The `` is derived from the Custom Object Type's **slug**, with any -characters that are not valid in a GraphQL name replaced by underscores (for -example a type with slug `dhcp-scope` becomes `dhcp_scope` and `dhcp_scope_list`). +The `` is `custom_objects_`, where `` is the Custom Object +Type's **slug** with any characters that are not valid in a GraphQL name replaced +by underscores (for example a type with slug `dhcp-scope` becomes +`custom_objects_dhcp_scope` and `custom_objects_dhcp_scope_list`). + +The `custom_objects_` prefix namespaces these fields so they can never collide +with NetBox's own (or another plugin's) root query fields — every plugin's +GraphQL query is merged into NetBox's single global `Query`, so a bare, +slug-derived name like `site` or `group` would otherwise be shadowed by a core +model's field. ## Authentication @@ -33,7 +40,7 @@ custom fields named `name` and `description`. ```graphql query { - dhcp_scope_list { + custom_objects_dhcp_scope_list { id display name @@ -50,7 +57,7 @@ query { ```graphql query { - dhcp_scope(id: 42) { + custom_objects_dhcp_scope(id: 42) { id display } @@ -92,7 +99,7 @@ queried directly. A field pointing at a Site resolves to NetBox's `SiteType`: ```graphql query { - server_list { + custom_objects_server_list { name primary_site { # an object (single) field → SiteType id @@ -115,7 +122,7 @@ fragments: ```graphql query { - binding_list { + custom_objects_binding_list { name target { # polymorphic object field ... on SiteType { id name } diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 54fb909e..6e211959 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -27,17 +27,27 @@ logger = logging.getLogger("netbox_custom_objects.graphql") +# All custom-object root query fields are namespaced with this prefix so they +# cannot collide with NetBox's own (or another plugin's) root query fields — every +# plugin's query class is mixed into the single global ``Query`` type, so bare, +# slug-derived names like ``site``/``group`` would otherwise be shadowed by core. +# The prefix mirrors the ``custom_objects_`` table-naming convention. +QUERY_FIELD_PREFIX = "custom_objects_" + def _query_field_name(custom_object_type, used_names): """ Derive a GraphQL-safe, unique field name from a custom object type's slug. - GraphQL names must match ``[_A-Za-z][_0-9A-Za-z]*``; slugs may contain - hyphens. Collisions (after sanitisation) are disambiguated with the type id. + The name is namespaced with :data:`QUERY_FIELD_PREFIX` (see above) so it never + collides with core/plugin root query fields. GraphQL names must match + ``[_A-Za-z][_0-9A-Za-z]*``; slugs may contain hyphens. Collisions among custom + object types (after sanitisation) are disambiguated with the type id. """ - base = re.sub(r"[^0-9a-zA-Z_]", "_", (custom_object_type.slug or "").lower()) - if not base or base[0].isdigit(): - base = f"_{base}" + slug = re.sub(r"[^0-9a-zA-Z_]", "_", (custom_object_type.slug or "").lower()) + # The prefix guarantees a valid leading character, so no digit/empty guard is + # needed on the slug portion. + base = f"{QUERY_FIELD_PREFIX}{slug}" name = base # Reserve the singular field name *and* its ``_list`` companion together. # Checking/recording both prevents one type's list field from silently diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index de125215..01f19fd4 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -54,11 +54,13 @@ def create_token(user): class GraphQLSchemaGenerationTestCase(CustomObjectsTestCase, TestCase): """Tests for the schema/query-class generation helpers.""" - def test_query_field_name_sanitizes_slug(self): + def test_query_field_name_sanitizes_and_namespaces_slug(self): used = set() cot = self.create_custom_object_type(name="Widget", slug="my-widget") name = _query_field_name(cot, used) - self.assertEqual(name, "my_widget") + # Hyphens sanitised to underscores, and namespaced with the prefix so the + # field can never collide with a core/plugin root query field. + self.assertEqual(name, "custom_objects_my_widget") def test_query_field_name_dedupes_collisions(self): used = set() @@ -66,8 +68,8 @@ def test_query_field_name_dedupes_collisions(self): b = self.create_custom_object_type(name="B", slug="a_b", verbose_name_plural="Bs") first = _query_field_name(a, used) second = _query_field_name(b, used) - self.assertEqual(first, "a_b") - self.assertEqual(second, f"a_b_{b.id}") + self.assertEqual(first, "custom_objects_a_b") + self.assertEqual(second, f"custom_objects_a_b_{b.id}") def test_query_field_name_avoids_list_suffix_collision(self): # A type whose slug sanitizes to '_list' must not claim the same @@ -79,9 +81,10 @@ def test_query_field_name_avoids_list_suffix_collision(self): ) name_a = _query_field_name(a, used) name_b = _query_field_name(b, used) - self.assertEqual(name_a, "foo") - # 'foo_list' is already reserved as A's list field, so B is disambiguated. - self.assertNotEqual(name_b, "foo_list") + self.assertEqual(name_a, "custom_objects_foo") + # 'custom_objects_foo_list' is already reserved as A's list field, so B is + # disambiguated. + self.assertNotEqual(name_b, "custom_objects_foo_list") # All four generated names (singular + list for each type) stay distinct. all_fields = {name_a, f"{name_a}_list", name_b, f"{name_b}_list"} self.assertEqual(len(all_fields), 4) @@ -135,8 +138,8 @@ def test_real_builder_produces_assemblable_query(self): query=query_cls, config=StrawberryConfig(auto_camel_case=False) ) sdl = str(built) - self.assertIn("gadget", sdl) - self.assertIn("gadget_list", sdl) + self.assertIn("custom_objects_gadget", sdl) + self.assertIn("custom_objects_gadget_list", sdl) class GraphQLLiveSchemaTestCase(CustomObjectsTestCase, TestCase): @@ -177,7 +180,7 @@ def test_get_live_schema_rebuilds_on_new_type(self): # A type that does not exist yet must not be in the schema... first = live_module.get_live_schema() self.assertIsNotNone(first) - self.assertNotIn("runtime_thing", str(first)) + self.assertNotIn("custom_objects_runtime_thing", str(first)) # ...and must appear after creation, with no restart and without manually # clearing the cache (the signature change drives the rebuild). @@ -185,8 +188,8 @@ def test_get_live_schema_rebuilds_on_new_type(self): second = live_module.get_live_schema() self.assertIsNot(first, second) sdl = str(second) - self.assertIn("runtime_thing", sdl) - self.assertIn("runtime_thing_list", sdl) + self.assertIn("custom_objects_runtime_thing", sdl) + self.assertIn("custom_objects_runtime_thing_list", sdl) def test_get_live_schema_cached_when_unchanged(self): self.create_simple_custom_object_type(name="Stable", slug="stable") @@ -199,13 +202,13 @@ def test_live_schema_drops_deleted_type(self): from netbox_custom_objects.models import CustomObjectType cot = self.create_simple_custom_object_type(name="Temp", slug="temp_type") - self.assertIn("temp_type", str(live_module.get_live_schema())) + self.assertIn("custom_objects_temp_type", str(live_module.get_live_schema())) # Delete via the queryset rather than cot.delete(): the schema-drop # behaviour only depends on the row being gone (which changes the # signature and triggers a rebuild), and this avoids the unrelated COT # teardown machinery. CustomObjectType.objects.filter(pk=cot.pk).delete() - self.assertNotIn("temp_type", str(live_module.get_live_schema())) + self.assertNotIn("custom_objects_temp_type", str(live_module.get_live_schema())) @override_settings(LOGIN_REQUIRED=True) @@ -283,8 +286,8 @@ def test_scalar_fields_query(self): model = cot.get_model() model.objects.create(name="First", count=7, active=True, status="choice1") - data = self._gql("{ asset_list { id display name count active status } }") - rows = data["asset_list"] + data = self._gql("{ custom_objects_asset_list { id display name count active status } }") + rows = data["custom_objects_asset_list"] self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["display"], "First") self.assertEqual(rows[0]["name"], "First") @@ -298,10 +301,10 @@ def test_single_object_query_by_id(self): instance = model.objects.create(name="Hello", description="world") data = self._gql( - f"{{ note(id: {instance.pk}) {{ id display name description }} }}" + f"{{ custom_objects_note(id: {instance.pk}) {{ id display name description }} }}" ) - self.assertEqual(data["note"]["name"], "Hello") - self.assertEqual(data["note"]["description"], "world") + self.assertEqual(data["custom_objects_note"]["name"], "Hello") + self.assertEqual(data["custom_objects_note"]["description"], "world") def test_object_relationship_resolves_to_native_site_type(self): # A single-object field pointing at a Site must resolve to NetBox's @@ -314,9 +317,9 @@ def test_object_relationship_resolves_to_native_site_type(self): model.objects.create(name="S1", site=site) data = self._gql( - "{ server_list { name site { id name slug region { name } } } }" + "{ custom_objects_server_list { name site { id name slug region { name } } } }" ) - related = data["server_list"][0]["site"] + related = data["custom_objects_server_list"][0]["site"] self.assertEqual(related["id"], str(site.pk)) self.assertEqual(related["name"], "HQ") self.assertEqual(related["slug"], "hq") @@ -326,15 +329,18 @@ def test_object_relationship_resolves_to_native_site_type(self): def test_multiobject_relationship_resolves_to_native_device_type(self): device = self._make_device(name="dev-a") - # 'devgroup' rather than 'group': a slug of 'group' would collide with - # NetBox's own built-in `group_list` root query field (user groups). - cot = self.create_multi_object_custom_object_type(name="DevGroup", slug="devgroup") + # Slug 'group' would collide with NetBox's built-in `group_list` root query + # field (user groups) — but the `custom_objects_` prefix keeps it distinct, + # so this exercises both native multiobject resolution and the namespacing. + cot = self.create_multi_object_custom_object_type(name="Group", slug="group") model = cot.get_model() instance = model.objects.create(name="G1") instance.devices.add(device) - data = self._gql("{ devgroup_list { name devices { id name role { name } } } }") - devices = data["devgroup_list"][0]["devices"] + data = self._gql( + "{ custom_objects_group_list { name devices { id name role { name } } } }" + ) + devices = data["custom_objects_group_list"][0]["devices"] self.assertEqual(len(devices), 1) self.assertEqual(devices[0]["id"], str(device.pk)) # 'name'/'role' are Device fields → confirms native DeviceType. @@ -361,11 +367,11 @@ def test_polymorphic_object_field_resolves_to_union(self): # Device.name is String, and GraphQL's same-response-shape rule forbids # selecting both under one response key. data = self._gql( - "{ binding_list { name target { " + "{ custom_objects_binding_list { name target { " "... on SiteType { id siteName: name } " "... on DeviceType { id deviceName: name } } } }" ) - target = data["binding_list"][0]["target"] + target = data["custom_objects_binding_list"][0]["target"] self.assertEqual(target["id"], str(site.pk)) self.assertEqual(target["siteName"], "PolySite") @@ -376,17 +382,19 @@ def test_multiple_types_in_one_schema(self): a.get_model().objects.create(name="a1") b.get_model().objects.create(name="b1") - data = self._gql("{ alpha_list { name } beta_list { name } }") - self.assertEqual(data["alpha_list"][0]["name"], "a1") - self.assertEqual(data["beta_list"][0]["name"], "b1") + data = self._gql( + "{ custom_objects_alpha_list { name } custom_objects_beta_list { name } }" + ) + self.assertEqual(data["custom_objects_alpha_list"][0]["name"], "a1") + self.assertEqual(data["custom_objects_beta_list"][0]["name"], "b1") def test_tags_and_base_fields(self): cot = self.create_simple_custom_object_type(name="Doc", slug="doc") model = cot.get_model() model.objects.create(name="D1") - data = self._gql("{ doc_list { id display created tags { name } } }") - row = data["doc_list"][0] + data = self._gql("{ custom_objects_doc_list { id display created tags { name } } }") + row = data["custom_objects_doc_list"][0] self.assertEqual(row["display"], "D1") self.assertIsNotNone(row["id"]) self.assertEqual(row["tags"], []) @@ -448,9 +456,9 @@ def test_related_object_hidden_without_permission(self): # View permission on the custom object, but NOT on Site → the related # site must be withheld (null), while the object itself is returned. self._grant(self.model, "view-co") - payload = self._post("{ server_list { name site { id name } } }") + payload = self._post("{ custom_objects_server_list { name site { id name } } }") self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) - rows = payload["data"]["server_list"] + rows = payload["data"]["custom_objects_server_list"] self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["name"], "srv") self.assertIsNone(rows[0]["site"]) @@ -460,8 +468,8 @@ def test_related_object_visible_with_permission(self): # the previous test's null was permission enforcement, not a broken field. self._grant(self.model, "view-co") self._grant(Site, "view-site") - payload = self._post("{ server_list { name site { id name } } }") + payload = self._post("{ custom_objects_server_list { name site { id name } } }") self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) - related = payload["data"]["server_list"][0]["site"] + related = payload["data"]["custom_objects_server_list"][0]["site"] self.assertEqual(related["id"], str(self.site.pk)) self.assertEqual(related["name"], "Secret") From d7808727eebd054b244a972b88fb9438b84f0370 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 09:02:03 -0700 Subject: [PATCH 075/115] update docs --- docs/graphql.md | 27 ++++++++++++++++++ .../media/graphql/graphiql-refresh-schema.png | Bin 0 -> 76817 bytes 2 files changed, 27 insertions(+) create mode 100644 docs/media/graphql/graphiql-refresh-schema.png diff --git a/docs/graphql.md b/docs/graphql.md index eced5c8c..84139cec 100644 --- a/docs/graphql.md +++ b/docs/graphql.md @@ -32,6 +32,33 @@ queries only return custom objects the authenticated user has permission to view and objects reached through a relationship field are likewise filtered to those the user may view. +## Using the GraphiQL explorer + +NetBox's built-in GraphiQL explorer (at `/graphql/`) loads the GraphQL schema +**once when the page opens** and caches it for the life of the page. Because +custom object types and their fields can change at runtime, the explorer's +autocomplete suggestions and **Documentation** panel can go stale. + +If you **add, remove, or change a custom object type's fields**, or **add a new +custom object type**, while GraphiQL is open, those changes will not appear in the +field autocomplete or the Docs panel until GraphiQL re-reads the schema. (The +change is already live on the server — a query using the new field works +immediately — it is only GraphiQL's cached copy of the schema that lags.) + +To refresh it, click the **Re-fetch GraphQL schema** button — the circular-arrow +icon in the GraphiQL toolbar (lower left). This re-reads the schema *without* +reloading the page, so your current query is preserved: + +![The Re-fetch GraphQL schema button in the GraphiQL toolbar](media/graphql/graphiql-refresh-schema.png) + +Reloading the whole browser page has the same effect, but discards whatever you +have typed in the editor. + +!!! note + This only affects the interactive explorer. Programmatic API clients fetch the + schema via introspection whenever they choose, and always query against the + current server-side schema. + ## Querying a list The fields you can select depend on how the Custom Object Type is defined. Every diff --git a/docs/media/graphql/graphiql-refresh-schema.png b/docs/media/graphql/graphiql-refresh-schema.png new file mode 100644 index 0000000000000000000000000000000000000000..367cc3822601ef117f31c104417ef9624672cecb GIT binary patch literal 76817 zcmeEuXIN9&-mi)s6cGgh5rt7iq)V?#Q|U!PS{NcC1VlOnLS!5PsVYr6O0N;=B}Anu z2$5csC@u5|A%r9(xtlrX%zNgY0q%$U@$x)*vPpLKTI*ka|GL(5V?*6zM|qF#*|X=^ zt(#h=d-fa}+_Q(}$>D>*C+_&yqkH!31v_hM8bdU7HQim^JlG?bYk3^L^Km}DDX;KFo9-*Y1M|^Bh5F>|}5dp(V55Vsj+>XUS}HLPLhJrt7}{V`%rswgv_r_EO2WDA{`o}cj{PDso1O_%Okb`_jJohY6@hnfFTkFoDL?n$ehb1nIz7|&_ha$M!{ zu;jPj_wkRYZ&rk$p4fWI|1iyb#v3&M<)xtT>z2;PA0$~4EHy(-vJlS zKQFh8{qjO-f4$fRwV2;$&zTWf9?u2sWyyEo%t<_M4N!eQ_?D(I+p#@+AF)glfd5?` zEpI(AFxYbixIVmR|6blb2Y{=+z)Ni}-=EhydoS-{`Q!6_d-jAm@7e$Fdklg1t-lw* zYwI`vdf%6|?>~1O8q8w(&o#>v;Ep{7s#qo9?a1Ss_q_J(;kvl>+I!3Nm!Es~XzaPA zb=};5?;^%8&2|8G{&BPpFA4kN#Lb&ePMky@dFtl+RJ294e9~UI%wuslRl!d9UZ9zc z+&$sc`sD@e((rRu`;$e-4!TMrZ(cicLE{DzGI|B&g$^fQgJHiml4m*v2!3cJK4TS0 zJ{D9<@H5h+eT3^!!uIUlxBu{o^BPaKyO=e%_s$Po`epk&{{7<{AD;Ti@7wipIv<{r z9z8xQxZCs2l{6gMabm!*8uI(jmH0iKJh9uTKkYH9yA8O%p5v!IlJ9KTZlgXqw6MI} zfakwO1GizmD`;&?*ng8v#nO*`C;D&Q7V>DniY4J-`-NS2b^iOu`oP@=JnSv>{UcOf z;W)>?XRz&h*#|K!I}68ojV!*s?1K#-J;Jw>pZ^Tv|8Du*f3CM(FXpiSJM6I-&D}-? zM2)=LfG1}Ezo`9H7XF`#n&iRju%neVHSH~0aV%_m0 zJKvX;xc`31Q11Y=&61chCnhFVUX%OT5)CO<&n`09Wol=hJc%Oh{jQj>iyY2_G}@ri zwv#h6@o%s;_lvAGk!|8xKThtX3&vgmc_7QLcqDhh8MfRHPhqr@A2niOrGah5> zwj5)lW1>p29^gJ>5P6wZ@`ohODcz3Ujg9S;Y$Uwg?|^&mdx# zjSxl3mv@hr3Rnj<|Kd7z$Te*&DVDvF9Bl$)G>*U&OXuh1xRd8;>}St>8|LNf zbHCVjEiZ{9O1uN$&09>U;PC=ZVa_vxA)9y+rm8c4_|%^zw&pz(V+_kM)u$Kukvy6M#_s_i!nqU=pY9K_i1|Rc}2Y` zEd@z@KU28^dA+@aW~puIce~Fu*4-rNmyi?#_wMB4s*KE0)B8m=Rk>B!fr05>c~CPm zvoz_v^75MeBHgS<`#oo%nkzG*EQgM_jNdOC>bfY$vF)=p?i~JXiHPX5Kd}QTx4tqi z-6xw`ZdaOxnU?A@9b2OwAUzHo?yitGAIAI?pu%~zQ7{R#e!GYS`(I~v{VF)MBt~XfHtn?-F%|Ky{}2w z_QUWFI`vHgpf%mW>*OvV6?H1)I@(e|J-$pltBV+Z{)BK#__FT~Hhj$*uxhBew41x=ztD9+ z%uax&tJl{rESIY_bjB>XeW@rfUy~Q4Y*=IINqc{$Ii+*0h@pt9Ctozqz?x?(ePNW* zaSM>2UO#F%bWHHLPaBoNa_BA2Xp=Z3)P!F`JiiQ9JrlH(@i=o5K-HI;y}5>eL!|#D zA{tA8jtWdb2C!JU7S%x~zf8&}vV(&$uf@rivC_jTm zXGUG&Y>SOl2=K%=)uk^=!JwXBX|3~IwWJuD-oeXYvS}=i;2*RGu-k_4p!u=PeScRGhwkZA;2`06V!z+}8VX_COWXuU5FHXp+(;Qo3>6j$JrnP5yoyNrukk(g$$e7#7SOVN{-f#MOd)u2N z)EgqhZY7kxWrd6qE`7=1lL#=3$Nh|9*9{ucF3ddLX{O91PG&~aDU%V*MiDu$O)`S% zrq=H^ULo(;#GD4@uRp3=YAV1D4xaOvkB={8{$Otu>u|~}f1iU46R;JSR;L#m=>)A7 zc3o^YvMaC5v9a6eL{an?J9wywfL3i2waNZz4!+gm106=@J7P)J5J0^^gIvMg5I`$L zO30&kM#aTkPY)t#D%b%MNYttJD%OdK##!+(tfJhW_>dB;@ulIrSg=4(28*|LfXFg_9Pg0z?gN zi4B?{a-b+owR2L41I!*2{YE@_X^eY?9Y?AUSJY%eaEx2=4b<8VZFGz#NTLi!Q(ucU z=^i4;gI0dKtj?^%Ykf^#GWmIZClJy=Y&nq;w=QlA;{TG=Glg4DWTv|N-PcRQRl*bK zU=Qn-14O*eS=E~mW%^jT+Ijd`(HfDKEWzC+@ki90KW!*v5GoY4L;clI#l!vA zROHT zZkkYcdR&t_yXvZcJe*tu;#xV)tuEof@Mj7%O35K86a=yC)y8kErk9{Grc{$p6`=oYp!H{T`~|=M zYUvYBzbLo6XH>d_t=yuIM*0ze2&&3yu(pWduTli=VzckPL%CIrMk*H^i{J|&I4=u~ z8OD&B_;kQa&~A0YNjJ#jB|2+7NGsbAy$Zg!lGZ@Zl zxh+O(WF6h|sD5Zm{j!VN6r2m$>r`TaHjm1Y*yog%BAw)ALpQdP*;K}% zq*EwglX`i36f3AO0(EIv9o$wKUV}>>TofnBAViwr%#c;hr>2ht;_n7N^8WFXc>9X1 zY;^Q$t3gA)*HleIR7ec^}D+-0HXjWJtx zi$-M|nT-L}U#3#O9m1Yl5ld^oPZ!lhs0C z-^|iQ{GIy8Q=4@?QyHo>Av^eFDTk3w^)C~P_hbau*F4r<7N$24V3I{Mukn2Yly-5& zE63Ae)FT{B`s#vE3{T&gl}0baXn&e5yVmA}eDK*788|baBk8K$nU$#oJ)vMc4iTYH z^yb1u$FwtZsL3l}6xWI`hEGyFrr^Ophr?y^jF#Tcd>+TB*&wdX3i*{rRmAq0rf(|& zlC%3-C#$X#bla#|oC`Yv?A9Y(?g8$g7zla&EbTX;8Lb7ynX+z0FGp9q%?YAl@KS?? z)_8+_>4D&mj(9~?Ki`fs?q@J`cX?hah4Ymc+Ax`@C($5IW&Uhe<6B|Ju8f8w0w~Wo zbo2NBd^*O@SFR3koNz=(cA;n}P~^rkdywFSU(?1)cy{E9pKLdCrQJOe%3Pas*V!CI zF?3rHFnm4-{_AZ>FN(~MIz63Mc<$L#0Ri!DW+aNCzert`mxxrTKeG`jVRL#~nQ)7T zG(D-n9PgW~jV;^n@{1$P9H5S<6K(tKQM1rDQfi1YTZ%&;v}+zWavNsv1g6%wMnVIk zSJIrZ#c}L+l^$m)7A<3vW@Oyvr?44Trik2J*^UO^+yxZ?BVJ9ao~|YgpT5%;E5Pjm zcy<|v9*uk<=?%VqUfvtWF&;xK7t=F6d*8p>5z&p0#6PI^OZXxVh82{B1@E9XTVG{e z+RC!6I4(fipcLBvJpHn=DAI+QKK>B3Bj&(onU~&3Z||jikUSj!;C-j1K&W1p-Y&XV zz#k%2CDZ>>s#lX|en|-Y>A@}g698gOo(IAKfIa25>EMcFPcD`3gdz4$8W zL7Mn1+b+YMxxS_8Qw{*{IgH+p&Wa-jj0fLEkv!EMccJC^pKQRO=nrS2x?(_bXPbJ8m#G0(h|#;x zHR!C~K~Lx5MGv+QA`A3F-uJJ(Z&!D0Z9Wju*J-6_i71va>xz*V;uh|(R8Jc`x7+yN zE&w6d)3@*W_)+c(LD?4kKAZdAX8Y=D@&~$}IR6d#;Rh~zc|rO1_7G?6wl7^>SPaBM z|7pTqIMX^8R%Lot{F98KMDa+5%V$gY!I!%wi_aU21GBKX|71D_Cjk(_t<%P4=b@XI#1p|9`TfBZ-# zUgFyu+v**ZzvyNb+30$w%S`|3`?_mUSGsOWC?M4<0;b*iV8e0`;w5V=_%no@qIh`bo;K2+g&f_Qf zo%~#LmOCRTht8zOEF~+mr~5rzNN!iI4pbO7K_A>6Y_`Z!e^Md-239-T{7t(HW$}H4 zW2=c`xheq-DT+OpzJBdm{7gYoqNng~fz^rgC-x0m3d}}-ds&Vt?{SW#JTyw4r%a_i z4=xzzO$F>MxTuLq~1o6jtW9{OSR7%0Y+9G;oPl3AkBrP zKOT;Gcm4GqF@^QP)+6z5O*fCo_>;A98r%o$N4>zdw>}Ht?o@h{r_jndgYYgn)S-%eVBY=Dj z)^6-Q+v;i`R6)h1|JT#KF{^Crt1i>+owPgKTh2cWSI9?>&wr3H2615bW@D-GR z(kRmH375#7HD~|4A&@uSBjL<=_GwS*oB|SQX^YIpya(OzHm}?<-2puJsh}~V@h;%E zOG0)_$c|68M_23x}X50D)jJ zwg^)We)H|lvy64_V`m2qM;fTQ(ep(PM*IqfyFeBFNm$kSU0QWsXSl%lO&@BgN!&RsHB8M}E ztOy`T_l}B@6|uj1{O8T62V(}YI^zL&GY&s)4|8j%@i2Cjiw2niduy^)y$O#qR21wm zGtA)jpcb`qIA!?hx46(}{ne~h5CD*sT6EOz;wEHW+0Q%5&M-SIRQUe7cb4}@uZMoR zU9YcO(1>fk%cJRXk$tNY{GOt2ZgbqiORW;dqgH({X9g1$+<*MGrWZ4wv7@QdAh9lv zy;D6{_rn?$Q@heY)^f$vp{R#IA35Xb-FcFSpEQES(PRz7j2-KA`_?+@6=sT2D-FH zw)@O8XDgl%e0e~%ezpLIJD#q)6Nf*%W1Z3`Rb|#owXLqPFY~QS9mDVO3}tYFYQ;i( z8hzYbv0U>+R+B`0;WsNm1KtCus;tIaxayqwLVAP8&fB zE)SPyTTEukLhkk}lgGV;GJ;mWAeG!}uejCycsUCt34pky5MpXJlZETSeECt47tS$j zB%&3Fjo?t9&kF+ZD^vlQORDXOL|XSs7-D_rMBfHY z9MOY1O1m$AEs%F#E(&+bNR%E&5ub`?Z`MgL&rc9gBw3F3SR<5u_>E(*_?ZTE@ORsD z{yO}rJB{a}Wxq69CDm*F=$3oYvRfWB13S}|3MO)+Hndv`&GL0h24#n8Fl!H^jPjt8 zYa1`q-;X_ZDoJ9uY=hQ??~ph67nJ#jYP154FAbAT4;iC;kD@nd?l89&aMD>))FXDc zD=W_`n&3CAB9WXfIaa7V5saHvx#1g*`Fv51%r>QLh^4gqlL^r8M-66NjJTpA`8J4DZx7Wn} z$PaNwYWI2y&Fhg7$CvHF4M52|veF6z*rEkbrQKQwbRslbr3VEpi4R}9jaSM}1mM$@ z3PN?oR0K!23Z)!ySUa5~OcV@zqF3NolBM>|_uy&UNfxf*I*3u_ionqLS&#J+XO37U zX$+cWS!4Sa!;mYdhu=HuY3n8qSs@-~C%Be$#i4>d?iQv-k%Davgm>Z-u=)=-IcELZ zs4@Hv%@61MZ*d6RqxrXR##o>7`1Jb<2hpmx;(=kTXRdl8e}h6w<9evID5i29h;4JL zWOaz&nim9TXbZE?h{X@%RoNZK*7;2et6rmt8L+$Ye0+R?YWKDewlt2hUXz!%@Lo%J zgn0W@dDt=#Vx60MmSgEoF)(!w4_fjsyErjx7FlSXg<=64J{W9iO#S&rcTMA^i9t6J zPQUe%_+-WgCm)!Zug1iLqbNn(Iw}F$R-CFwPj7styI~tk_mw)o9o;|Z?tdk8t$1uX zx;7Av4c8S*<1QbR9&C?y%&Xx-nZGJOm(+J?yLlM5?@Q>Gba!?x1dEv!%K@?3fUkDc zZtcQFx3{Lp#@e!~1D zknXkin0*Z}+1&ZzWsyO{H%Me(d3Ef35IjF&;V~XbRzF)&PcNehookqy4NtAZN_%{m_1AUz-WWN_#p1f-FhS=ra3@J6L57t*LFX+wUcQDK!$#5 z)~gM=jIXgHy&c}1uj#c~sKI!E;;qL_^GJoFeOB{zc4`Qf?$kJGjMpV7Wo31BGoV7x zSJk$pRL!fc@)cdvZuX`mVff7Ig?_WJ$k+^lzAx@KUdwtx;}K2EiF*yahL%-57}ajd zfdk!@&Sr11@n1NBMwtBDlkKtz<|8Pk#;tohl+h7e3_^GZR?Vx?e`}RmW8v#1B2~`$^w@qfN(Ll)z5Mi4tzUB~(q;7`cluB^C+Ha6?tz5pVP;&=a&$8H;-7cqt zJb+5bFzaf#?-(z@oGImy4^l&V)cN2;?5gB@MZZ`I%=tZ`?2u~T3gI_?Lhu(r z=Cze0V=@Fh9`!7P7Z4GvWwt=xXkY0L35t+M2Y7{Jq9=7xK6hFu2*ryNR0+EDL>#D2 z)nSa07_N33C{QFpFYjU}_Jo`aDeAI>yG$mYnYAUC#rO|Y)o9sx>Raj2m)xO_@$EoS zrRPux<8aBW%+1>WUO_~Qn)7=36F~i32#;Qxi{HzSYw&}Ap%$^T7qB=z@A&KN$=h^2kg=7?=v ziwa**3jnutCFROT$GXH$gwtM>t4)a49obD-K>Ld>fngsxyd1{e8B2PC-N~v#6JL4y zsL}#H3tIMleL&0iRu@>ljLV}(&*@3F6k@npzG002m&iWe6X_ZKUEOz_GRM1mI`~Ix zP*=e)8?$NNp`AM8LXIBMx49-`rS!vHJf?Q;8APxw7#Dk`U(-!i2#l;QOX}Grhx6eC ztGPiNwNh|Wq5XpIH`|0GUnK_B;LYw$l^r^A?|$f?Q7_90IW$-jz50^1cXSzME7X)D zwF3eI@O%(R!BDf_x$K^*$#Rh%TsY9La@bo~(rG5LHNxmfpV^R$6AH_O$HjfkkW11f0M4i3f`Tq z+RYX5r`MyrLwd6Lbd*ANf*`gx8pL&E&)8o;;KEt$CGU%1q;FZm%iXvuxz!4I(A#$B z`|K|tup(d6^^1iQ2gCW)n8o3x%Is{@k}lN}Kp$-299!Bh=pK3l;JESSg{M2;r+N9? z;5aW*l|ok*Dq~J{aiq!s zq?!6uh^+q2HmI!40q{=Z3&=)D(N?=nRZ`oBofAetS+5!*u+;Qh_Gu2_2OK)lyVOQC zkVk>}4~atT{+vJZ-b$_5YCiE56DGloI&)!(bOHn{GrtAV`cIe99T2_c1AYOz6 zudZy3AnDQ%TVUVcoXwXi)HJnr>$GDXWQp;R5NGiycB7j$Ut5Lr+T-++E_i>@EqgSh z`b@VQdp->nyrCNDbe|2>BqSXNw#*K-QU_>_wDDA%BmV&e5~raR!e?ll@T#~=4%Q^s;+>n zI^pCI@57bVc{C?;P0dBf8t2>NsUfbL8x{={Yng~!$qNbHmhg?j_#^PK(Nh9YX_8qn z&%$zZ$haF)y~ecL+OJ9{AJ-s)!LvTdVQN7~coJYTnv}2SaJ?rR)YjddqX|JlG{7-t zEKc_E@iAun9^f@$5exG9>+YbWCJRcSoDHt{wk<;Kg11=!5TkiiR>|5j@v5|v%kpx0 z@&~-VivDRc8IGeS7yzb=2N@}&1%EH=d4p@|HNWNbV4gbnWh35pE zQZ&Z_@*n;FAT7p~)X~M5b~1`->sAd8YN_jk5J4u(%^es{Ow~1G{3c7!=A}?L)w#n4 zTPH#HT-%)NO__O`ULHKY*^i!vTTu@t)&#qN{rU)`^34a&WfF6~l7CR`j;%pS&`+=u zg}kdW$SPT+2$1C$xNzt10y=A+_2zTqBQ#U(YBJ2H{ReG~6kWL!6&|v}bftt1YO74=IC7-6)#uVC zKvx~r=U+xUFJbrtP4lo5bzeNFIo(6AG1bbBq4N2;fx2DEg8|6=>fEDo?fJN$`d1t% zXNc@oa;G52)bF+Mn3*={b{S4O^O}`tn+|^9v^G;$6q2qXw^U#2ZjY1=)#X;dm;x$E zw;BBUYioKdZ+n2jM5|Z3PP#LvGSi?JOBd<$!DIUg@kg9D$*LLH%6Hy&9s22#;$2|U zr1P-=IZ-#flsE)JBKQr-v z?5+Ll{W;|0oQ;o^xgTCOwE>#ba)?J;u@vHi*1?oRLptNgK%^iIC)aUt7+?| zuY#JJXi(gsKFBAi<7yWPyY|JPybk$U5=~PNxKrP)I71%_QYdv)@vxb5kMz8>tcyx& z^BPJ2BI4va!?wtM)rd&>ywIC^`u)%`_Qhy%2A=$Cq2yK}Jz6=Auqjvp|1=z5&iL6U z(id#{@*@Mol#}SrymNmZzjFNAac{e6y1K1RO-=B8p@KJ#v$dEtA|fCo6OK-1Geh3OdyNKqD;FH3d(d?gS|F2&~w6ME|A`hzWcz9sDgbH zLoqnxF2-Oak~3w9%&9vvo$LwmqCNt5=b;H~WxqXJne640^NgdBht~O6S5^Mrh7dA; z;9kLt7fVY!pQygXFT42 z1WwpAI3z$B@@90yFt}fbaB<6-Mz-l??R?z^LzIEyS2V=A+e0gG`enyT_$1XEDUQrq zgWj1BG%2LLGsQ(Ns$|hw%l~s0z&A^g%-fi-1W!9j8M?4>42w@uBGJh5byHFj{M{Oj z)R1ZDzCOF8@Gt(8e8-WcL!&8-7pm+2P%j-d_eD#lbDG#c$Uq3IKB!88cVq(@C{(>X zuKrVJq4rsu$eiLG-{R^*UuJ*Fs19Uzya+##GtDMdyL|A{JD{$05j}rMAUZV08tUC5 z6)i%n&mYV;GDSP1=BEFqkCya&^9$POc04u}=LtX_O>&@h-r(2Atx22;Ao{tsxHk90GE_lxZbV$y_#AiH?dD|%psj_F4e-0PJ zWyVURqz45zU2-fTj0|ywU5Y_0y4NjZjH#ENsMD!^0f#SoEcMOR%Xv%L<04O%H6LXm z@s!Yhz6%*OF$r;@&}@2c@w>VXeqDPIxs?wyK-i)#I4}qywheuJ;=tA^%ek_uzZX^| ze>=d{C{AY<%#_XpZ&(fpUQZ7c<( z%TH(bf3rVa$z2$Os-2E`7`(6^|7G(nK9g_S!$8CfiGZOq5(Yyv$j)O zC8XSQadIu!qE88OGWxaWjvbmtPSL>)XTku%wR8^1Qfu}RNbJ5*h}bud)meyBtyRJ4@6J2U+Xg#X@ok12b3V2>&DzJXLDYB2*W?eSODj9=nplhPJ@$x<|MX&=DBnCe5 z+Z3FrIj0s(ugGsH1V_kz4;(YhMRBtr**n`+0T&=35GH2Pk>5-HH8-6$FKtdD;)dSL zBZ`9$LIv(FkX)R_?0%T)Hx_2Pf12y)3jPPtL!bK0Gor_s{cTv4o~F&QRa0$A_ftFV zL0+1xyBP>sng20tttrqd>c$tLFc&Z6@bj;o7*>?aOlDe@^rOjL+Hm&n(-Z_k-J7p{ zdM!fu+?!Ml?=>tx567nkn6|KVIUie&POVP7rWmlgaoH&CTTN9>>}z@DZtHsgUt53_ zqnL9I_9)u+Wl)npz3lZUbHf+P`5_P#FWr)_e-l+jp}+zsH%dfACKa4sIK`gU&8?DX zA{UzzRzZXqP2H~BaMLF5rI^d^BXPmE2JT>~nXt*Z0{LHEt)gfXGQFvXJ=<1JYd6&4 zCIdg8$ON{L_>DAPixI#kw)uQ*j2{^vy&IKt7%(PegGbC;Ri^4zEFI zz73aHt)=pbPMgiq9%pqcujkP=K2CJ;kKosR<``YmW9M#@hc@!o$SW<`RB~wSxSjjt zE!|v{wfd$HRhhOB)~ApXAup|;{JPC+Ev;AH$yI`}TsCH|6uWuwWmkh)=(;VWFU5co zeX=v=YPv-(p*h43fvqQ1VX??uh()wOCWAgW@UW;@e&qWYaze9ykX)3R5k)Ga7$h&QO zHbq?sHqB)8CQD_a=pG#kj6U~#`8`v=FOS6axjzbKzK#E4r$!+r=Ce&#JTl@LS8P^B zlrtt0M1=fXUt*Zg74tuzWqk4xdCF+_%ZZvC$Gwh!qBcL^l*h1p?Ah@U-QEyE2wtgF zSSu8*(G6`3BWB$fzrUTT_shtsQv&XE>bf{$aTf~NmvUm|Ch^c3Q#mncg9uXuT z`t|!vNm(xA>`3}ZX{_E;{M_0ZzyWL%e6|_xJojn5R>=_>ObU(XP;2rhez=&QyKsk* z_cY_+sLlP?_^2OAnHi&V^Nm4tV7hZ%doxkKPUf&q{B49^@YCzToHqgj6R39aHDfs1 z<9AKm(~}Esb*5?w^#BlHKzL;V12TbzYRYZ=@@loA=`};VOJbP@r+whK*nX(^=lq?x z>^bGZbKx6-pv!dHMjo*bmx+UW%%#dKTW8LYnh@S<2m|(<^)mQ*m{Cl69K#S!hdH}t zt7LwxXQu@sS8K2Elb0lb{T3D}b)Y#$<=-tJP{3I_8n3!sh{ z{*=k4(_A(B;85ESzKdo`If+*ilk~0#T%DQ(Hl}>-UN@mM3UUcoo)&3#b88*#*RHFx zwB9UurWq$S1%kKubJ0TsNtSbCI)vj5mA2D$5TEk{!E=*U2 zq(;hCJLwxH_e8U79{HBH*=Kohm5ro=<@fPPBGqEOwGO_%5ZJ8&d-h<>JdV=p-Tpo%5=*9;2~5ag$<=jHeD zS(i0ZxqGJO@~_7FPyJTMk#9JMNxgMN_T1kOIGMNoJ}JQ5$kJayGZxVYb8VwzTtnIi z^W|M*#Tf02dju?^UG*#eK{x*=PzN%JRPNc}y;Iao6c4a{z}S9uV~sEMi?j2ZTI$y{ z)8wyToH-BSYt+0Vs$SBR%F|ZHaYWiEs0_}eRvuhMVO~ks5E5M;M9~;S$d{x{ zi@5;OfH3OEcMJBsss;G!LS-V|fIU9|MgOg`daX{_8wG6fY4of!CHH8cxC2xa*9Sy0 zhlVI@>-4yGS>w#xxUc{lx=%NrEO$J?>m1s+OO$MJjs_J-%*5RZsKv_y^@=m2BEn*k z8y`}1h151agE=GuR=fnM|CsRo@Li>KVuu|W|02e`7Le9Tx4GE|Y()y*zb>0!wJc9C zgfx(S9(2)_OA(m01_quXQ<}@jR$kZPa}sxn*kDh}lN~_Y4x;o)!R!Uyh9zWLzLhvP z-lgtj0#7@zZ#?_W*|E`H+>egRveW-yZ1>Lt2O2i={{TIZh@{@=PU2b}p87H!wQOHRp{6*!)4=>I?Pzex9|)SUT$Ps0$Y=D~^> zKq!naJsujRm*YXf^{NvM;wfA!{ouBQfkn{D>X>i)^GJM5bZyYDW=vksZDj%Io(kWp z{Ja&f`SK5$#y^(C?6KB3`9gD!6!lO7J0Yr>aXey*deVa*4Ebv<%5!MYF zxF``mJQOPdqlbBP@PXhSEspJC%Mi}_p(h`Jz0LLf(_fSOZpYMARVT!scG0vaO&)T{ za;J~?r61o&$Tte7!&i%v)6b!my=G{%rdknSOm(%QK!k#qDfL)SB4#=6Nk=TWZ7&e>P$X<}s z0s?cIFh`9$?(_-?0lU?*GMNG#ATaLTnO+WAq6!NxV=|>})QPzSUCKSD%u_3}5~#~> zW2vWxQRxgwRO8ZKvjQhfoU66I?SJZ@5OE7rLX@~_ zf8>oRr(@9WRG&ro94k8Zb>@(-)kwDqEGS5Ow~Y*kqppTr=(I$a$mHeg_xd48^5CIk z^7)_}Q}=HchdqR+=H9lwGt*1^FZcn=C~E=i8Fn# z2fH(*103`(GaW85PtE`*_(agR5DZH8KFQNSM$H|ZC)>{ zF8AW4wN6mdgCe7h_zXn4|EE_9YI**Kv=%cX-2h_Z6313cHqdIS&x5Bl4pX9HL3Aqi zvg%fo%4ABLP9q^?t!>V$TScT}>xcsfWDpQOCW$(q`)4Knb0(6P|IpS(A6>S~OwS*M z2C0V}pV-P$r`pz7BLcj7Qx@9=8VZVw)`zk*tOKo-1HBEk#Z#Msqf8u(fU_f8q`;;+ zf%jxvmAKDv&lJCrP2rQ9$&XwxZq1I;0aj@?BzQ&=ys*+ z8ajD8)vu<_X4wij4yx~UWhnK%Q0na&{~sMICXD%G4MM?7^=mrOeO$4Tt3&QLQCXv= zYr^ix-;K6yL@f{sIl|fN71w8U?_5O+aC$*YqZ~A2{+94LzEoj^iW*^F4omx>>7jzU zMQ!0s)Tu-IbCP0g=M=jZll$u3^6#=+fsuC+X&$@D`isL`c(swM+qwt0HkPDiBiF*S zK*7*KjJDX&{kMfRg(RIPT4rbah_@r=(Tr&S2ec;34BtcbhT z9oC@*$5~W+D-8k>v%HxN^FyD<=CIM?JGrO3?UHg zOQkOjfU*|Kq2oF7M{r)C4ogtu!|RKg%1$~hP}*$zU1Zm2&~|wLpFuSwwtUEwx2t}f zel7Xy_|M}S1lfy{Z$up58J$URFgS*_-#*q!KRi|Sm6)2bb+Dj|7zj911t>==D=vI1 z3m`t=zq01Aup%s{Q+gJy1k;Z9%A+C4i07VmM3}ZzjNLWcxI4fJkyDN`4|j`st^j6L z^J1yM&iAdE+LM=6O^9WI0#O^^gcgLlUrWhO06pGT(kr>Z_yk zeR+m4c`<-1*n$b{t@S)e_t1-tq4(wIdmO98xu}BLpjCD}lRNIf+v`N-OKHWT`*^NKGsHLw2!GF+r|k(m3+s z2zS+BDO_-(&A?hnZ6VFTnUzji+HY@+AlA~#ym~HaV=O8E|Ze;mKu8k35vMf zm5@D$Pdo!rU@{vF@;J1c@Y_3GpnChiH!=VR1|C}=u;Ran_OkOUo*R{L4;E@MbI%m3 zH?&i1atOeko>KM`INZK8d>X9xt2p^>TyC_pneekata$VhDMT6$Nw$B8AA-2L9$vc5 zG%dHA^BdmSnI9VV#`|h)%I$EH;x-#h7?pS-rnr>HT=yXK=X9~#j~8FM3|LhA8QY@G zAG&kdIpjHKTNp!i^=Hb7nnyVPiRvNnS7j#FPyHlUs_=TcJpG4J_3!j`eg)o%v=Nyb zjeexn6T*fM*~*u!0>cAZp$~IF2*P-(L};yN>%?2%TzT^n3o5^y+Rk!*;!OMpF^dxT zlwYuz%|?IW#xkS3yZgG&)Lr&vMovp?^FoZvB$uKvSN%4XKH&!pk+^K7e^=G<4btZF{(D$ZwH#rF)uH z^X$W1XmGb*2}=HIFlhlDR!b)-;HhA~Wwc-6Y=Hyvtvll5ZnLsK@TkrK_Kw?E|o zbLxLwu<3t)kT6df@m(Aj;A)S0UNtCcakX3i*>Fz}$HZ(8y|}fk^m*a5qaHqNQqCR2 zGm)!jaiau5X!uQS(Yy^rH@h;qcV20iK~9{BdJ;kUIIEcW*TMgHFUZAT9#g;8eM?0i ziut`3I?#KfePDo#v`XQ&JF-+)ksnrjzlUFYED)lf`G44Z^Khu!_iy~Z-EFA2lN3en zlp-W$-y#)dsqEVn6@?`GI=StHB4nE(Ygs0Hwjo)PW$fEv%3v^<#u&ql8P7$Z&;5M9 z->1>{`5n)nzdszu9FCdy`&!QBb-vE?IxqQ3+ull&lejxKUgIq9_A&r?yQ&>P+y?2= zrY!f*d@nN+5B8IIZOJu}>Kp;jom0+r*o`&k-&o)yhUH_RxW@Nwwd`%ykD~`fopfT2 zR+^k#1TcN9)(!8tovCg!ekngMb_8mux5fg5oM)Gb86QamlrM|$E;9nIs?%nYh1F}U z7u(+~B#5`HU+x%st;Me(J$-U1wWWlG@Nvs z9}$>H6`hE^Sa#7bC*tt=GiE?(fH|7;BC8U@sx?iOz^Lx@vjmy^b)@z5||7w2v z)>y?$eq}?2P8xobo?3lSbXdUTkSJk;gwPcmDP$Vb=_Be}Z`(9{<=~e;G~w%x*i+;k zv^Frf5=fTHZt-b*v(s5!VZ&=SZm?luwnS2egmib@vmj}0HASr?iLDh1N)HV3l>J{Q z)@jPIA2Y3`afDo-A3Y{IV01`iAcR)c34yVwRA)=NtTQxAC|Dw$){pp@SA&3M*t z<}>jP=Z3TMZRFcG8R6t`WbLk>B{=?5$^*DhBhGhzj2a-` z7w+mFK0>XrLw)4rIW+B&D_P++Jl4i=k%jZo7uXl>QnE@OQJsx}C0bYV#G5M?(k|IZ zG?P0bofe}P(bW|S6SFZg?9QnI5{;VL#8uDRSa|11QAU*j@5mZ?1p8{HHM7B$lKO$S zJX&kQDdIshF>-G0f)dR8oTx63IY(;MA)Vfv;Oq$r8*5h^nNy(jcZwq4G2x5NHpRTn zJ?6V%3rA0S|ZP(Q(*dC%kZE>&D^31Bhv_!cj z+d=_ifz8QD5F>3(ebv=jS8k6vb|^B?18!~hu zw+4Ng>E&V1!f2R5M}}N482-Jn(lWtiqB3?M!SA8Aukk9x&+{ooTVp{=tmrSAt_so3 zxQ=e?1YJrMmT9z!VW1!n_vZ~rMBq>r(0Mal!^HL7R&{f1aSIC3Em}mMy5Wu@7RFml z@*zWQUQAs$b$yl5>{!E#(^^web}w&HZys!lS;Gt;#;LA-4W6jfGetCplOgoH)lBs6 zMaMn3wOC4v@IV!Oc^0Z9V5mg;)@tCUlJnab3K$Ds~8~C!5g`+s~;tfp3b>dHK zXmc`0xIbR2%E2Ns;H_Hp&Si(&f*hy3RLQv(b;geNkDpo^sG>$z8sCdgiVR=J6_V?O zp+>p$oud6J)O-1_7(r!3%^X6_0I$vp)K|{%SxeO{AF9skv4ininM$%rDG|eP$PWZ{ zI#;p71w=2;EGoq3oZ&r{(!UHdaoh_K@Ea?M*8`NJ-*@XYv8{3_ABJ<%@-Qj9g~)}k zahw~UR8mbJcH<(S?-C*^l?J4J&nP1+qq8zIZz6kbM+4$f8980^W;iLj)dy z=M3y4tCO@wwID|Dec}fjNZrvD8WtX%Aa2o)aRM&HXKC8yP^P_-}u<+C~^s3|@|5*~v)MOoW0qS>3T6B(sd`LckipHA1-l#)M9$BCo*idWZO|E?_G=S!hGHpz7l zo{cJ7l1jqD+cDI{Fd4TuAs2g_Tcj8?O4Aq0&7j{Et3(XnBN7E#$|}*LOG+t^^zxGU zGTM?fnv{D_HLge~bEd*EM>c9}{Z1S9Axw^nj*%kj@QXtP!eS`*q8WP}g(ChuZoWFY z2^xJB)u=^R@vtG%+fgT&&Rbph+=CnI1bVD6IkoP!^u=jWuj;3c0Dv$qUX;{5|F%+o zg9fE2-dg;N=_CHxt+&s@UF&w8JoDa~^05mM9BeG9;NJQCjbC|Llh(wWINtjwp3G|H z|E6*)*PgLPn@BlU;r$z-1|jM4EojVXKe2AjEya1Az~&t72wY_ebRpgN8@)beB7AX? zE^s?H6D8VT?Fd(pW3Z09E#DgIGkJX(SwXKBz+hI)aMnB5Fg{p&*~#72*7w(j2cr0O zLDACO!iAqAWI88d2;Aj+&KSi1A-hH7otJ>#};|qT=?fTE` zzzBLe8_kE*`>Q(!*Urs7Qd?(mK+ed|eSgWNv4B&6>|eObAMRWf%{v!^d6V~Wui+j2 zZ!cl%(ii)drM;{(k>icy{=PxWv4XlEjliyzP@3cn7cDtOzw4$%;y3Qx9wp@1oVI+2 zh@d>grf%w5L>wKuI{pDug0@n?zz7hZL(H$&P+bFRAW`R?{i{shxxayJy2^>pqDs8? znu<|&U$iVzzJJSk2$JOmw?A1T2@mj|4wOvSRfw)24^^zJ$j+|zmp4pel@>4hh~jj@ zpnpN>MXcZcB8aztWFYHgK?FAcoO`6!Am4rZB(~PYsU_KIG$MQ7ye?5&$D96$2hrHt zBO?$#xHepkP5G|k4o$hPj3L@zPYFj^XMUe|2WO@c=0ORi){H-&Q9=8e)t8cyR_Gm#Nijo3m9Ch zeT@T)?l`w-t_m?ABAo-}L-7fva8ENBlN{oKE3*T}o?~WX)XIhxbw_R^Bra~z5%QdS z7`@+;6_k&XF!9AvP8Sbc33x4JibIsJ>gYSi3)iY-FuMq9=5`Yvu8NF5KDT zx6yAjLBMB-V$DIp%^8@@U>MMs9F@U3HbOsXf{a|A%X(64L}>~qj;L)U@lMo*C}cO$ zLI3uJ>gmRnl@r~#$P-4Nh@KjBh|jG3%?*x2M+J;|3A)|QrA6iMkq^yX)^#C@JZ`OM zyq8=d&9jubwoLs6w8eb!mEW(l#obkgx?SlkSBnIp&hXuX+&bzf1Xpkz~DyyZIL@As2(#)`ShOAS93nL}xGt9)Day*5-0CkAnQq zAJQwbY?jd-WWJl~@>AE?`?#miZ%7Cv9TNAQeUze6 zTcFg;N~U|S*wBlO*XY;{SsOj|O^5U+ti@+>rQEJaCAPUGhFsG_d%$BS-tGxAYOZq? z?EY;{)3Dd8JR;`bEH4mB;j0i0XAni9LY(OM1D<1 z$-xEO7OhhFlpQID1dNS&q^G#?1PCA93ytw&0!=}xxRPyI!p@*Dqt}t?cm4&@&t3bS z8y~R4MHW;Y3hO}>L{jpV!<=S1ee7(KjB{n{$}$`kAB{A^M87zWmO$+ulYL^7sf#U* z%`kYSytkxbl}d$J=UYYsMpi(cy4Wx}*M{p`L10&ylWDIKRZ+vS=lKE`e<8fT`F}g0 zSEOuS+zGl7`m=GZj))fAy}Yq7Z2Xpi^qv@9fl?Wo+|su`F>mO35I75ol`iIk`Bwn0 z0Sr=v>;-^H&UyG_yxpB;TM2bG-*F;YQZ-Nq>zPi8gH~8s)PuMldN&WVCEe`PrV)%9 zNc@G6rLk-V7lsiuT}3qfL?iy3i=TYFtwa% zJv)U4MCtqa1ekv1$7$fbTB=sQdOrJqMTBedyf8*AE7<$|gFAly;oRxp!^oG2a@*V! zwDn>U3stMy9>RO#uPB_f9YO=Un{aoSt@pnUHRrm3_iELBYW`&)df>%Y`1Gso`G3EF zhs|dHxB!^GFS=hRD_6Q^#b^>C9>}s2jk8=>cs?zmFUUlWSLtI;rCv$qZpkJGQxE;e{NKM1BL2}cnI8Ezul{(#hBBdd5$yUg4% zyJ{0k0ucFKKn@~k1>$F`6^f6KVlSc%CdL~R8N&i)f!Za)O<0UK)S6;Zw%zKD>Z>+U zwZrEB$tN=T(0>-VuOxyWYjkbaNkehRF{J47)6V=^L6FfQl^iEcx1avjJ!1;psZ3VO zKj)Z#jC=oDy8w!<2q|7}hG?uDz}rPYIoN_0T8%8jTQ*dH@rkN_B4=3RXl%+iA~Ntt zz;+AyJKI5D75_Y-#rBcHjD; zHP=;}yzYV-hh3YZHX|`t;cNP!TaPdU`thTwu+u+H^m2 zTLERSzZxyPIf7Of;hwUMtlE)+R1pq;4#yO|zHSb>YYl=ur%6Q{%J_Ij1HBDfwqwue ze&8u9)Wwm-qpkM@u?|#Gkm7@^@MKgFHW<9 zAD(pLbNTxhYpi7e*7)uC5&GQiuV3Qw?$XxSdDK%ByuIKw^!CE*Z^jKUXx`JPU(s9d zIDz5fuBcq!wUz<5-)WY=bUo$bgoC}7sP5|F9m%+oi^aCc z1d575iqAF9_MVFDigP5~*U!gTLXCc9Zx3wR+X9TjL;> z4;y=qU<`FMR&jXbs>pgfP|rV7vSKwCHSmYg-_3BJB^S6zUES=$d*6XZNkIk)`g$10 zkYrhwK;N7;M8T$S5BiiLELIaKFS-8>n=3P$MFTr!Q{M^&A&mAYjjR`?rAThBIWk)I=alQ z&<~Z?G6ADIgE{6~Ev|Q)3u<8Zs1Q;7j|KY4hqOmK#7j$qi>A5fJdZwnwqE7gPP^Dc zKRk<1wD!$xvS6Bl|CdGlq6J=PGA5pS@+S}fXS8X&JFiRyAdxQ+GYUvws{PR2FO$>C z?(+mcUs&efGVhELUz6=@nXb+c%X`iP=v;t|%dYK}`qiuY=>i2cs3p2cqH!~6>clu! z5>EB}-c{7aU_@K)bLtP*5FGcWgukJ`s|Crr5$GS!f7O_tdQh)uTsS&%`yqayv)s

$gkd7C15N!4mh;+MmiQ3v>%<7o2FU;mFNKYJ!OK;Qq1u{P}NgnT%BWp$J!aVXLS zn_;J}01v3O&1iN7B0PBJ?DjtgQ5zqKr!lrx`+jGsFxkSs0skMcuc`#gplE*`<`mz7dXrh^r%WHb%i4tf61x6!clT8ZCQ^w=0`g6uWo|r1!~pM|(9A2E`(L#4$CT{x+URi;|iu@}_$`;JKxH{lXw7pe}lvJeP%&zV#PWt3d}xoozzh z#UaLr=!30hT(WN2N=H2~>m!^bEs(6+=jyA%?vf?e*~(Z;u*#yM#w`T)rw(;doYvMf zn3`7AtUZ(5I1-zqAMAW3PG)_Yo4c%#{olO+=CLg;HCm1|gYdS3DE0DL!>OaciYg5R z!MH^pJFo3~y74h1z1P+CqM*3h`MUL8=`Awc9*@24hHOy8a!%c{M!7yj8H9ZyTFJX` z=BRGr<2?&OQWFFX!2c2F2Rx9@2@{@vYafYY?%m8|@>f}^K8}tfWIQ{a?+R43i&l-% zMI^c31MR8|zQ=@<*^rgNSN@UupMMAd22I&R-RAHve|8@6;1X$Fdp2UE1x-!y>GEZu z1}(ynX;d=zNLxaZx033s;lcC@tNUJ-MSfGJKE-dRwDM(9gSby#6NNr+ajUaDrVYW~ z(@o(kQ_d6S3y^h9bLgU#BEiurK=yFL}Q0?e-r1*FYE8s?+1nr;vE8f}9wcZE<-{`xk>qxIn>j z%c`Rj*Ad8@ho9!(fq-(Cbg@?f^&!;;QP3F#mXGz2P6RZGu^w=ndhvM&4G~CpgM|XJ zpJuVAlO+TmdgHlHC^UAhh;;&Ze_b zYi(Vvjg9^V9mi{sZ#0*BQJkrS{_rY?4*+|m-Y_GiF7{->)agD^K>&Kr+^YSxq(*|g zyK-lf->PlOb+sn1KXv;rD3d_vAy?nj)>Nmv=EeNxIl#g9=4i87 z_~M#F)J7|9prJgbrdRLn@w~KtEJ&`O#73=woahNdd;AFkkUue5zY2PmfuT=eD82i_ zKQK5LkB%INR`v>fE`J-nE`Vn5Qw^5ynuD;SYBt zn=>@HDGGO0nOnneoG%P{_Z#4Tg+amr7jG$9QCz&M*!E+}xiky!{lTELNl8%L{2T}h zL!yE_sLPd{gM}vCZxC|_4&Od)vH|B$?ZVYVYlJ&LDi@whpmwYfNcXl05ccS&f@!|1 zp4xTe=eqyTH2wgiQRWn*MZK;E^l?iE&~IYNN%zMxkY8L?cR}LPFq5Pm_D# zOkT~-V7pFIgZ_N$lhnGJelqhjJE~tFY|;p)?kW6AWy>~4ilk?$tTwUZf^?>dR(5u_ zgM))1vR?X@5$06u?oY|=iu9L9WH7UbR#x*cNp9>{<+khTstVBUq72g#>VNy7ufzE`VUnPs9F7SKe$gwA2;eUv>>P1Ptx_zoUK? zfrcLf4DAfs?#a(a{~r+ToH1ZZKeEYhvw2%WBVq~=R^Y zCeEOp=s4);fe{&+7jA6d-2`rDJIC_8h3y3T05{%`^;bcTssJ!XwECji71mO7`+Ynr zUbD4d5cxAKJ!(*dXH1B1)RkP1bfpH?tC?8TeaHM(Q=}E&v z9G>*Mijxtt&wYKU1Q*dMhbZU?KE>!Dbw=6_X>IAuN&SWFixa9IyelfnU{O`wXPVnXzAYR0mwh# zAr@P5*;}_X&Qh2hy3H%^ZDPy$f^JFX`nDzMr+0$=$$G-GoNnX~s^s50v=sg!*qJJ& z)Y#k{%<22*=N%&oZ$9OM`(?x%hqPj_HeeTYQlb@zqgS@UZyzebY?Aqge|4_x zow4fh18&Uv)5FwmYhiU6iBj{u#E)fpZRu#LYy-;%NqC&i$lflRhtbiJ)X>l*N{rkM zr{BD8i;J7*Tg;DEpfZpAz$Ka-V19JfV)p;g>!0m|J_3w(u9k9Na_Q66s1QG$M=;Kb z=MQ7s-eBZC?O3cI6K}#X9%mVT>j@I6`*;|1ZATv0JAi>sr{n5+j!qZe{`F9=v2SXR^e?vLxSkv^z@j)y!^vx#wStDn`ojWu!&)fA<= zXp><_-p%ltX9wU%kxCDtb8jb2L2I-{^O#ya!3{aTb&vBoH(IjLKih^)E#Td8Pc-kF zINyoQEgO~gk3#v1SC&7#-9zk0Sn6itSjl7VnjT6n65)??y6oy^nzs`kcKXmqR*V+QJNMM)9 zcWRXHw-=wTMu&jY4w7lN?``-h&@Mze?r~a+2@=$SOP^`f+KBnz+5dA^EU6fprOz0x zSC_{i$L5$VxE9ItWU7(x_4sSLTyu3VnukklqKz@;Ox5{=E5L$g70y4pWnUB4BHS41 zom-A_SzKNbV7xz_4pXX?c{Ea*IekJvlUJGAJxM%sJEkR($US^Stx4l7wk=JAbzETZo$Y6hI@Q%)>nA~B#n>?Y;fnPT)# z!ctbEbP!>olq<)t08MpP)3c-CdbmoT!B8B*5I z=e3G2TAkx0MUiT|<5J^$CxopEXh!oZJmIr}beUOR5C& zFxQ;GhDXovz@Y^#xq(dIs^p?> zWe`7|>LRvFIWd-TFj1{HHazXIvnGn~?`dAkg`5%UgD$|lg9~(Zm|B<w^+ENgCDX4l|A16MO%6OFJ@_^djA(U2-os@6~=u^PW!`zdrSU&(5A3h zB|02qm>u0_xdv;VYD-9*FKi1 z(_L$kjzSB`(kB<-43}$3HrAiHW-YAC{o4NE2n?h?zmU%8ypXO8WvttmaWL_(r{(-R z=T)BOz;O@{xyBP$bpu~RmCC;{-pjhLJokx{4P-fhR}Y=l-&E}<$vea?pCi10$H=rM zP2Zv|qo#pu81IXGcCS0+_j&ghSpQhl9!aME=wdzVwwb1b! zqdV;mA38Y@@_YZ8O)GH6W2d|_`R!}Hje&}WpruLA!PgF-M{=^Y*;&c8d@`r-J&ja> zEEQK**Ke}S8_9s{UasX+uC^*yXs2B)Q0-9mrW|7}0(qX$ef!9Q;-=?hE^`m~r(+;( z3F3Qb;=m?0>OvWOg1{t-UxxXAGkBO%nM z%Y9(3?zLeqjR~Idf66WEHnGq}22~@R1dzL^+@E!pCRSt_U!k1?rm3d{d>+HCrwBN0 zMyu|jligDOg<;D+o|vAj)`3!y+JYB)kXkQS@|`1Rn8?lXb7#R0erlM@4B(b^N1jZX zNZz1+;%teGKsdl8eYI=_VT$aawsrWh5VaKj*Xn;=@EOxj%gD&Eg(np;gbw=K zjw*u#Zw-0l3dM8F36^t`{xDZ@;PZ)*7yN;p8-IPOPe2RXbK&D6vl>E?tH)`GZ<^=5 z`f~mDJzRd>twFP>9Wm_Vx5qv*^On`1e;D})dv`V}K|?eGof!+HICsB)rvrLobaqx0 zqab0vCtxv25tUG~Z*6IMnVVpu=IfTEw=sO7^E%{r0<*V*);3x|HPicC3I(}_syCj< z8rh`3q1wQGuG&T)Q#jRVw?Jflje^di(`yO!FF6Ty?)O8HRN*eK*{9u6WHL5VWjlsTp^y>M#u2HR(8EL zgIxu|0A#Es$6-taiIYkjn7JDQyUMP`eR#Sqe{;! z)hzmAkbE&S!%bDfeJI_O*Y=vCeVC#5g5bV@wXqB7<9&S>O6gCT<6%`9unHh>$`b4v zhh_D~WEM(yw!Q$T>T3BEP8k*s_BBGjNoGXgLO)ZuXoo&LoqE*0ztX*1YafgH*{-!Q z!l`4MwhOvJ`PM!biz9*#K)2`xN2Ju$=wG>Xh3tA8sdd-6=E{QVIMhd#a+uZHaG!kd zw=i~BQN_#2E>r@!H1guPqh|Uo->Gm7^{2>NDq)RYZ!v;&@N?CZdop`v0>3Qhz_jH3 zmNWFiVEgiNGQ9>HsV~<^>S}a*!E{C&hoT-UZ}ZP0_^o*ERpPTouf(zUk7gIarELv* zlzD}N6C`lXV&cmbF)Ts-Y821Svz9JbZ<=L5Sv;GdNXo|(j=@b?J5iEBGJi~>HCX;Q zSyE42hOj}_M5t1#_+r)GVQFFm!TMIa0@29vV$ z+R(8@%pH<-6`8#z+WScqa~vuVuZI?72=!-$(7A494nr%iRG*yDajXcj@$tcdZ2o2&Do!`{qhG zb453dr075Hik@=9#weyEMS!=8eyHX<*Y~!+^WpkW-vxF+l@-i9;j4Ccme6n|4LAww zmT^wFkB@Jtt81Cy;!veTqX6TcU8oVFJm`bfa090=SvM^!>*D-}jj!FDHXSL4CMI?| ztEHT;l)DVs;dS71%xtYdd*?b3|3@@{ig* zo7v-6&+8Y*0D?5g0c~0O?UodEGu zi;why1w3S1d02D+jaF!g!VivfTdDi{Vs?+1y0WA~*0l0$iv~EG`kxQzh0wlz`~1I+RvI5sGLEH+bl4p1c^EKN?YG`qp%XGC zux{%ds;j(mClveOMkO*jNa&?8#cWHhl-N_0l3lXnX2Cun>MZqBVaK}4c$FZFRiM=p z_qRAg25|dvqDCt|$LyHI2tdY8iXY!9mK&)W?EX_UN5Ue?S+23qg21+Bmni|MJmB^v zS|;ZT;eka(-@t&2!>NS@&swMMN0B02Gwa0Y*mF>?Bz;1S+d}XTujZG+J=HDtXgp<{ z@4#GaPaqYJr%7@I9K9G`op-ptq~9Uc{R_Kz1rn++SB7E_K0 zZgNX+5LUpqS#fM0aMKS`#SPZMT$IWBJ;X%R=nGZUZ&jyytHp8XP90dvz5dy1e($rM zU(-WkRIVlDEyO7p5F@N!{9IarHQ{_=Oo`BoVd3Mil32!vsEi-LMo5-dw8#yCi!Zqb zP>I~4{ey_w9HEw*bE&*WMrM6YK7~|p_%ZH8S5M5QydHa~R_lx(NIr$CKGlbnveSI} zmv60iyq-!ta?#g;7n5xG&inS|srB%~Kkr--eESY@qUqbR_sTpgaela@!lg1hJNwD# z8;yv^v#U>)Y{y3v)A7+rYwaJ9{7c;Gl?neSyX6^hkos#@IlC+w)HWW-#4^fb}%JRZzy*_Afo6CW}%HdKRx^$TFp0cza6!DE099KVG zW$0TWHy9!P=vAsJPKACO&^aZjLnqrx{9Uw@aBWHj7-w9nV7jt&s*10LX{w*i2orq* zMO}ZLvm3~NV!U#tZ=N&}ArU-;`9cP_P6yKm^TW8&CA}5-hFF{N6yr)*%cBb)j#g zc{b*m!EC@Wz=5LT^7}R?s=^*Jq zwc7&Q0-Z()@mL(KAMpgWw2Ky&hX8Da-u< zY13~TSysYspu{(ASo8Qmg|u{VAa#X|kF6H)KD+h=FfSXZc0ISjN+zfVn9?bcbMK_+ zuk3Ik8JI!I$L6Q2Ms@oqqQ=k~3OSKJ-q)|EpsS=ZYm3Q2SoYm6E{$%=IUY9YV*`go zBXAv&&Zk^d>ZJYs_e|EgXSCl`;VkbNc5jP+eFJ4l8SlCE>M}|`k@o!9EMNTf2lTo4K)O!pvQJ(U(=7f@($ zGgIv&gKN2iS*+x;hgQB#;oK(`pI9&h-ry-PfQ4trxRm}Ly$-DD$l@f8>-^jfYi;8Jw~9k!GG(7o!lpX1sse7#o>R0uq~;|3}Oz^x@UE2jB6 z;HRy_B}%6{ON6{uW|M9Zs!|;7^4_wNa%>Pjkfc+6DleK7rEG$&eVI_ITTFElAeIU$ z<-);R0(e#%c&yauqL511Ih4-G#k(*GO1Ux`=4+aiq~=Ps<4{Q%rOjltI)as$o}T{D zEyFAh2BZ9sLtT6Iz7|x(OkxFe<%GY@<*vqyI&fYpV_mxD>&$=!k6paWXG@?r0@Seyw@mGH&%QSR*m|i z!7a)6t|wi_#52xr(*5zxv%NVX&a`dLS045RKI)~6JHXT;m%d(A3GbhMc&oM~MpW^v^ohV2`!>@<|KT#R7^d6Te+QyT^!4?vw0J7Wvb|lZ4ke zVId3EvDF0^G27N&IkIP+j%f*`l2Dizmr-7ri;4f1*>Rn`?uvYiEj?>J6+mC5 zKzQW9tMY=ex{c2MV+Y=;`V%Af-PI3?=I3ytjDHeDQpTbKhb|~T=GX_xI`O>>(KNS= z@{<o{{!p3i2%SBk$R=v;pS=hU&C?ZC8mS?)}J~=jf zFi)M>ijp~qd%B-Rh70785?_GY3jKuoAa>1|j${z!IHKK@Ucv+?!7}!e37i?{Fh-w?L0Il#+k8v?3U3D_9!Ah4eia*VRKr^e<(Q6 z_i6>Pv>D&1@JOWaOysC+L~^21lz85Rlk1;&*I9akesX0Oe}=p+QXvimL_<{P4JUVhInG>h8r zb*A%AI0b?9Ge@4@pHQE5eMtbdZ-$-kJ)ME!jhQPbNH4MA z!dfJhL|UNNo;u=77HA0)9IWJeyw#K(Ddh#WzVbUi#&1Q3l_vn5 z*@#I`*Ih2hcr$2#D$RXqa(HWV{)%e`nkzAa;^Iy-w6 zKDtTVF(fQpu6BQ*9W9Mi;r3XpG)`r-OWn1Aoh9&=?6W{yBsi8YJoWVjLG!?cmq>Xq z0x#a28;J)BrPppNrgsc&~?*BumO3_ z`<|V!^nz4*K3g~WXqD&W43QIuN<1e%eK)Zcam@s2WCxdKqO3>8XsuEB=JSd$=Mke zD{1G}tu=DBg2bGRb}o)o^L+i8#WwX*Y5E{_c6~}`X7}$bO6)$1&OoRnHqHyRl3+1& zR}YEbD9;L8j@V567Mk=rL$)cu=gi%t<=@pTWU0)MIr8_X`Yd`&1g*}yL(ThUO{B7e z-7LS24;e#BYxViW#0t{wXdL|eoCew-3W{5v6K*^co2;HsMQ{mE;KyZvq*3Nsk9u-@ zQC>UDBMWnWpqiC&Pn&DG(PBbm{WZ+*>Q6bhs4-u+u@JnU9t7Koy)zw!H+E5k71jG z{)&OWYdTh45#gr8x3&@=A944#%{8tEc=i^a{hx7r2{SXHH126mN4x4nAckwMY-X=! z-|+=MHsD!G@L>b=JX57mvOgty7-moNc4R&dxmtXrxXz@;1)$ zxSr6H)z8GP!na{%bD;5r2WUKTSknKPg4KDY>+j&oW7ZRj%5LZ1XL?br#V=lH45S8d zoN`xTcr5XOKx1jL^@Q{do^3GKX((9BrMi$Ss%bSlkgH7~L)Vj_c3uhZquW0-#x*%c znO*qcLm$522^>GcbG~TOA4Hzf*=_qC#*A&F9M9@ifp+W`c>o3KC;oi}O$3ze`s~9j zuR$SCx8lNu!nRrmyo0aXeCslK(L{~j_`{Q^QKjU4_th|qs5c`=a7Y-S5(BxGKaW3g z_UEy@7IQWcB=nV1uh-i3D4KODj}1;O_&-*tvAqd&<_$zdsBfdAmJ&?bcJKdp!816B z2J@3gF4Da*?C7O?o?CJ%Zf66B4FE(%_!5_{R$C4dc|}$livHU+O;Hy*UZi~ag^qgK z+|HHxuq_I1vK1djY|qLO@^r*|Yr$|W?n=|-i*{;iAVkuGo-fdcyuWj)e0bZXeZhV$ zpF{2~*MZ#g{s_N|AWqv@JNI3Yb=>Ih8id2JW)&<0|CF{_)qr$Z7`aCTNgn|KRHSS= z^Upu|w9VPU!w-QpygcNt#1eP>$)b6Rj8d|SFKy-An?ly_K62<6fOG=Da%J|R8GeT#C&SwcYpOerCmlvw6dlX4G z08{0R3hLZ@e@Rir2qdDyDxEf zu>G+cs9c^7l+^L$@}Q8%;AIc!AZGrx`ajVfN{OGQf&C1tD=#&sp!Qir=?4A`yUL)ab z^0vb;dtSW)E1)^HKUMHt0}G-iM#$v6lJ^)b>-x6|&-03QlWk+z0{gb;O2gZ+_8??N z_rkLPDmLns1t8PDWLqtc0c@8(15RzNhN}7MRNSj{RwBQob z7lY=OW%&JMD=9Cp#$Z5g3YV}>yYzNM2j}TcDULxZ$zQ#*XJ-`XP&KQo6ekq2-$KO+ zb`Zb|faWX`ywL-_RQs{nhrJ*zs%n~w+^nOiUd|~j6la^$DwQm@jo@;g-mErzT4P7O zktX9`U=}NMJsY}?)qMJ`fm~a4>SM{&e6O#?I0&a9_!5We-W}9(P@O;w6OM7uM0f+{ zXJ}ig(El4q#mP|(>*L>tQO$$3Ak(gK$g{Y{yWMRL`EN@pwwNk}?!D&~uLlQ3B$(J{ zg~bmiFhW0VSnzejhM-)(a2g-zkJ7H5-FZv%nwlnH(^vy_^o;&&+%@DTmUp@^H_yz~ zd+kdVZJ`2FRji?UNsrT5_0rAE02e{l4!ymsX98c0UwcaGsByM@+Sw|Hi%*$^c`t#x z%-3;Zq|SE7#|@-a6g2opY+%a$%Kf3Yvpw^2)-nK)cmKh!j{I+B7?+PX|M$dqpM3xEicvB;1YT%D9Gzb4wm0I7kvnZ)Fqn>>}1G_m9_(VX%Htd{;9;zB%1C+6fV ztp^8EoCY?q#P%E5-O(z_`@@(-+FKFX3NKjm+G)e;F8-5GdbQWEvA0yrbzO_r` z)ic2dJihwZ$SYmXP4Eqm#X2b@AE=11E@Rm1WEH8o?-|C9m%v$p?MpeU^IA{QKKbVn z=vKT~Zz*=X2x)}zkgU7LEMtVFy3+6aAXF*G^y(BqZdXI!!1%PH3?YSwiNg`ihWw zqNhq??k#-(2CUNh+niIqy7Ff#LE$3@zoP8^i0yW7&nqC(3H}$noYCbDk$&Nwh`&|a z3Mx*CfmmrOJK2p_OgGeq>kBQMYclnSsPYirx>?A%M~M1Hs`J`|Y+{7e10i4PBy{I$ ztw?`Eoo z3Z3z8P%YTuSx_|3In_!U66Y6Hj$7NUs|k}``EUzNR5_EO_RzHO+=xfE;aEX#D5)>scIjhf-ckR744SS+?Uc%R% zdUIBxPbL0TAK@$ck7g?l5PoTw0>Xba2OR*wT>mb;!Nsn~@BqeBY0;N7 zohp9e)JObFMMS9~@%MyizJzw|yxxRNU^sg{9#ea(-3dP#HJY|Z{@=X-{tM{MEdky! z^^(}&WtKtnSm8b>F(RDDAY}(4P9JeGUd$Y+lLGPS(%5|zaPEAdSeiXbsoZ0TX`lQp z#Jy92@>2yaUkL4P}f9cZvO2rR1yG2PYbU*F~|0f3V z{j=YjZrd}@$N)3c^&5Epz9(EM4UZ2R*T{m>%xG{I-BkGQaLpzAbN`w`)N}6aT5d}| z6=nV0I%#jW^Cz_P;R&bu%uG$kc`LSU+N5f|i9QDCPo&8Q06DLi?$>W%U^Lo4=~@1S z$lHXD+h6rPEda!v&LAuGO|nHlDSO+dNB+BjlXZO0H{s%T=?;ECsC9fZ51_Jitv@K7 z>nreuNB_eYYI=MNJ)x#n3k6mz(J)uQYbO09mU+X~O$PHqIZzpEjDXGHpZ+=UPEa_0 z%a=SC?yY}4aBvPN%`qXqXqCAA)b{0BP;3Cmz&b#N>B1B%doK8H>#{C<01pnp9M3UQ z|Juw)3ZxcP7^do{tVW@yB+->M5+JJAqBmKupM*O#;wqcUl_YB+Ue2~O14e^EQhi&Ip$v$HvWf& z?eS00ehn-Xg6F^-0VMF4S5|CoU=CPk=kY<9uK^(Bz$#=?!?_I*C;x3$v;L@i`u!uo zbXfq?QD-tlwR^XpIoGBDcX!Ma6P~t-d8*N;lkF_EoDS3&-l`oic6+wuItxOLCGnNqJVsN6}T8GUJ~?jXic8?pr?n*64bm(Ok6VQdE?I z!uQ@50eHJ6*DMw!Llt^uohBL|-6n6Z&zV4LmF2 z-GI^zpAx);!cwic-_=Y-3)A_{H`_GoTs#L9u$^_cnGUuisVZ%^fsiKroMV*wq)zcQ zs2Ix08EF^Lc6wEn8vseoYLg)9DsYPXffU`TQiv*@dllf~0IQ=@%yQt*uV|T$_Z>a1 zM~(w}9s$ztAs0#nx}G4)u$mUA@Uqf*qcadhx8h_Q#1MxX2FjGXBOcDVTpmz}>Rc{R z>#U35%wLCL6yw#E4|VhdHJcu#etS$W@Vf=Rrg3hvnnRXl0tu;??w>7XsgSeF>@{P? zYMpEEql)hF)eT>?7((W?4jH~lO)||iPB@v-ca2~ zfy^m0C_3k#!!x(-{qTNv=?eXudvWoDRp%QHDn~F!BGb=DzB^+D7yQ_FtBJ@N6Dc9j zIEU1-KXX~Oo1x*EHEf{)1>bA?P>p!SBpHfiNMCmhV?BCVGlX;h9sPj zxM-ec7f7PlQ>Zwums|RaUfX>PDwT5I|4a$c$b+%@=JnP}_7~rnAmdFlTQ0^-AdN7~ zi*yTTaa%#8+$uW$2asK1@xJ_5A5u9kZGnS6Fe%3>7EUh^cP0~S($9?(Wc?%P*L+24?AVIS}7e*ISs0ux`E*ZNz$sMXskVw@cS8zzXLBtj7*jy z`38zP@mH~7jv%eYZbmFlmZrt7$aiMdPAM_skST|HZOQ%Y|0+TSpa_eE`OkHP^+sP^ zn~(?Ln_aq+mz>MjDR}n24g_DLH?udt1(fw{BJzrS#ru7^A~AM;vR%j^Wz4Drh0J>~ z=ylWxT~D8jm%mJkD=IL~;8F6?rX!58Ohbdw=J+Q&o@>I?G!3&M-mSmJLMH`a$v``v zO$K~Q+uWORtJu|GIcPRonBIqKh!$?5b-aQxzGSuI2<18zvCx`=<*oiOQ+PKpVUUi|J##Fm@8@zM?R`p#mIEHsK;>d}A)ezNL!@$Hw%Fk!UU;~dLk z1bZDQ)3aTkErDU_%(CI=SF%CGKwUVNFjm>HtvzaDeBsdb;^UYB&PW`z;gO=RBaw2* zKj&2-aGlSN&}!I4MwHHmn2!g^-N1qvgZNqIOw_HjL)}$?u` zHaTv`H}=gl=aNwKlM9UCl8-cm{{4D8|BQT>A$@M_q7&ngwfaJ&n3Ml>Fdo<>;leI{^EFwk6d6+wg31d9Ipqi^x3Eg78aCNwUYy%te)is%cR(-tyFV#@95B1$0^>Yo)%t3gb1QNXxIZqYK0n+B1+~WWZKcTNX@9|<;?~pUtE4K| z=~q~GX>+Id2}4}J#^>wxBZ!GRq%}&tQ=hvx@0lzaxZHk^`7-0F-pv7LNp@mF_R^F zEqluYH@^97-Mc%&aTrBDH(;#aXgfV!7+yMB))X`vdV<#ulio2K18L%KX!;bS+cALq z`XgrkGv$sx9M2Y)kyiKd1WhdjbOe9#aP1J5>QRBpV-{P}>nI$8(p}FsDJoq6E*m}bY%fX|28W#2# zT_oJ0Y-^}G-5@CIZZ-0mvU={%8D`&MHw?=!S^e`KUCAepIbR&230SS@9l*q}TUA}B zhLkF;P;cXpwAj~v*HXFG8Y*t$j2PGMW&HaVLPO;h5ko?mJ?f<0()ciu#7=pISl+j^ z7(H1s7>qIVooySDkv_w`YaaHN(^Ip}_G4nbq_T32RnAo$XPbGR%n!vby+1WOH@xrG z87Ot~3TyVF))~rjTn7_z?PH0HP;o^WKZ0Ox6`e_r>UgPzYie7j%= zz@$NcR-YED>=onvtz>m=T;WOX5@hAYN6vS^pxEoLlZma{KB)ZhQ5Zye_aql!V)m~X zm*FZwE2u^GS1EQy6$8)6Z`E?^vypj*$uW;5`tgR4n&JCIVWq>j89!6wrmd*eM9h@AryfEDwXdookGpn@&a^{r| zC!rAagS7?Wq~r^C@(Lt-2Sn?5p+%pXOZFeg>r3-L@dp+rJ4q?*>gPg4Tq9z*p6qun zc)(Y}x#cby2GkKX@>tuc~L&dGbsQ#0> zX||=^vr~Fld9PTS_df=30B;I!&9*E${N;cW+Hs3LBK0!P=N-g`6$PA&!{ld} zrLb2F`I0GMSjtvAGeHEz=p?XYq|_!HqF@RMafjPi20JNcIxUI$Wsu9 z0su;Vqy*yq$C;skTk}Vu!(!k323h|5_;X`gyKe^6E(8%*u8q6*cl|_xHEYP!EHPJB z;y#Wd`3(24rOD_Ot1b`p=Md`0QAW_{@1j`EK4{J9nLP`A*?-q)!-k>LsY*r_^&Z z53vNO_f83Q)f%(3V0t86h_2Rx7xmlb@#)11YiM{6b{pOAOLOYpODB)IYAnsfq|t~N zG$OozJJGmuJ28$EDPcu0&_lJqR?*>nhO8W^K@w;5!RB=@?2!Beo|d7jAM zlw3+i@>(z=+&%T!Z604KE8BkNTFi5K(=8aq1vq!WK!!IQwME+!Zx;|gt09t<#Mktg zt}qhf&TbP<8(I^4d2=LhrgNpCk)<_rZXcZaimIJt_R09_z+3GxxlpJonRSEE>;xU7 zEfn&J);BAwn+7v-7cSK&2jK!y^^KR)XQ9s}fM1TNdmq%taoMM#dZ;}$L0K-rcX^@# z3jF@?T?hO6o7S-)619O*O)8nCJc=PG63pVRHSQVvGd`wl>Xe#i0z+mk4ypk3&VMwa z;c?+*uFyow-3ngrdrspUr~X<>T4?VR6Zh|h3G zYm`_HhR5RX>eG2J%(y_9Lh_FnmWUs-=7T2b5`ARC+-p%Vjf#J$W)63&k+junj6i!T zYOJ{ODDj?1;=aE8Xgr!2m|8ESI2QX2!`niX5LSLUlRKG(tB^0mmGQ=8q`f+1WK!;+ zt=W(R$^_mbhL6~n%%@a~^5?KxQj|~Cux$Q3Oy$8MIhfin6U=|l-*po`wt^I27SXz) zJx-i!BkoWl&?va=G{COUQAl1BaVwRe|HCuI7EH^{jDU&ps4i#R*u3w;Q@0$;Sj20j zwg2x`OVt%M@IYQgb&YE9$0olAi}EFddnoRh@j)5hR5Pe*XSwkFzShIhMve^Ro0TXc z!-F(iFD+fiI=Z5zV6_u)`1Mn%h3CDYIDavBqqNoO!C z!?r$Z$1IkxGz+?GT0khQ?NoTF4DK8Mqax2}ob}rpsij-5iifj94vN5zT#OJKlghDF z#9hVGYJ9RFWlFlh3Gzpp6PP)#x;qh!wuofIKOE+r=~uM6p2S$32}WOdqX-bTQ|)%k zsP@$O3$#zX(RF;~gjL7NTg5^eb$WtXYfz3i2Vj#&Z~z5gO7zb@2rd3Eyvs{-YFB@6 zQL6@(#CVTm8H)rOuU;EwnoTr)GZo}X5=!F(gGlZ+751dWt^cQGA}`kXfvWG+8`1j5 zBxuc&PczJD=X3s88zXFAx0L2PZ_UL7h;c4+c45z3iwABAPZ+fzIrYNRL zaXRSqLTzdLePQ1(sKdC^NTSV^9%|57lGx1^ZmMOeq9Xn&e8bR_`mxa2;$r+S=;i#d z(8c`rIVj{}(jRo+_Y158>8k}QZdB#PHLnGh!U+gy)o{diz*{Kv<%jI(;B(xD|N;MF^wMS)A zt}g{?-7@7h$EubFc=3c;@)jiza)(0wzF-IZ{2l9|Ht3auV-FT-w-|4*wD7e&m9st} zPN7iw`u+54xrnT*ib!Z~_m!6*z%<>_^+-W~*~caaM2@2t#i1>7^!6uL0{;uX!)0=e z;KP4t?C*XO)Fddy5IE`uAn@NkBwT_X64E28d8rp&4*1&wYn>}n5B$X#)T`~Dv|U9+ z!S^e@81-`JiuvWmCQK12cyyc1v@0BQqK3?`#8)o>p|+L2WHefm{G{cP5Nl!(<`N~T z{7X&hAiGgC)&CgoRnRhxmc#x)W=i_;#naRoEkJ#z$^nM{mPt?6xtB(B2?3tRo8#XP6<&xz! zl)e#j@v?=GKn>hoIQ^Ba4(ChEy#Ah%;?+GH%DN z28NwqF7k2a**)m7zVv2Fx)shJ)27x98$S-GDlJZ1Eh{WMdxp0An$<{oD$2l&BV@Fe zvsTKfH_*SFbHQ0w9(H?H!^?@~4-^)Uj)}{WF`l|S))%jNqSzvQF7KYsWBx|BIuG5a z*;#AGdb}-v=JDws*tyTlIo^^HbB;^Fr+~IAezQ?UmI1fgJSQ@w%rm}Q(#8H~tS2)4 zkN3MV3X0sxc1DTkQsa0R^#IC}=>b|r)9LoBh5c)>A@~(42RMPb1E3v;VKHXfOE2w0 z!K8lewpo|U9xyiqt)(t2vX+NFPiO4Zc>v1h3ZThPCA}u!HZXu?XY+B6<ON0a z;?(3-?JOCPqI|?Q?VKI+gG*Ar>Pqju$*y>=;x-_;OkAo5z3FeHymYbPsLwDVAd zP$g>T8ky(R0e%E)UUAxc%XTWw!}aMEMjmkC40Z_ z1AiWhyzAwf^QT_SZyXUD&-c{K?Qn^moGm}pm3~hArax|ZRAYms6D$5SR_Xa3V8j&V zn{#{3P^q!$=ViSdzFe>dmE{Quhuewqyh77}Y)_Q)e!*>lt4K#c5UaL=GzY~*FQ87~ zw#y@5YB)*v_u@;fB(s^0qWRUc% zRL@A}(sD#>n)6o*a5=peI_8dw_3x>R)2btau&xbBjiHf4S?P4{3gg-6YzbWgFVI=} zYCOGjHfZdN@GO_1mUW27g$Kh5-MQC!Fn;JGg&lN%eCy0Ll@O)$#cj?_0qQtB$$elP zGkq8-F9tIABntlQ1^Z=bW{UE zp~&Jw&3K$*c9Tk@-%k3agY)y)dqII?{I?0pr*QFDu2QZdUM{$`YD1cH`|HoNI<^wQ zTos4ZItleo4y5GEr1-WiMEiG&xa|a^W-*6rSjjQ`VAe!CtZ?nB%4G2)I7_VDRT`G6 zsjQ#vH_|+KN2hHr5_6=t;PP&KR>OQL()TWf9=*xb>3IA}*+U&AAaNMs z0pjNeS8?`B$5IOJ*t9M7wj9oHRys^5h!2YBukG)wd&jJ2zMqXkk3xK|UU(w{in^c~ zVv|=geX}WL2c5fZn|s|;#h#Td%<+ZuiG>AjiQ^5?(==}`wTkLa!!&A~b6Ysn zVZY>S<_Qo^BlW|UX-K)fw4YTiNLbsN|mI)THMKX{1LJR)+)Uv5j>G9BT5= z#8eW5@<_;f=PVVX4y~d0&y#D$MisZxcYXB2#7>6TmIO5=KH=YsM?w65k)(1|hKUe| z6i>8cJ^48&H~~u}-1HgePm0e$&r14U;JpgZ9BmTeionk())0c`iDkhaE$L}e#4a3p zpNdYB8aG@N6udnV#g$JNi;{!uwgF!(mACStbpOqvZrS}rq_qv43B0KJwEEEpv!u#- zzdFcu|FPvqit;#w{CUIs|?1|7%3Ok_{LFx`1-BA~$d`F>$G12HT{%S?TWcqek;bROmHf;(M&nK;MB z|IYke4T~l&mc%^(cYlscQbU9lz*|(s*}+|ITj%#@Wymg+usX94)_N9NL5LRH zIUZGCSd&wTr!2jr1@B%);PVgH09o#rj$wO=Kk&^SkAxv$L{%OWzU1Arg5mSHqL@1$ zTXa@PJ^>l?Z-VAOFDU**iVXfXyv?Iql+R`m_EO;7{!W=-W(!ZYrmJfOZ!e89bHptC zUZAsZ?fQ#!qPzSE(eW0TqadQVJiMj)lZboy!5W&|XNEJb8pjN_orkj^6(@*_mD@Kt zBnNB96WsEJhXM`Jsrrr-U0d$P3z7U7(u2(?&jT2{|!F2GMCLx%yAZ+O3OxitdG#QvTrR35}GH#?5 z`*8Q5YiwH$yE{Bq)o-bvl5>e_e#=Akw+i2Q;OV_N%$SUiyI}rWkudsVe+(VNM8IS_84lmC z?(OK{s~oq+GynE))Q`4C6Li!XSzX)$yNS8H(PWm^0SeVvQ~ahg5x>!wTwRet4MAj^ z?LVVJEK>#p*XI1rw`jSZckHGm_s;rvQY`k>O2{D&yBj@!wqRZOX7Erz(QtqV-@*U^ zdnmkEvO_v)L5;VxGFD2}p3=OfzlAlO&~BRhSC1V|tug@_0Rf?bsIh!k3m$4e&h}0k1z*WkhY@6~ z-BVY{QB3)(xqiM<^J@w1kV?djp_eOqQ%Wr_ZdPt03R$$V-m&y!CX)j;qc5f~qV5N3 zjL<>ZyTvpCwo*fQ7Q`}Ke%-K0a(Y?GXn;bwB<{58KPO%P@8bqpgp*(zNuzGc_RQIj z&oR3wg$L)hPsSL<|I{~YRnpn)!+V6gpSA_k+#N7E!A=X}1vrGNI@=SBvXk&*A@w@^ z3X-oUfa7a+z{G-T@B1GyH0Jd4THvz%yl*04tYm!|h=1vGe+%nLf6fi2l|+94tzp}o zhd*fsT5egp4Z3yqTkytf$H);~4-Ds5}Y zDu2%CQ93%9sNsZmB>RwUWeD8Id$b2SBba#?7K?5fyyv}aIkev-BVr-`1n%@=*MXhy z^Xa)2(M{dR)WYDIdpkYP8?7T=HFd!lxmXl)bWG7GP?KfZ83AIE)|Lq=uJv}PSlNmq zPLYKBR2LPt4hj|cu1W!st$4j$!xEK!gmF6~P`PR@g5{rt#f&H2uIR|%FS78^>#cq8 z{UDls(%*2oz}NNU3GV)<+*fqbcKUgv*Wx^}@c@+#4X8Asl(!J`em>4_)J=WY-FB^eKWKe*@^;6k z2x@=(WhQ+B@yx$v_jFc!P2f-6ptj}KZx5HU-BEqbyQi*ap!l&_(I3Dqx%YlEx-ZVv z!Q;YmL4{U-Aj_z@lFzMh4xp_8YyTw?U%dUnS1q5^p-#87G%zqY={?>$QvfhAp#r~P z)s@l0o4~}#rUV(j_nvTenhUaef6_2p5#%5^x0kxn%k2!G*YMSXLqb);swF7kPGAQsNF3>U{t-iz&Gc#;!~ zXdekQ6_?}JZ6`nM(c#Q726t0lgly8UlgR(t*O&NMGJ*T}qTILo`&S06{b49mW=ha| zh7YI@^hvzivYP?g@(6qUmX>_2#-VdY$Gmu~lQA(s@vC&ernHcLfIxmF78$47eciC- zka2Tj?3bmsxM&&R;!&3bgn7Q_#x-yA21G0S7(cixf4JCHn%M-EW-Qy=W005 z?j9XQy}fP*b)@i=+c^->mj}uWe6(u>nrQG^yi$@^vN(Gvn;UF-vs!@-+%Su!w=?de(1551MYcy2FracRMT@T=5=3`_yuw~CB-TkI#hR? zDg34PeBmPW&43C!&%SUft!so4gvX$zB1=X8; zC(o1~=y#5~+@CKvfPeNy{WkCNo0_KY_!DIGDa)y4ri6~9hX0M)NAUR$st&(w)tU20 zz~baF2dC8SJG88&O|KHeq<%is>BDtq-sb^3kmx+ek`Gfmc#hv(`u2k*M*t#jc) z=2-2r#ojaP5dCgd^G2&&Gpqsdau9DYtSxY1D-q3;(3(9C(o&fnvUNVGfq}th3=u^( zy*%4_*E(2aN0@Ar!d`t25r0h z@~cMA)PoLFuLy(KM<-#FxTL=?aAqh|`!VAx762%!or~S;PLrKH3hbzjdlOg6a8kr-aIb+#iJIzwq8& zbJm6tVuE^RWYRR9`1oH?-|nsq(nn}h)4tY7rS>)!WfWP1G zDwv&4KLhh6>dGO_f0f;>$qolBf5!a0BCXnB782akv1zgDtK@ofrnh$JR>8VxR7Wwv zYY=r@ZTNk-GGAfMaN=LfngoRl))3vSzpwKeH}7oBY7O2*q|fzrWS0p9ym(pFe7(Uw ziK_|OdhXec>v7#ONCxh%|9Whl_qP6(k8HKxq`MAX@M5hfQ^DbZ|AZU{CHL-a614Yg zP~`7B`$7wVOH7Z#?dx6Uw;=s7hGZ7EW<2e$BVdB|S~zU_we{Ys2U5qLFVvMEUxyXm zfHwGnL6SMz|J(cdd+q(vi0CVou3G|=r-6GCv>J)~mj(Y1;;nM$zbyEVd<97J zFAM%ZnB`B??q4ammUjGC3jVW3{#OyNrSAUciui2m;Yx+LPP>g3C9Xn<1u*>hzUqy$ zd{L;;zHU3jq^Cg5>v$UYW|6+SNrDSPL51H_U7dE+bPRvHZ?J#F*gK8Tc= zhTB{eoOSzMT==Gg2MmTgJFT_jzl6#-8@OsHo~FxZIsehJxN09wHCCCZh}NB(AOp^< z^JzEjrQjV|uI>osp0s~G6Xq^h(2u3tjBFf28urwOHY=sGwt0|)9)$UiX=0}|uCGIq z{}&soumtmfTBe1z6Vcvo;Q;B`J{x~9^8tuww`>IoqaS2^7o3@sJ~oq0C4*9w0f*7B z9wNp$yue;kjMB?cRX&Gol09+f+ub|ADxK->EIS~Ogn`juUn$-tNpkBDV9Fx|;EwhB z%(523J|hsUPm-ckg>ZH7tPrR+eh#>yv(b;V_TJPb=9Zr!yVuO5*~BSYtvnI!29Y$1 zVW|FMY078j5t;Qo6-_YGulq{7`Fb|5_jfP&yQ>#=F~Rb8uc0l$a)m^m3VDae2RQ@n zq|+Xj4fwh9NhRZ*w`|6}Qq|!#T0wzv7lRnR4)+&XxfSx~#Avb_f=g9F0U8u=2&f_7 zu?~GsXP%9d>r!#;lRWq1&I2ZwUK=y;)#y*(LTu3Oqv+t;hTp*W1o!PZA40jGYG?L;mbQNxG$y*M`>!edA3#oxiwo_sjpjvx zRz380&KOBb7EVD(i6Eh=Hy1GvO$k(30ApQF)^O{Fl};Z+%U@p(9Jj(B#{zfRJU%vc zw9iLE8&$oeB*SYC9lNgz!p7BjE1^Gew3+_{N6Qh>>5nNz%KYT(=-k-v>gvkIYoi*U z<)l^e@8^dmG)Wb^HKtz7^L3pT!!3^aY3@D4VbUci6PXu9^8FYj8DXiDN5BXPfg2xx zS5^I+!N=Sn&ZtNTBA;|kkMp;4-~nf)|K~qPll0F0dUcIwR9VZkj`(GZC|(UmdVUSt z*zr5!+5J+d`PaKkWkWyDmn18lT7U$90_iIwkq#*sHE-@pZkUb+ztSQ0;sm)5UI;am zM6}_)T5fDaBKIqdH>0HA`rG9H&M!_Kgw%xzH>H~0nT8o>sUo3*xu^UX064Nd7X^o1 z{u5HEy~;Zy>Yc|v)r*ndO3QVMBwVhzshP`JC{PS-ifYDk9yZB{V%)U?KlCueNC zdfRNm81mR-gz%%dQL$-*xXN9JS7u%#@0aGdT7=WO$HZp$Kp%rdA6~1Pj4;SAU++?Z ze3TTRuxW3d){{Gq>Z|I8jP4)YV4FImqk~_%G8Wd8Y0Ll#q~suKzs>v#D@BYVb455@ zN6Z$T8JK;QUjSe6ZomT|{xj3r9k01SM$cXY#ga{lZMaiDn^zq}Y+2OfS= z_CvX4K5rP0+>(Gp^U`IWA>J`=^b>3gu2{sf9@VRHpHA2mVagT~jdE_E6nr~DaPl#b z?aht}Yc;X``}9!BtLkOT&M8PA~2zeYuNU{Xwwc&Fejma61v4?nL2RY+%$I%ZMNr~T$B+{Qo(mwL4 zRBH*Yf15_MMt)Z-p5aor*)?;%4a0@I>br9+APA}pZZ9KT#+`Tlr!9;o+HLnxiMS(Wb%OwA9t5fUP{@!Nb!5^>2s!Dxzl)~_-BHny=OPA_TFl3y0dPS*?(h@Z}-tkP-C38ko71o z)PE&k_W^(&i4kGw`6l@^{uAhEsA^z`mW+|6CZhexJj5sR@ z<)wBg3a34>D4WW?GoD(=FBL^iA&&os^W5NJmJulS^Pu3PA3qG1HMp4p=JE-2{aa71 z=F<}FdgtzpevAy&X4y4FqbsG6`O=CP z*Wu6*Y1tPyD|H7IpzAt^<}aqjX!3e!3;z*=>-nCB`$`#UP&wlx4A-}n*6al<*ZOq& zTfbz4rnTe|OA}OXMd0A#dbYFW3k`7L2Hn>~W<=k}n3O~?b1X_iKWgxX3zUVs*=f*} z&)qP$zBHRL(1zGI_(|^6F1b3Ib-A>76tuH!lM6Ye#l81q?8>amI>|1lv(A30e}+s2 z$tSx6C!g>zDAuAg`P^`=7ykU#HI6q)%Q zKh7UYSFGETuB3b^T;*}}*&TJ_$8KKODHiUuY2#a}+0B#5{Rv*ZD3qCVP|axR&dMBpy^trl_(9L^)~>w+?(C*p|g+kL!RMXtf^daC8y;Q3M01LU4Q zf_zX3>{x7<2;5sn-a3G4e!D(~jrExRVjls+@IpGG&X>(`K0*Q&B2$A}YCEJm;Y_C= z{q5V8PfBU3BBen}-SaZU#!Tk`%*y`5w}LNg9FEb8z#=#M_uFD$d!TvIrh@w>Y}gpK zH=I=%r$9_OnXF#)#9WLPcE4ynKWy4PB$i>uuV~?W?Jo0VE?++B(3m{lLgY{E<$n;- zi0KQ{uU+nRR%u&mLdi;H*yGAd18$38H-lpn8QVw8QoMvI!TqR zl=7n{zb2WgX;2RnlO1pGb4u=pL;TX?s^@!Cmg;oMY%2>YE>2Akf9AGln)2G4n%ma% zQlrk|vGVZa*4=~o);rqtdtRxesPkoW#AF2zhsv zF<1y^FG8ga3>c`t2NOSL=2UW+u73H>ELvDfUdTH>}%Bw4K zcXQ2XSrW&vL(5W`o$eTT4xCT$Yl)OFM`uhy%=uGfo~`-v0u=tN&$hyu8*b38n=yx` zUd`1=of?i{E~_;z2OhP7#5TWb9@`Q@UL^Wb?faGwic4p&%YZlX&*I%9SvaN}bLqF) zIf5#2&??Dz^E!kSReiEi4v~!+#S!9P5LkU9Wj7T0^>R*JSc_(IACx>>Gq7Z~65PNk z!thNmGK=GQU>F*&MQfhj!JE#+RH{t(K*dc9SzD5Tbdq-UE4i9&>?QHlIpYe$8XLh} zvOhOa6ig5NTyUvQBn9)W2mXGO;EQxeaeED?TVf>c1Fn|S6}iGICgoU@8tYINVl);7 z*?(#s?yr*CV^(&P+csi(lrX-nzI`Q^k3}w7ZV%#d%RIUFApVCVL62MQ+c@irCw+>T zk&F()QYS9c*Ih%8{t}~L6&!TwQ+}>)DElj4rkuN_!7R05rC!y~{#0$UqF*@nJX>xfgxBij#Oq~QCZVQ(6{lVCfki$& zReK=5D{Rzuo@C9lugRe&pH@zo8VTBEviqXG1p9TZvpM0qQp%`C%;kJS!05%#qOmlG zN65-OKhFu63BJ0MJE_}hoM-$+P&q5n%gsi?im<*KZwboIH4%q!3HOJQBD+%Zy0lBWCWdbI>#P+lM_z4*LK81|7VZkZRBy z_3kFO(wL9_%(B1s4v~n?K<$n0p6Gi^D~frRn0~+tmQT2T>~p%dW^g|yXT_h-GZq@w z9Ff>_DNXAbkqzHJqR)(MI3Mw7Ht=m8S!&GPe#)}Rb)Z)oa-^-EJ$qLvDuNx)D3rbN zlM2n}rlG)-xTF2UoJ5#3w;Fery)^KUkb-=kn;fCP@a7^8%dMHJ$N_Gxk%YGe*_U_r;s_!G1Gmk%y?JMao(OP+S+Go}w^&%ZZSRU-)edFi8 zDxPn7c|Un)z2+{mp`)oCt0i~lxa_Z2bB#X_j-yaz)o68-GHJhD>}M6Z!|5xQ=}x@n zn!?B3jVh;Kk0gC99H^Csq^ow@Xax2XH=nb-@6&g{1ePqiN8oWJ?G2fGdD1#t7(G>P z36niRimN-ZSdV_CNL-SNotjNKeQdz_hvnlx-?|OL6dtqtKw~<~_4MB6z|B$kn_SQ; zN3Y)HDLDla_nB5N9ps$IPbTyHZ@0kjC=`{rxn?ek@P~0%IDHv>Lsv^#-V#*}nTQwX zJ;WV9=j2~T)ga8AaBZsBjc}c93On^Y)AneQ4q$NIs#i&;ZeM$iGxQuSb=I7|np%^w zvFNfU;#mo;ks1X??#y0vYVA8g-`r)~HRvob<2z)}QPJ38=0}lA)8q3+6ue9LOkyVf zkDS<%+24!taQVJf*#;yA;^is4CZvpuVkQ%q`&v$n|+FiH;itq%6xrZ^H^4wZ6N<~c#G>}O8#vN zVM$9|NZ6DlXG0iTt6vaDCT$M6zKxZS?n@+?CaPfH(k^*e_o*ZGKlh7GJdqvkkEPM6 z<*~Eu&0;i}w}Pz-U}b4;tT)Xkd36OTT@!j!kz6-MQ|M0ED=s!#uzj{}(OgA3`OOKk zd@L7MK`pd7+3W0*omnXNxludY82Yub{~#u*`$FQ6O`Dqr2gipJ?lD$unm^K|gOAAx zWZf^DZ_6&Z+^!{kIN4z}Xf(g_UOIL>zf|k95S?D&;auq=`R*zff2>PK`+vZ&SF>3! zj-R?Fm+6o(R@rag=%X8vmF|}LB(qX7c7$4qR$B&=*^DelFSN|n>W@PecpT(rZ^ZvN z%v5#w=|mpRsAPA|N|pCzm;Z{CXLATTkKqlPPcq*P3@y@ zeRSW$yZ6OS&9j99_YQ>Ei`BwSk!JGn4cDXx;`h;?mZs`n?lA)YcK=aXqb zZb<3Eny8-&&fH-IW|u@wSl%XGcaJOd+w=?bGF_%b(dWU>0^to|sEHg=B=L333$?G3 zbVysG<4#&&vZC0;tUsNtuCPHTcIJs$ZQzC5g4k_U=$^%bkh1jCYp-vI`jjl-pV${$ z`t8WBUpQ;&TxL}!UGX{wy@gX+7~|okkv#lK@2#WaNA8rY&Y_5Teb@=BlA>yawD2fW zEoP@c%1FkYt)OtK#&#S!5FfR1F3>Vk;++ZhQ*-iUfWsh?s-Ap0?WU&X|CGANt`t?5 z_fHrWphp4?ds3nH7B=m@8;-tI5NT#X6X}nbj`R9f*oidXiK?{CqXH>VI`)Hl#t=8v z@uxGij87+Bnx++%#noxa`b_k6U{-Zy?CbeE$GTvL4zDL(sxE?3r7skdO7r-^cRSpn z$+flm$qyfP__pt;sm<;3URCz3u3E{O2E@Z!or zE&jj{L-!hUw$K0Ok<2`qFP$+l`_xD4aG$HNNj}L}X`Q0L4I4|gy)h&1q@EZwcORrL zyrw_sOgN7&xq*i(oayZBYmn0sn;HztiQHw<$IjyXs{F#&eE$xCgi|fsC}dEx$ME3+ z`vax0O1C%lg(o}N2h>`frmy5{RoNx3y4Qk;wqgPZwu_H3p%qpnlVZwkd|vM&-W~~b z$?d%jHTA6UmPL8>yL1HQ_4aMA45_z}xo0BqfzyR&3vxN4nI+RE?hXeuGEHBIoa!i*qmb)=BzSW?prfP+Nt$w zTsCV1d2&JWiiKR!-%ElM3I-C!2fI_IrY0{I`h9`;mH0V~I^^m9nfvt5mQ+=xVrw?n zElPN1!hD!{OYVGb+%p27HNoI2i)7Rkpt3j7_Iyti>xd96F8gO4?t{^ADUfo zRndAj#(4DRH+tLkXTp}=FMR}`kVJ@m+S0dr^OG)j_%3Pi_Ltk4T*O4dUXG?1ZKv=i zfqx8GsdX>l^##?%OY+0hzN0CclGcboR(EHgQ`k<0!cCC^+4bi;aobPJeWB?*n+WMV z(3NX~{his9d6RqQ*m^MX&#fRuiu6;N4%Hk=q912&{@6eHc|b$?o$u%ahN6O0>5(Ub zx4rfFQBh^8P<34HMXW{+UT87POhiR~o!cQgz6-qdo+mW(wv>NKL7VLkcbuHu$fIxN z6}~q|O8a5K-#josvjy#WV9UlFeFl|xPUu)e9Gt)$kQ=?0Prw&mSclH4A*-hy4G+xJ zsxazk);Biy>uhda4Ben3A)-v2=?L8rrcSaK_z>Y=T92)`Wt!y;2*ayqe~&?Ao4CW{ zBbdeEO6%>r)DYZ3*L{9hO0MKtL2r+lU@ZzdnqPVwY*Em+TL+3X*na%YIqdPU>D1!x!N~=|6iBKV0Zuk literal 0 HcmV?d00001 From 80e45f4ac46e02488c73b54363e7fb8e6224007d Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 09:42:22 -0700 Subject: [PATCH 076/115] fixes --- netbox_custom_objects/__init__.py | 21 +-- netbox_custom_objects/graphql/live.py | 99 +++++++++---- netbox_custom_objects/graphql/schema.py | 14 +- netbox_custom_objects/graphql/types.py | 136 ++++++++++++++---- netbox_custom_objects/tests/test_branching.py | 76 ++++++++++ netbox_custom_objects/tests/test_graphql.py | 65 +++++++++ 6 files changed, 341 insertions(+), 70 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 28f0fb33..5bd875da 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -227,15 +227,20 @@ def _patch_graphql_view(): @csrf_exempt def _patched_dispatch(self, request, *args, **kwargs): - from netbox_custom_objects.graphql.live import get_live_schema + from netbox_custom_objects.graphql.live import get_live_schema, main_branch_context - try: - live_schema = get_live_schema() - if live_schema is not None: - self.schema = live_schema - except Exception: # noqa: BLE001 - fall back to the static schema - logger.warning("Failed to load live GraphQL schema; using static schema", exc_info=True) - return _original_dispatch(self, request, *args, **kwargs) + # GraphQL is main-only: GraphiQL has no concept of branches, so both the + # schema and the data a query returns always reflect the main database, + # never an active branch. Wrapping the whole dispatch forces main for the + # schema rebuild and for the query's data resolution alike. + with main_branch_context(): + try: + live_schema = get_live_schema() + if live_schema is not None: + self.schema = live_schema + except Exception: # noqa: BLE001 - fall back to the static schema + logger.warning("Failed to load live GraphQL schema; using static schema", exc_info=True) + return _original_dispatch(self, request, *args, **kwargs) NetBoxGraphQLView.dispatch = _patched_dispatch _graphql_view_patched = True diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index 3ce34cf1..4a22635b 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -18,10 +18,17 @@ visible to every process on its next request — no restart, no cross-process messaging. +The GraphQL schema is **main-only**: it is global to the process and reflects the +main database, never a branch (netbox-branching). GraphQL has no concept of +branches, so custom object type changes made inside a branch must not alter the +schema — both the signature check and the rebuild run with the active branch reset +to main (see :func:`_main_branch_context`). + The view patch in ``__init__.py`` calls :func:`get_live_schema` per request and assigns the result to the view before it executes the operation. """ +import contextlib import logging import threading @@ -37,6 +44,32 @@ _current = (None, None) +@contextlib.contextmanager +def main_branch_context(): + """ + Run the enclosed block against the main database, ignoring any active branch. + + GraphQL is main-only: both its schema and the data it returns always reflect + main, never a branch. The signature check and the rebuild must not see a + branch's custom object types (a COT created or edited inside a branch must never + change the schema), and the request's data resolution must likewise read main. + Resetting the ``active_branch`` contextvar to ``None`` (main) for the duration + mirrors how the model cache treats ``branch_id=None`` as main. A no-op when + netbox-branching is not installed. + """ + try: + from netbox_branching.contextvars import active_branch + except ImportError: + yield + return + + token = active_branch.set(None) + try: + yield + finally: + active_branch.reset(token) + + def schema_signature(): """ Return a cheap, comparable fingerprint of the custom object type schema. @@ -107,40 +140,48 @@ def get_live_schema(): if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): return None - try: - signature = schema_signature() - except Exception: # noqa: BLE001 - DB hiccup: serve whatever we already have - logger.debug("Could not compute GraphQL schema signature", exc_info=True) - return _current[1] - - # Hot path: structure unchanged since this process last built — no lock, no - # rebuild, just return the cached schema. - sig, schema = _current - if schema is not None and sig == signature: - return schema - - # Structure changed (or first build). Single-flight: one thread rebuilds - # while concurrent requests keep serving the existing schema rather than - # blocking on the (potentially expensive) rebuild. - if not _rebuild_lock.acquire(blocking=False): - # Another thread is already rebuilding; serve the current schema — valid, - # just one signature behind — or None (→ static fallback) on first build. - return schema + # Both the signature check and the rebuild run against main: the schema is + # main-only and must not be perturbed by branch-local custom object types. + with main_branch_context(): + try: + signature = schema_signature() + except Exception: # noqa: BLE001 - DB hiccup: serve whatever we already have + logger.warning("Could not compute GraphQL schema signature", exc_info=True) + return _current[1] - try: - # Re-check: a prior holder may have just published a matching schema. + # Hot path: structure unchanged since this process last built — no lock, no + # rebuild, just return the cached schema. sig, schema = _current if schema is not None and sig == signature: return schema - try: - new_schema = build_full_schema() - except Exception: # noqa: BLE001 - never break the endpoint - logger.exception("Failed to rebuild live GraphQL schema") + + # Structure changed (or first build). Single-flight: one thread rebuilds + # while concurrent requests keep serving the existing schema rather than + # blocking on the (potentially expensive) rebuild. On the *first* build + # there is no schema to serve, so a loser must block until the rebuild + # completes — otherwise it would fall back to the custom-object-less static + # schema and spuriously reject custom_objects_* queries that do resolve. + blocking = schema is None + if not _rebuild_lock.acquire(blocking=blocking): + # Another thread is already rebuilding and we have a valid (one + # signature behind) schema to serve in the meantime. return schema - _current = (signature, new_schema) - return new_schema - finally: - _rebuild_lock.release() + + try: + # Re-check: a prior holder may have just published a matching schema + # (always true for the first-build blocker that just waited). + sig, schema = _current + if schema is not None and sig == signature: + return schema + try: + new_schema = build_full_schema() + except Exception: # noqa: BLE001 - never break the endpoint + logger.exception("Failed to rebuild live GraphQL schema") + return schema + _current = (signature, new_schema) + return new_schema + finally: + _rebuild_lock.release() def reset_cache(): diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 6e211959..83ae61fe 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -17,13 +17,12 @@ """ import logging -import re from typing import List import strawberry import strawberry_django -from .types import build_object_type +from .types import build_object_type, clear_type_cache, graphql_safe_name logger = logging.getLogger("netbox_custom_objects.graphql") @@ -44,7 +43,7 @@ def _query_field_name(custom_object_type, used_names): ``[_A-Za-z][_0-9A-Za-z]*``; slugs may contain hyphens. Collisions among custom object types (after sanitisation) are disambiguated with the type id. """ - slug = re.sub(r"[^0-9a-zA-Z_]", "_", (custom_object_type.slug or "").lower()) + slug = graphql_safe_name((custom_object_type.slug or "").lower()) # The prefix guarantees a valid leading character, so no digit/empty guard is # needed on the slug portion. base = f"{QUERY_FIELD_PREFIX}{slug}" @@ -86,6 +85,15 @@ class is avoided. from netbox_custom_objects.models import CustomObjectType + # Start each rebuild from an empty per-type cache. The cache is keyed by + # (cot id, cache_timestamp), but a type that embeds another COT's type via a + # relationship field does NOT get its own cache_timestamp bumped when the + # referenced COT changes — so a cached entry could embed a stale child type. + # Clearing here makes every type fresh per rebuild (rebuilds only happen on an + # actual structural change), while the cache still memoizes within this single + # rebuild pass so shared and recursive references reuse one built type. + clear_type_cache() + try: custom_object_types = list(CustomObjectType.objects.all()) except Exception: # noqa: BLE001 - DB may be unavailable at import time diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index ee22942a..b15ea886 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -33,6 +33,7 @@ from core.graphql.mixins import ChangelogMixin from extras.choices import CustomFieldTypeChoices from extras.graphql.mixins import TagsMixin +from netbox.graphql.scalars import BigInt from netbox.graphql.types import BaseObjectType from strawberry.types import Info @@ -46,22 +47,28 @@ "CustomObjectRelatedObjectType", "build_object_type", "clear_type_cache", + "graphql_safe_name", ) -# Per-process cache of built GraphQL types, keyed by (cot id, cache_timestamp). -# A COT's cache_timestamp is bumped (auto_now, plus an explicit save() on every -# field add/edit/delete) whenever the type or any of its fields changes, so a -# cached entry can never go stale: a structural change changes the key and forces -# a rebuild. This lets a schema rebuild triggered by one COT reuse the -# already-built types of every other COT instead of re-running build_object_type -# (and its per-COT fields query) for all of them. +# Per-rebuild memoization of built GraphQL types, keyed by (cot id, +# cache_timestamp). It lets a single schema rebuild reuse one built type across +# the many places that reference it (shared targets and recursive relationships) +# instead of re-running build_object_type for each. It is cleared at the start of +# every rebuild (see schema.build_query_classes): a type that embeds another COT's +# type does not get its own cache_timestamp bumped when that referenced COT +# changes, so persisting entries across rebuilds could serve a stale embedded +# type. clear_type_cache() also lets tests reset it explicitly. _type_cache = {} _type_cache_lock = threading.RLock() -# Tracks the COT ids whose GraphQL type is being built on the current thread, so -# that a relationship between two custom objects (A -> B -> A) does not recurse -# forever: a back-reference to a type still under construction falls back to the -# flat stub instead of rebuilding it. +# Per-thread build state. ``cot_stack`` is the stack of COT ids whose GraphQL +# type is being built on the current thread, so that a relationship between two +# custom objects (A -> B -> A) does not recurse forever: a back-reference to a +# type still under construction falls back to the flat stub instead of rebuilding +# it. ``cycle_tainted`` records the COT ids whose build had to use that stub +# fallback for a cyclic edge — those types are intentionally not cached (see +# build_object_type) so the next top-level query rebuilds them and resolves the +# related type fully from that entry point. _building = threading.local() # Lazily-built map of Django model class -> its registered NetBox strawberry @@ -94,7 +101,10 @@ class CustomObjectRelatedObjectType: field still exposes the basics rather than disappearing from the schema. """ - id: int + # BigInt (not int/Int): NetBox primary keys are BigAutoField and can exceed + # the signed 32-bit range of GraphQL's Int. Native relationship types already + # expose their id as BigInt; the fallback stub must match. + id: BigInt object_type: str display: str url: Optional[str] @@ -115,12 +125,25 @@ class CustomObjectObjectType(ChangelogMixin, TagsMixin, BaseObjectType): pass -def _in_progress_set(): - ids = getattr(_building, "cot_ids", None) - if ids is None: - ids = set() - _building.cot_ids = ids - return ids +def graphql_safe_name(value): + """Replace any character not valid in a GraphQL name with an underscore.""" + return re.sub(r"[^0-9a-zA-Z_]", "_", value or "") + + +def _in_progress_stack(): + stack = getattr(_building, "cot_stack", None) + if stack is None: + stack = [] + _building.cot_stack = stack + return stack + + +def _cycle_tainted_set(): + tainted = getattr(_building, "cycle_tainted", None) + if tainted is None: + tainted = set() + _building.cycle_tainted = tainted + return tainted def _request_user(info): @@ -149,6 +172,43 @@ def _user_can_view(user, obj): return manager.restrict(user, "view").filter(pk=obj.pk).exists() +def _filter_viewable(user, objects): + """ + Return the subset of ``objects`` the user may view, preserving order. + + Batches the permission check to one query per distinct model rather than one + ``.exists()`` per object (which is an N+1 explosion on multi-object fields): + the related objects are grouped by model and each model's permission-restricted + queryset is evaluated once with ``pk__in``. + """ + objects = [obj for obj in objects if obj is not None] + if user is None or not objects: + return [] + if getattr(user, "is_superuser", False): + return objects + + # sentinel meaning "model isn't permission-aware → all allowed". + allowed_by_model = {} + by_model = {} + for obj in objects: + by_model.setdefault(type(obj), []).append(obj) + for model, model_objs in by_model.items(): + manager = getattr(model, "_default_manager", None) + if manager is None or not hasattr(manager, "restrict"): + allowed_by_model[model] = None + continue + pks = [obj.pk for obj in model_objs] + allowed_by_model[model] = set( + manager.restrict(user, "view").filter(pk__in=pks).values_list("pk", flat=True) + ) + + return [ + obj + for obj in objects + if (allowed_by_model[type(obj)] is None or obj.pk in allowed_by_model[type(obj)]) + ] + + def _related_repr(obj): """Convert a referenced model instance into a ``CustomObjectRelatedObjectType``.""" if obj is None: @@ -223,13 +283,20 @@ def _custom_object_graphql_type(model_name): """Resolve a custom-object target (``tablemodel``) to its GraphQL type.""" from netbox_custom_objects.models import CustomObjectType - try: - cot_id = extract_cot_id_from_model_name(model_name) - except Exception: # noqa: BLE001 + cot_id = extract_cot_id_from_model_name(model_name) + if cot_id is None: return None - if cot_id is None or cot_id in _in_progress_set(): - # No id, or a back-reference to a type still being built — fall back to - # the flat stub for this edge to avoid infinite recursion. + # extract_cot_id_from_model_name returns the id as a str; the in-progress + # stack holds ints, so coerce before the membership test or it never matches. + cot_id = int(cot_id) + stack = _in_progress_stack() + if cot_id in stack: + # Back-reference to a type still being built — fall back to the flat stub + # for this edge to avoid infinite recursion, and taint the type currently + # under construction so it is not cached with this temporary stub frozen + # in (see build_object_type). + if stack: + _cycle_tainted_set().add(stack[-1]) return None cot = CustomObjectType.objects.filter(pk=cot_id).first() if cot is None: @@ -292,7 +359,7 @@ def _resolve_relationship_members(field): def _relationship_union_name(field): """A schema-unique, GraphQL-safe name for a polymorphic field's union type.""" - base = re.sub(r"[^0-9a-zA-Z_]", "_", field.name) + base = graphql_safe_name(field.name) return f"CustomObject{field.custom_object_type_id}_{base}_Related" @@ -355,8 +422,7 @@ def resolver(self, info: Info): return [] return [ _coerce_related(obj, native_models) - for obj in related - if obj is not None and _user_can_view(user, obj) + for obj in _filter_viewable(user, related) ] if value is None or not _user_can_view(user, value): return None @@ -398,12 +464,22 @@ def build_object_type(custom_object_type): if cached is not None: return cached - in_progress = _in_progress_set() - in_progress.add(custom_object_type.id) + stack = _in_progress_stack() + stack.append(custom_object_type.id) try: gql_type = _build_object_type(custom_object_type, model) finally: - in_progress.discard(custom_object_type.id) + stack.pop() + + # A type whose build had to break a relationship cycle with the flat stub (a + # related custom object was still under construction) must not be cached: the + # stub edge is an artefact of *this* build order, and caching it would freeze + # that degraded edge forever. Leaving it uncached lets a later top-level + # query rebuild it and resolve the related type fully from that entry point. + tainted = _cycle_tainted_set() + if custom_object_type.id in tainted: + tainted.discard(custom_object_type.id) + return gql_type with _type_cache_lock: _type_cache[cache_key] = gql_type diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index b95da368..7e163210 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -16,6 +16,7 @@ import time import unittest import uuid +from unittest import mock from core.models import ObjectType from dcim.models import Site @@ -2791,3 +2792,78 @@ def test_sync_then_branch_edit_then_merge_lifecycle(self): co.label, 'edited after sync', 'Edit made after sync must propagate to main on merge', ) + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +class GraphQLBranchIsolationTestCase(BranchingTestBase, TransactionTestCase): + """ + GraphQL is main-only: GraphiQL has no concept of branches, so the schema + (and the data it returns) must always reflect the main database. Custom + object type changes made inside a branch must never appear in the GraphQL + schema, regardless of which branch context a request happens to run in. + """ + + MERGE_STRATEGY = 'iterative' + + def setUp(self): + super().setUp() + from netbox_custom_objects.graphql import live as live_module + self.live = live_module + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + # The startup guard returns True during the test run; patch it off so the + # live schema machinery runs exactly as it does in production. + patcher = mock.patch( + 'netbox_custom_objects.CustomObjectsPluginConfig.' + 'should_skip_dynamic_model_creation', + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + def test_schema_signature_ignores_branch_changes(self): + # The signature drives rebuilds; forced to main it must not move when a + # COT is created inside a branch, even though the raw branch-context + # signature does. + main_sig = self.live.schema_signature() + + branch = _provision_branch('GraphQL Iso Sig', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create( + name='branch_only_sig', slug='branch-only-sig' + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + # Raw branch-context signature sees the new COT... + self.assertNotEqual(main_sig, self.live.schema_signature()) + # ...but forced to main (as get_live_schema does) it does not. + with self.live.main_branch_context(): + self.assertEqual(main_sig, self.live.schema_signature()) + + def test_live_schema_excludes_cot_created_in_branch(self): + # The assembled schema must never gain a branch-only type — neither while + # the branch is active nor afterwards in main (the branch isn't merged). + baseline = self.live.get_live_schema() + self.assertIsNotNone(baseline) + self.assertNotIn('custom_objects_branch_only', str(baseline)) + + branch = _provision_branch('GraphQL Iso Schema', self.MERGE_STRATEGY, self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create( + name='branch only', slug='branch-only' + ) + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + self.assertNotIn( + 'custom_objects_branch_only', str(self.live.get_live_schema()) + ) + + self.assertNotIn( + 'custom_objects_branch_only', str(self.live.get_live_schema()) + ) diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index 01f19fd4..31f06755 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -141,6 +141,37 @@ def test_real_builder_produces_assemblable_query(self): self.assertIn("custom_objects_gadget", sdl) self.assertIn("custom_objects_gadget_list", sdl) + def test_self_referential_object_field_does_not_recurse(self): + # A self-referential OBJECT field (FK to the same COT) must not send + # build_object_type into infinite recursion: the cycle guard breaks the + # back-edge with the flat stub. Regression test for the str/int mismatch + # that left the guard inert (extract_cot_id returns a str, the in-progress + # stack holds ints), which caused a RecursionError on this configuration. + from netbox_custom_objects.graphql import types as types_module + + types_module.clear_type_cache() + self.addCleanup(types_module.clear_type_cache) + + cot = self.create_custom_object_type(name="Node", slug="node") + self.create_custom_object_type_field( + cot, name="label", label="Label", type="text", primary=True, required=True + ) + self_ot = ObjectType.objects.get( + app_label="netbox_custom_objects", + model=cot.get_table_model_name(cot.id).lower(), + ) + self.create_custom_object_type_field( + cot, name="parent", label="Parent", type="object", related_object_type=self_ot + ) + cot.refresh_from_db() + + # Completes (no RecursionError) and yields a usable type. + gql_type = build_object_type(cot) + self.assertIsNotNone(gql_type) + # The cyclic build used the flat stub for the self-edge, so the type is + # intentionally not cached — a later top-level query rebuilds it. + self.assertIsNot(gql_type, build_object_type(cot)) + class GraphQLLiveSchemaTestCase(CustomObjectsTestCase, TestCase): """ @@ -473,3 +504,37 @@ def test_related_object_visible_with_permission(self): related = payload["data"]["custom_objects_server_list"][0]["site"] self.assertEqual(related["id"], str(self.site.pk)) self.assertEqual(related["name"], "Secret") + + def test_multiobject_related_filtered_by_permission(self): + # The multi-object resolver filters related objects through the batched + # permission check (_filter_viewable). Without view permission on Device + # the list is empty; granting it makes the device appear. + manufacturer = Manufacturer.objects.create(name="Mfr", slug="mfr") + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model="Model", slug="model" + ) + role = DeviceRole.objects.create(name="Role", slug="role") + site = Site.objects.create(name="DevSite", slug="devsite") + device = Device.objects.create( + name="dev1", device_type=device_type, role=role, site=site + ) + cot = self.create_multi_object_custom_object_type(name="Group", slug="group") + model = cot.get_model() + instance = model.objects.create(name="G1") + instance.devices.add(device) + + # View on the custom object but NOT on Device → devices filtered out. + self._grant(model, "view-grp") + payload = self._post("{ custom_objects_group_list { name devices { id } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + rows = payload["data"]["custom_objects_group_list"] + self.assertEqual(rows[0]["name"], "G1") + self.assertEqual(rows[0]["devices"], []) + + # Granting view on Device makes it appear (batched check lets it through). + self._grant(Device, "view-dev") + payload = self._post("{ custom_objects_group_list { name devices { id } } }") + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + devices = payload["data"]["custom_objects_group_list"][0]["devices"] + self.assertEqual(len(devices), 1) + self.assertEqual(devices[0]["id"], str(device.pk)) From ab36fb6c38bfce02ef48be830a3748aebc93d910 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 10:38:26 -0700 Subject: [PATCH 077/115] fixes --- netbox_custom_objects/__init__.py | 6 + netbox_custom_objects/graphql/live.py | 172 +++++++++++++++++++----- netbox_custom_objects/graphql/schema.py | 31 ++++- netbox_custom_objects/graphql/types.py | 62 +++++---- 4 files changed, 205 insertions(+), 66 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 5bd875da..3355a3fb 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -387,6 +387,12 @@ def ready(self): # are reflected in the schema without a NetBox restart. _patch_graphql_view() + # Keep the live GraphQL schema's signature cache fresh across workers + # event-driven, so the per-request hot path reads the cache instead of + # polling the database. + from .graphql.live import connect_signature_invalidation + connect_signature_invalidation() + # Register netbox-branching integration hooks (deferred-data reset # receivers, branchable resolver, ObjectChange field-name migrator, # squash dependency-graph receiver). Guarded so the plugin still diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index 4a22635b..338c2d45 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -10,8 +10,10 @@ every process, without a restart: - :func:`schema_signature` computes a cheap fingerprint of the custom object - types and their fields (counts + most-recent change timestamps). One small - aggregate query pair per request. + types and their fields (counts + most-recent change timestamps). The result + is memoised in NetBox's shared cache and invalidated event-driven by the + post_save/post_delete receivers in :func:`connect_signature_invalidation`, so + the steady-state cost is one cache read rather than two DB queries per request. - :func:`get_live_schema` caches the assembled schema per process and rebuilds it only when the signature changes. Because each process checks the signature independently, a type created by a request handled in one process becomes @@ -53,21 +55,19 @@ def main_branch_context(): main, never a branch. The signature check and the rebuild must not see a branch's custom object types (a COT created or edited inside a branch must never change the schema), and the request's data resolution must likewise read main. - Resetting the ``active_branch`` contextvar to ``None`` (main) for the duration - mirrors how the model cache treats ``branch_id=None`` as main. A no-op when - netbox-branching is not installed. + Delegates to netbox-branching's own ``deactivate_branch`` context manager + (``activate_branch(None)``) so the meaning of "main" stays owned upstream + rather than reimplemented here. A no-op when netbox-branching is not + installed. """ try: - from netbox_branching.contextvars import active_branch + from netbox_branching.utilities import deactivate_branch except ImportError: yield return - token = active_branch.set(None) - try: + with deactivate_branch(): yield - finally: - active_branch.reset(token) def schema_signature(): @@ -90,36 +90,131 @@ def schema_signature(): return (cot["n"], cot["t"], fields["n"], fields["t"]) +# Sentinel signature for a best-effort first build made when the signature query +# itself failed (see get_live_schema): it compares unequal to every real +# signature, so the next request with a working DB rebuilds. +_UNKNOWN_SIGNATURE = object() + +_SIGNATURE_CACHE_KEY = "netbox_custom_objects.graphql.schema_signature" +# Backstop TTL only — invalidation is event-driven (connect_signature_invalidation). +# It bounds staleness if an invalidation is ever lost (e.g. a cache blip during a +# write) without making the steady state poll the database. +_SIGNATURE_CACHE_TIMEOUT = 300 + + +def cached_schema_signature(): + """ + Return the main schema signature, memoised in NetBox's shared cache. + + :func:`schema_signature` is two aggregate queries; running them on every + GraphQL request is a needless per-request DB tax when the schema almost never + changes. The cached value is invalidated event-driven by the receivers in + :func:`connect_signature_invalidation`, so a change in any worker is reflected + everywhere on the next request — the same freshness guarantee as polling, at + one cache read instead of two DB round-trips. Falls back to a direct DB read + whenever the cache is unavailable. + """ + from django.core.cache import cache + + try: + cached = cache.get(_SIGNATURE_CACHE_KEY) + except Exception: # noqa: BLE001 - cache down: fall back to the DB + cached = None + if cached is not None: + # Cache backends may round-trip the tuple as a list; normalise so the + # equality check against the stored signature stays type-stable. + return tuple(cached) + + signature = schema_signature() + try: + cache.set(_SIGNATURE_CACHE_KEY, signature, _SIGNATURE_CACHE_TIMEOUT) + except Exception: # noqa: BLE001 - cache down: just skip memoisation + pass + return signature + + +def _invalidate_signature_cache(**kwargs): + """Drop the memoised schema signature so the next request recomputes it.""" + from django.core.cache import cache + + try: + cache.delete(_SIGNATURE_CACHE_KEY) + except Exception: # noqa: BLE001 - cache down: the TTL backstop still bounds staleness + logger.debug("Could not invalidate GraphQL schema signature cache", exc_info=True) + + +def connect_signature_invalidation(): + """ + Connect the receivers that invalidate the cached schema signature. + + Called once from ``CustomObjectsPluginConfig.ready()``. Creating, deleting, + or editing any custom object type or field changes the signature; deleting the + cache key on those events keeps :func:`cached_schema_signature` correct without + polling the database per request. ``dispatch_uid`` makes repeat ``ready()`` + calls idempotent. + """ + from django.db.models.signals import post_delete, post_save + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + for signal, label in ((post_save, "save"), (post_delete, "delete")): + for model in (CustomObjectType, CustomObjectTypeField): + signal.connect( + _invalidate_signature_cache, + sender=model, + dispatch_uid=f"nco_graphql_sig_{label}_{model.__name__}", + weak=False, + ) + + +# NetBox assembles its GraphQL schema once at import and never changes it for the +# life of the process — only our custom-object slice does. Capture NetBox's +# static contribution (its per-app/plugin Query bases and its own +# ``StrawberryConfig``) once; every rebuild then regenerates only our slice and +# reassembles. Reusing NetBox's real config — rather than a hand-copied one — +# keeps the rebuilt schema in lock-step with NetBox's own (e.g. its stored +# ``auto_camel_case`` is ``None``, not the ``False`` a copy would assume) and +# removes a source of silent drift across supported NetBox versions. +_static_query_parts = None + + +def _get_static_query_parts(): + """Return ``(query_bases, config)`` captured once from NetBox's startup schema.""" + global _static_query_parts + if _static_query_parts is None: + import netbox.graphql.schema as ngs + + # These bases never include our own contribution (our startup + # ``graphql_schema`` export is empty and we never write back to + # ``ngs.Query``); the ``_nco_query`` guard is belt-and-braces. + bases = tuple(b for b in ngs.Query.__bases__ if not getattr(b, "_nco_query", False)) + _static_query_parts = (bases, ngs.schema.config) + return _static_query_parts + + def build_full_schema(): """ Assemble a complete NetBox GraphQL schema with the current custom object types. - Reuses NetBox's own ``Query`` base classes (all core apps plus every other - plugin) so the result is identical to NetBox's startup schema except that our - custom object query is rebuilt from the live database. Our previously - contributed query class is identified by the ``_nco_query`` marker and - replaced. + A ``strawberry.Schema`` is immutable once compiled, so adding/removing a root + query field requires building a new schema — but only our custom-object slice + (:func:`build_query_classes`) is rebuilt here. NetBox's Query bases and config + are captured once (:func:`_get_static_query_parts`); the result is identical to + NetBox's startup schema except for the live custom-object query. """ import strawberry - from strawberry.schema.config import StrawberryConfig import netbox.graphql.schema as ngs - from netbox.graphql.scalars import BigInt, BigIntScalar from .schema import build_query_classes - # Start from NetBox's canonical Query bases, dropping any stale custom-object - # query class we contributed previously, then graft a freshly built one on. - bases = tuple(b for b in ngs.Query.__bases__ if not getattr(b, "_nco_query", False)) - bases += tuple(build_query_classes()) + static_bases, config = _get_static_query_parts() + bases = static_bases + tuple(build_query_classes()) query_cls = strawberry.type(type("Query", bases, {})) return strawberry.Schema( query=query_cls, - config=StrawberryConfig( - auto_camel_case=False, - scalar_map={BigInt: BigIntScalar}, - ), + config=config, extensions=ngs.get_schema_extensions(), ) @@ -144,10 +239,19 @@ def get_live_schema(): # main-only and must not be perturbed by branch-local custom object types. with main_branch_context(): try: - signature = schema_signature() - except Exception: # noqa: BLE001 - DB hiccup: serve whatever we already have + signature = cached_schema_signature() + except Exception: # noqa: BLE001 - DB hiccup logger.warning("Could not compute GraphQL schema signature", exc_info=True) - return _current[1] + cached_schema = _current[1] + if cached_schema is not None: + # We already have a (possibly slightly stale) schema — serve it. + return cached_schema + # First build and even the signature query failed. Rather than fall + # back to NetBox's static schema (which has no custom_objects_* fields + # and would reject otherwise-valid queries), make a best-effort first + # build under a sentinel signature so the next request re-checks once + # the DB recovers. + signature = _UNKNOWN_SIGNATURE # Hot path: structure unchanged since this process last built — no lock, no # rebuild, just return the cached schema. @@ -185,10 +289,18 @@ def get_live_schema(): def reset_cache(): - """Clear the cached schema and per-type cache (used by tests).""" + """Clear the cached schema, signature, per-type cache, and build state (tests).""" global _current _current = (None, None) - from .types import clear_type_cache + from django.core.cache import cache + + try: + cache.delete(_SIGNATURE_CACHE_KEY) + except Exception: # noqa: BLE001 - cache down: nothing to clear + pass + + from .types import clear_type_cache, reset_build_state clear_type_cache() + reset_build_state() diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 83ae61fe..61f86c7c 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -22,7 +22,7 @@ import strawberry import strawberry_django -from .types import build_object_type, clear_type_cache, graphql_safe_name +from .types import build_object_type, clear_type_cache, graphql_safe_name, reset_build_state logger = logging.getLogger("netbox_custom_objects.graphql") @@ -47,14 +47,27 @@ def _query_field_name(custom_object_type, used_names): # The prefix guarantees a valid leading character, so no digit/empty guard is # needed on the slug portion. base = f"{QUERY_FIELD_PREFIX}{slug}" + + def _taken(candidate): + # Reserve the singular field name *and* its ``_list`` companion together. + # Checking both prevents one type's list field from silently colliding + # with another type's singular field — e.g. slug 'foo' yields foo/foo_list + # while slug 'foo-list' sanitises to foo_list/foo_list_list, and the bare + # 'foo_list' would otherwise clobber the first type's list field. + return candidate in used_names or f"{candidate}_list" in used_names + name = base - # Reserve the singular field name *and* its ``_list`` companion together. - # Checking/recording both prevents one type's list field from silently - # colliding with another type's singular field — e.g. slug 'foo' yields - # foo/foo_list while slug 'foo-list' sanitises to foo_list/foo_list_list, and - # the bare 'foo_list' would otherwise clobber the first type's list field. - if name in used_names or f"{name}_list" in used_names: + if _taken(name): + # Disambiguate with the (unique) type id. The disambiguated name can + # itself collide with one already reserved — another type whose slug + # happens to end in this id — so keep extending until both the singular + # name and its ``_list`` companion are genuinely free. Without this loop + # the colliding field would silently overwrite the earlier type's field. name = f"{base}_{custom_object_type.id}" + counter = 2 + while _taken(name): + name = f"{base}_{custom_object_type.id}_{counter}" + counter += 1 used_names.add(name) used_names.add(f"{name}_list") return name @@ -93,6 +106,10 @@ class is avoided. # actual structural change), while the cache still memoizes within this single # rebuild pass so shared and recursive references reuse one built type. clear_type_cache() + # Also drop any in-progress build state (stack / cycle taint) leaked by an + # exception during a previous rebuild on this pooled thread, so it can't + # suppress caching or corrupt cycle detection on this rebuild. + reset_build_state() try: custom_object_types = list(CustomObjectType.objects.all()) diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index b15ea886..6aa0a2a1 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -48,6 +48,7 @@ "build_object_type", "clear_type_cache", "graphql_safe_name", + "reset_build_state", ) # Per-rebuild memoization of built GraphQL types, keyed by (cot id, @@ -84,6 +85,21 @@ def clear_type_cache(): _type_cache.clear() +def reset_build_state(): + """ + Clear this thread's in-progress build stack and cycle-taint set. + + Called at the start of each schema rebuild (and by tests) so that a stack + frame or taint leaked by an exception during a previous rebuild on this + (pooled) thread cannot suppress caching or corrupt cycle detection on the + next one. ``build_object_type`` only clears a type's taint on the success + path, so a build that raises after a cyclic edge tainted an ancestor would + otherwise leave that taint set on the thread indefinitely. + """ + _building.cot_stack = [] + _building.cycle_tainted = set() + + RELATIONSHIP_TYPES = ( CustomFieldTypeChoices.TYPE_OBJECT, CustomFieldTypeChoices.TYPE_MULTIOBJECT, @@ -152,32 +168,16 @@ def _request_user(info): return getattr(request, "user", None) -def _user_can_view(user, obj): - """ - Return whether ``user`` has NetBox 'view' permission for ``obj``. - - The top-level query restricts the custom objects themselves, but the objects - reached through their relationship fields are *not* covered by that check, so - each one must be gated individually or the field would leak objects the user - cannot see. - """ - if obj is None or user is None: - return False - if getattr(user, "is_superuser", False): - return True - manager = getattr(type(obj), "_default_manager", None) - if manager is None or not hasattr(manager, "restrict"): - # Target model isn't permission-aware; nothing to enforce. - return True - return manager.restrict(user, "view").filter(pk=obj.pk).exists() - - def _filter_viewable(user, objects): """ Return the subset of ``objects`` the user may view, preserving order. - Batches the permission check to one query per distinct model rather than one - ``.exists()`` per object (which is an N+1 explosion on multi-object fields): + The top-level query restricts the custom objects themselves, but the objects + reached through their relationship fields are *not* covered by that check, so + each one must be gated here or the field would leak objects the user cannot + see. Used for both single- and multi-object fields so the permission rule + lives in one place; batches the check to one query per distinct model rather + than one ``.exists()`` per object (an N+1 explosion on multi-object fields): the related objects are grouped by model and each model's permission-restricted queryset is evaluated once with ``pk__in``. """ @@ -292,11 +292,12 @@ def _custom_object_graphql_type(model_name): stack = _in_progress_stack() if cot_id in stack: # Back-reference to a type still being built — fall back to the flat stub - # for this edge to avoid infinite recursion, and taint the type currently - # under construction so it is not cached with this temporary stub frozen - # in (see build_object_type). - if stack: - _cycle_tainted_set().add(stack[-1]) + # for this edge to avoid infinite recursion, and taint every type currently + # under construction so none of them is cached with this temporary stub + # frozen in (see build_object_type). Tainting only the immediate parent + # (stack[-1]) would still cache the outer types of a cycle longer than two + # (A -> B -> C -> A), permanently freezing the stub into their subtree. + _cycle_tainted_set().update(stack) return None cot = CustomObjectType.objects.filter(pk=cot_id).first() if cot is None: @@ -424,9 +425,12 @@ def resolver(self, info: Info): _coerce_related(obj, native_models) for obj in _filter_viewable(user, related) ] - if value is None or not _user_can_view(user, value): + if value is None: + return None + viewable = _filter_viewable(user, [value]) + if not viewable: return None - return _coerce_related(value, native_models) + return _coerce_related(viewable[0], native_models) resolver.__annotations__ = {"info": Info, "return": annotation} return strawberry_django.field(description=description, **hint)(resolver) From 982c34200dfdf9e79c50ad0c3fd406a7d5dcba7a Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 10:57:15 -0700 Subject: [PATCH 078/115] branching / graphql --- netbox_custom_objects/__init__.py | 16 +- netbox_custom_objects/graphql/live.py | 269 +++++++++++++----- netbox_custom_objects/tests/test_branching.py | 219 ++++++++++++-- 3 files changed, 402 insertions(+), 102 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 3355a3fb..45f81eb4 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -227,13 +227,15 @@ def _patch_graphql_view(): @csrf_exempt def _patched_dispatch(self, request, *args, **kwargs): - from netbox_custom_objects.graphql.live import get_live_schema, main_branch_context - - # GraphQL is main-only: GraphiQL has no concept of branches, so both the - # schema and the data a query returns always reflect the main database, - # never an active branch. Wrapping the whole dispatch forces main for the - # schema rebuild and for the query's data resolution alike. - with main_branch_context(): + from netbox_custom_objects.graphql.live import get_live_schema, graphql_branch_context + + # GraphQL resolves against main unless the request explicitly selects a + # branch with the X-NetBox-Branch header (GraphiQL has no branch concept, so + # an implicit UI branch via cookie/query-param must not leak in). + # graphql_branch_context scopes the active branch accordingly, and + # get_live_schema builds the schema for that same branch, so the schema and + # the data the query returns stay consistent. + with graphql_branch_context(request): try: live_schema = get_live_schema() if live_schema is not None: diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index 338c2d45..b7cad8af 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -20,11 +20,14 @@ visible to every process on its next request — no restart, no cross-process messaging. -The GraphQL schema is **main-only**: it is global to the process and reflects the -main database, never a branch (netbox-branching). GraphQL has no concept of -branches, so custom object type changes made inside a branch must not alter the -schema — both the signature check and the rebuild run with the active branch reset -to main (see :func:`_main_branch_context`). +By default GraphQL resolves against **main**: GraphiQL has no concept of branches, +so an implicit UI branch (a branch cookie or ``?_branch=`` query param) must not +leak into it. A client may opt a single request into a branch with the +``X-NetBox-Branch`` header (the REST/GraphQL API convention — see +``netbox_branching.utilities.get_active_branch``); that request then gets that +branch's schema *and* data. :func:`graphql_branch_context` enforces this at the +view, and the schema cache is keyed per branch so each branch's custom object types +are reflected independently of main. The view patch in ``__init__.py`` calls :func:`get_live_schema` per request and assigns the result to the view before it executes the operation. @@ -36,14 +39,21 @@ logger = logging.getLogger("netbox_custom_objects.graphql") -# Single-flight rebuild lock: only one thread rebuilds at a time; concurrent -# requests serve the current (possibly slightly stale) schema instead of blocking. +# Single-flight rebuild lock: only one thread rebuilds at a time (across all +# branches, so the shared per-rebuild type cache in graphql.types is never built +# by two threads at once); concurrent requests serve the current (possibly slightly +# stale) schema instead of blocking. _rebuild_lock = threading.Lock() -# Atomically-swapped (signature, schema) pair. Read without a lock on the hot -# path — a single reference read/assignment is atomic in CPython, and pairing the -# signature with the schema in one tuple means a reader can never see a schema +# Per-branch cache of (signature, schema), keyed by branch identifier (None = main). +# GraphQL honours a branch only when a request explicitly selects one via the +# X-NetBox-Branch header (see graphql_branch_context); each branch gets its own +# schema reflecting that branch's custom object types. Each value is an atomically +# swapped (signature, schema) tuple, so a lockless reader can never see a schema # that doesn't match its signature. -_current = (None, None) +_schema_cache = {} +# Signature cache keys this process has populated, so reset_cache (tests) can clear +# every per-branch entry. +_signature_keys_seen = set() @contextlib.contextmanager @@ -70,6 +80,50 @@ def main_branch_context(): yield +def _active_branch_key(): + """ + Identifier for the active branch (``None`` for main), used to key the per-branch + schema and signature caches. ``None`` when netbox-branching is not installed. + """ + try: + from netbox_branching.contextvars import active_branch + except ImportError: + return None + branch = active_branch.get() + return branch.pk if branch is not None else None + + +@contextlib.contextmanager +def graphql_branch_context(request): + """ + Scope a GraphQL request to a branch only when one is explicitly requested via + the ``X-NetBox-Branch`` header (the REST/GraphQL API convention — see + ``netbox_branching.utilities.get_active_branch``). + + Browser GraphiQL has no concept of branches, but a UI session may carry a branch + cookie or ``?_branch=`` query param that netbox-branching's middleware would + activate for the request. Letting that leak into GraphQL would silently resolve + the schema and data against a branch the GraphiQL user never chose, so without + the header we force main. With the header, the middleware has already activated + the branch and we leave it active for both the schema rebuild and the query's + data resolution. A no-op when netbox-branching is not installed. + """ + try: + from netbox_branching.constants import BRANCH_HEADER + except ImportError: + yield + return + + headers = getattr(request, "headers", None) or {} + if BRANCH_HEADER in headers: + # Explicit branch selection — honour the active branch the middleware set. + yield + else: + # No explicit selection: force main so an implicit UI branch can't leak in. + with main_branch_context(): + yield + + def schema_signature(): """ Return a cheap, comparable fingerprint of the custom object type schema. @@ -102,22 +156,33 @@ def schema_signature(): _SIGNATURE_CACHE_TIMEOUT = 300 -def cached_schema_signature(): +def _signature_cache_key(branch_key): + """Per-branch cache key for the schema signature (``None`` = main).""" + if branch_key is None: + return _SIGNATURE_CACHE_KEY + return f"{_SIGNATURE_CACHE_KEY}:{branch_key}" + + +def cached_schema_signature(branch_key=None): """ - Return the main schema signature, memoised in NetBox's shared cache. + Return the active branch's schema signature, memoised in NetBox's shared cache. :func:`schema_signature` is two aggregate queries; running them on every GraphQL request is a needless per-request DB tax when the schema almost never changes. The cached value is invalidated event-driven by the receivers in :func:`connect_signature_invalidation`, so a change in any worker is reflected everywhere on the next request — the same freshness guarantee as polling, at - one cache read instead of two DB round-trips. Falls back to a direct DB read + one cache read instead of two DB round-trips. The key includes the branch + identifier so a branch's signature never shadows main's, and the aggregates run + under the caller's active branch context. Falls back to a direct DB read whenever the cache is unavailable. """ from django.core.cache import cache + key = _signature_cache_key(branch_key) + _signature_keys_seen.add(key) try: - cached = cache.get(_SIGNATURE_CACHE_KEY) + cached = cache.get(key) except Exception: # noqa: BLE001 - cache down: fall back to the DB cached = None if cached is not None: @@ -127,22 +192,48 @@ def cached_schema_signature(): signature = schema_signature() try: - cache.set(_SIGNATURE_CACHE_KEY, signature, _SIGNATURE_CACHE_TIMEOUT) + cache.set(key, signature, _SIGNATURE_CACHE_TIMEOUT) except Exception: # noqa: BLE001 - cache down: just skip memoisation pass return signature def _invalidate_signature_cache(**kwargs): - """Drop the memoised schema signature so the next request recomputes it.""" + """ + Drop the memoised schema signature for the branch the change occurred in, so the + next request for that branch recomputes it. The receiver fires inside whatever + branch context performed the save/delete, so the active branch is the one to + invalidate. + """ from django.core.cache import cache try: - cache.delete(_SIGNATURE_CACHE_KEY) + cache.delete(_signature_cache_key(_active_branch_key())) except Exception: # noqa: BLE001 - cache down: the TTL backstop still bounds staleness logger.debug("Could not invalidate GraphQL schema signature cache", exc_info=True) +def _evict_branch_schema(sender, instance, **kwargs): + """ + Drop a deleted branch's cached schema and signature. + + A branch's schema is cached under its pk in :data:`_schema_cache`; once the + branch is gone that entry can never be served again (a request can't reference a + deleted branch), so evict it rather than leak it for the life of the process. + """ + branch_key = instance.pk + _schema_cache.pop(branch_key, None) # atomic dict op; no lock needed + + from django.core.cache import cache + + key = _signature_cache_key(branch_key) + _signature_keys_seen.discard(key) + try: + cache.delete(key) + except Exception: # noqa: BLE001 - cache down: the TTL backstop still bounds it + logger.debug("Could not evict deleted branch's schema signature", exc_info=True) + + def connect_signature_invalidation(): """ Connect the receivers that invalidate the cached schema signature. @@ -150,8 +241,9 @@ def connect_signature_invalidation(): Called once from ``CustomObjectsPluginConfig.ready()``. Creating, deleting, or editing any custom object type or field changes the signature; deleting the cache key on those events keeps :func:`cached_schema_signature` correct without - polling the database per request. ``dispatch_uid`` makes repeat ``ready()`` - calls idempotent. + polling the database per request. Deleting a branch evicts that branch's cached + schema so it can't leak. ``dispatch_uid`` makes repeat ``ready()`` calls + idempotent. """ from django.db.models.signals import post_delete, post_save @@ -166,6 +258,19 @@ def connect_signature_invalidation(): weak=False, ) + # Evict a branch's cached schema when the branch itself is deleted. No-op when + # netbox-branching is not installed (there are then no branch-keyed entries). + try: + from netbox_branching.models import Branch + except ImportError: + return + post_delete.connect( + _evict_branch_schema, + sender=Branch, + dispatch_uid="nco_graphql_evict_branch", + weak=False, + ) + # NetBox assembles its GraphQL schema once at import and never changes it for the # life of the process — only our custom-object slice does. Capture NetBox's @@ -221,84 +326,90 @@ def build_full_schema(): def get_live_schema(): """ - Return the schema for the current request, rebuilding it if the database has - changed since this process last built it. + Return the schema for the current request's active branch, rebuilding it if the + database has changed since this process last built it for that branch. + + The caller (the view patch, via :func:`graphql_branch_context`) has already + scoped the active branch: main unless the request carried the X-NetBox-Branch + header. The signature check, the rebuild, and the query's data resolution + therefore all run against that same branch. Returns ``None`` when dynamic models are unavailable (migrations/tests) or if the very first build fails — the caller then falls back to NetBox's static schema. """ - global _current - from netbox_custom_objects import CustomObjectsPluginConfig if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): return None - # Both the signature check and the rebuild run against main: the schema is - # main-only and must not be perturbed by branch-local custom object types. - with main_branch_context(): - try: - signature = cached_schema_signature() - except Exception: # noqa: BLE001 - DB hiccup - logger.warning("Could not compute GraphQL schema signature", exc_info=True) - cached_schema = _current[1] - if cached_schema is not None: - # We already have a (possibly slightly stale) schema — serve it. - return cached_schema - # First build and even the signature query failed. Rather than fall - # back to NetBox's static schema (which has no custom_objects_* fields - # and would reject otherwise-valid queries), make a best-effort first - # build under a sentinel signature so the next request re-checks once - # the DB recovers. - signature = _UNKNOWN_SIGNATURE - - # Hot path: structure unchanged since this process last built — no lock, no - # rebuild, just return the cached schema. - sig, schema = _current - if schema is not None and sig == signature: - return schema + branch_key = _active_branch_key() - # Structure changed (or first build). Single-flight: one thread rebuilds - # while concurrent requests keep serving the existing schema rather than - # blocking on the (potentially expensive) rebuild. On the *first* build - # there is no schema to serve, so a loser must block until the rebuild - # completes — otherwise it would fall back to the custom-object-less static - # schema and spuriously reject custom_objects_* queries that do resolve. - blocking = schema is None - if not _rebuild_lock.acquire(blocking=blocking): - # Another thread is already rebuilding and we have a valid (one - # signature behind) schema to serve in the meantime. - return schema + try: + signature = cached_schema_signature(branch_key) + except Exception: # noqa: BLE001 - DB hiccup + logger.warning("Could not compute GraphQL schema signature", exc_info=True) + cached = _schema_cache.get(branch_key) + if cached is not None and cached[1] is not None: + # We already have a (possibly slightly stale) schema for this branch. + return cached[1] + # First build for this branch and even the signature query failed. Rather + # than fall back to NetBox's static schema (which has no custom_objects_* + # fields and would reject otherwise-valid queries), make a best-effort first + # build under a sentinel signature so the next request re-checks once the DB + # recovers. + signature = _UNKNOWN_SIGNATURE + + # Hot path: structure unchanged since this process last built for this branch — + # no lock, no rebuild, just return the cached schema. + entry = _schema_cache.get(branch_key) + sig, schema = entry if entry is not None else (None, None) + if schema is not None and sig == signature: + return schema + + # Structure changed (or first build). Single-flight: one thread rebuilds while + # concurrent requests keep serving the existing schema rather than blocking on + # the (potentially expensive) rebuild. On the *first* build there is no schema + # to serve, so a loser must block until the rebuild completes — otherwise it + # would fall back to the custom-object-less static schema and spuriously reject + # custom_objects_* queries that do resolve. + blocking = schema is None + if not _rebuild_lock.acquire(blocking=blocking): + # Another thread is already rebuilding and we have a valid (one signature + # behind) schema to serve in the meantime. + return schema + try: + # Re-check: a prior holder may have just published a matching schema for + # this branch (always true for the first-build blocker that just waited). + entry = _schema_cache.get(branch_key) + sig, schema = entry if entry is not None else (None, None) + if schema is not None and sig == signature: + return schema try: - # Re-check: a prior holder may have just published a matching schema - # (always true for the first-build blocker that just waited). - sig, schema = _current - if schema is not None and sig == signature: - return schema - try: - new_schema = build_full_schema() - except Exception: # noqa: BLE001 - never break the endpoint - logger.exception("Failed to rebuild live GraphQL schema") - return schema - _current = (signature, new_schema) - return new_schema - finally: - _rebuild_lock.release() + new_schema = build_full_schema() + except Exception: # noqa: BLE001 - never break the endpoint + logger.exception("Failed to rebuild live GraphQL schema") + return schema + _schema_cache[branch_key] = (signature, new_schema) + return new_schema + finally: + _rebuild_lock.release() def reset_cache(): - """Clear the cached schema, signature, per-type cache, and build state (tests).""" - global _current - _current = (None, None) + """Clear the cached schemas, signatures, per-type cache, and build state (tests).""" + global _schema_cache + _schema_cache = {} from django.core.cache import cache - try: - cache.delete(_SIGNATURE_CACHE_KEY) - except Exception: # noqa: BLE001 - cache down: nothing to clear - pass + for key in list(_signature_keys_seen): + try: + cache.delete(key) + except Exception: # noqa: BLE001 - cache down: nothing to clear + pass + _signature_keys_seen.clear() from .types import clear_type_cache, reset_build_state diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index 7e163210..b744df32 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -11,6 +11,7 @@ """ import datetime import decimal +import json import logging import os import time @@ -23,9 +24,11 @@ from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.db import connection as main_conn, connections -from django.test import RequestFactory, TransactionTestCase +from django.test import RequestFactory, TransactionTestCase, override_settings from django.urls import reverse from extras.models import CustomFieldChoiceSet +from rest_framework.test import APIClient +from users.models import Token try: from netbox.context_managers import event_tracking @@ -51,6 +54,21 @@ def _make_request(user): return request +def _create_token(user): + """Create an API token (plaintext key) across NetBox token versions.""" + try: + # NetBox >= 4.5 + from users.choices import TokenVersionChoices + token = Token(version=TokenVersionChoices.V1, user=user) + token.save() + return token.token + except ImportError: + # NetBox < 4.5 + token = Token(user=user) + token.save() + return token.key + + # Provisioning timeout for branch tests. Override via the # ``NETBOX_CO_BRANCH_PROVISION_TIMEOUT`` env var (seconds) when CI flakes. BRANCH_PROVISION_TIMEOUT = float( @@ -58,8 +76,13 @@ def _make_request(user): ) -def _provision_branch(name, merge_strategy, user, timeout=None): - """Create and wait for a branch to reach READY status.""" +def _provision_branch(name, merge_strategy=None, user=None, timeout=None): + """Create and wait for a branch to reach READY status. + + ``merge_strategy`` is optional (the Branch field is nullable) — only tests that + actually merge or revert need it; read-only tests (e.g. GraphQL, which can never + write, merge, or sync) leave it ``None``. + """ if timeout is None: timeout = BRANCH_PROVISION_TIMEOUT branch = Branch(name=name, merge_strategy=merge_strategy) @@ -2797,14 +2820,13 @@ def test_sync_then_branch_edit_then_merge_lifecycle(self): @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') class GraphQLBranchIsolationTestCase(BranchingTestBase, TransactionTestCase): """ - GraphQL is main-only: GraphiQL has no concept of branches, so the schema - (and the data it returns) must always reflect the main database. Custom - object type changes made inside a branch must never appear in the GraphQL - schema, regardless of which branch context a request happens to run in. + GraphQL resolves against main unless a request explicitly selects a branch with + the X-NetBox-Branch header. An implicit UI branch (cookie / ``?_branch=``) must + never leak into GraphQL, but a request that does carry the header gets that + branch's schema and data. These tests cover both halves: the header-only branch + gating, and the schema reflecting whichever branch is active. """ - MERGE_STRATEGY = 'iterative' - def setUp(self): super().setUp() from netbox_custom_objects.graphql import live as live_module @@ -2827,7 +2849,7 @@ def test_schema_signature_ignores_branch_changes(self): # signature does. main_sig = self.live.schema_signature() - branch = _provision_branch('GraphQL Iso Sig', self.MERGE_STRATEGY, self.user) + branch = _provision_branch('GraphQL Iso Sig', user=self.user) branch_request = _make_request(self.user) with activate_branch(branch), event_tracking(branch_request): cot = CustomObjectType.objects.create( @@ -2839,18 +2861,19 @@ def test_schema_signature_ignores_branch_changes(self): ) # Raw branch-context signature sees the new COT... self.assertNotEqual(main_sig, self.live.schema_signature()) - # ...but forced to main (as get_live_schema does) it does not. + # ...but forced to main (as the no-header GraphQL path does) it does not. with self.live.main_branch_context(): self.assertEqual(main_sig, self.live.schema_signature()) - def test_live_schema_excludes_cot_created_in_branch(self): - # The assembled schema must never gain a branch-only type — neither while - # the branch is active nor afterwards in main (the branch isn't merged). + def test_live_schema_reflects_active_branch(self): + # With a branch active (as the X-NetBox-Branch header path leaves it), the + # schema reflects that branch's custom object types; main, unaffected by the + # unmerged branch, does not. baseline = self.live.get_live_schema() self.assertIsNotNone(baseline) self.assertNotIn('custom_objects_branch_only', str(baseline)) - branch = _provision_branch('GraphQL Iso Schema', self.MERGE_STRATEGY, self.user) + branch = _provision_branch('GraphQL Branch Schema', user=self.user) branch_request = _make_request(self.user) with activate_branch(branch), event_tracking(branch_request): cot = CustomObjectType.objects.create( @@ -2860,10 +2883,174 @@ def test_live_schema_excludes_cot_created_in_branch(self): custom_object_type=cot, name='name', label='Name', type='text', primary=True, required=True, ) - self.assertNotIn( + # The branch's own schema gains the branch-only type... + self.assertIn( 'custom_objects_branch_only', str(self.live.get_live_schema()) ) + # ...but main never does (the branch isn't merged). self.assertNotIn( 'custom_objects_branch_only', str(self.live.get_live_schema()) ) + + def test_graphql_branch_context_honors_only_header(self): + # A GraphQL request honours a branch only when the X-NetBox-Branch header is + # present; an implicit branch (e.g. a GraphiQL cookie, modelled here by an + # already-active branch with no header) is forced back to main. + from netbox_branching.constants import BRANCH_HEADER + from netbox_branching.contextvars import active_branch + + branch = _provision_branch('GraphQL Ctx', user=self.user) + + # Header present → the active branch is honoured. + request = _make_request(self.user) + request.headers = {BRANCH_HEADER: branch.schema_id} + with activate_branch(branch): + with self.live.graphql_branch_context(request): + self.assertEqual(active_branch.get(), branch) + + # No header → forced to main even though a branch is active. + request_no_header = _make_request(self.user) + request_no_header.headers = {} + with activate_branch(branch): + with self.live.graphql_branch_context(request_no_header): + self.assertIsNone(active_branch.get()) + + def test_branch_deletion_evicts_cached_schema(self): + # Building a branch's schema caches it under the branch pk; deleting the + # branch must evict that entry so it doesn't leak for the process lifetime. + branch = _provision_branch('GraphQL Evict', user=self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + cot = CustomObjectType.objects.create(name='evict me', slug='evict-me') + CustomObjectTypeField.objects.create( + custom_object_type=cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + self.assertIsNotNone(self.live.get_live_schema()) + + # The branch's schema is now cached under its pk... + self.assertIn(branch.pk, self.live._schema_cache) + + # ...and deleting the branch evicts it. + branch_pk = branch.pk + branch.delete() + self.assertNotIn(branch_pk, self.live._schema_cache) + + +@unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') +@override_settings(LOGIN_REQUIRED=True) +class GraphQLBranchEndpointTestCase(BranchingTestBase, TransactionTestCase): + """ + End-to-end against the real ``/graphql/`` endpoint: it serves main by default + and the branch named by the ``X-NetBox-Branch`` header otherwise — correct + schema AND data in each — and a schema change made inside a branch never leaks + into main (the branch is not merged). + """ + + def setUp(self): + super().setUp() + from netbox_custom_objects.graphql import live as live_module + self.live = live_module + live_module.reset_cache() + self.addCleanup(live_module.reset_cache) + # The startup guard returns True during the test run; patch it off so the + # live schema machinery runs exactly as it does in production. + patcher = mock.patch( + 'netbox_custom_objects.CustomObjectsPluginConfig.' + 'should_skip_dynamic_model_creation', + return_value=False, + ) + patcher.start() + self.addCleanup(patcher.stop) + + # Superuser + token auth, mirroring how an API client reaches the endpoint. + self.user.is_superuser = True + self.user.save() + self.client = APIClient() + token_key = _create_token(self.user) + self.header = {'HTTP_AUTHORIZATION': f'Token {token_key}'} + self.url = reverse('graphql') + + def _post(self, query, branch=None): + headers = dict(self.header) + if branch is not None: + # Django maps HTTP_X_NETBOX_BRANCH → the X-NetBox-Branch request header + # netbox-branching reads to activate the branch for an API request. + headers['HTTP_X_NETBOX_BRANCH'] = branch.schema_id + response = self.client.post( + self.url, data={'query': query}, format='json', **headers + ) + return json.loads(response.content) + + def _data(self, query, branch=None): + payload = self._post(query, branch=branch) + self.assertNotIn('errors', payload, msg=str(payload.get('errors'))) + return payload['data'] + + def _assert_query_rejected(self, query, branch=None): + # A field/type absent from the active schema is a GraphQL validation error. + payload = self._post(query, branch=branch) + self.assertIn('errors', payload, msg=f'expected schema to reject query: {payload}') + + def test_main_and_branch_isolated_schema_and_data(self): + # --- main: COT 'server' (field 'name') with one instance --- + server = CustomObjectType.objects.create(name='server', slug='server') + CustomObjectTypeField.objects.create( + custom_object_type=server, name='name', label='Name', type='text', + primary=True, required=True, + ) + server.get_model().objects.create(name='main-server') + + # --- branch: a branch-only COT 'widget', plus a NEW field 'note' added to + # the existing 'server' COT and a branch-only server row that uses it --- + branch = _provision_branch('GraphQL E2E', user=self.user) + branch_request = _make_request(self.user) + with activate_branch(branch), event_tracking(branch_request): + widget = CustomObjectType.objects.create(name='widget', slug='widget') + CustomObjectTypeField.objects.create( + custom_object_type=widget, name='name', label='Name', type='text', + primary=True, required=True, + ) + widget.get_model().objects.create(name='branch-widget') + + server_in_branch = CustomObjectType.objects.get(pk=server.pk) + CustomObjectTypeField.objects.create( + custom_object_type=server_in_branch, name='note', label='Note', + type='text', + ) + server_in_branch.get_model().objects.create(name='branch-server', note='hi') + + self.live.reset_cache() + + # --- MAIN (no header): only main's schema and data --- + main = self._data('{ custom_objects_server_list { name } }') + self.assertEqual( + [r['name'] for r in main['custom_objects_server_list']], ['main-server'], + 'main must not see the branch-only server row', + ) + # The branch-added 'note' field is absent from main's schema. + self._assert_query_rejected('{ custom_objects_server_list { name note } }') + # The branch-only 'widget' COT is absent from main's schema. + self._assert_query_rejected('{ custom_objects_widget_list { name } }') + + # --- BRANCH (header): branch schema (note + widget) and branch data --- + data = self._data( + '{ custom_objects_server_list { name note } ' + 'custom_objects_widget_list { name } }', + branch=branch, + ) + servers = {r['name']: r['note'] for r in data['custom_objects_server_list']} + # The branch is a copy of main plus its own change: both rows are visible, + # and only the branch row carries the branch-only 'note' value. + self.assertIn('main-server', servers) + self.assertEqual(servers.get('branch-server'), 'hi') + self.assertEqual( + [r['name'] for r in data['custom_objects_widget_list']], ['branch-widget'], + ) + + # --- MAIN again: still unchanged by the unmerged branch --- + main_again = self._data('{ custom_objects_server_list { name } }') + self.assertEqual( + [r['name'] for r in main_again['custom_objects_server_list']], ['main-server'], + ) From 1d2695505c2e5f619cc4b4b976e2d743a6daeaa3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 11:18:05 -0700 Subject: [PATCH 079/115] branching fix --- netbox_custom_objects/__init__.py | 30 +++---- netbox_custom_objects/graphql/live.py | 89 ++++-------------- netbox_custom_objects/tests/test_branching.py | 90 +++++++++---------- 3 files changed, 72 insertions(+), 137 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 45f81eb4..c5ccf2b4 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -227,22 +227,20 @@ def _patch_graphql_view(): @csrf_exempt def _patched_dispatch(self, request, *args, **kwargs): - from netbox_custom_objects.graphql.live import get_live_schema, graphql_branch_context - - # GraphQL resolves against main unless the request explicitly selects a - # branch with the X-NetBox-Branch header (GraphiQL has no branch concept, so - # an implicit UI branch via cookie/query-param must not leak in). - # graphql_branch_context scopes the active branch accordingly, and - # get_live_schema builds the schema for that same branch, so the schema and - # the data the query returns stay consistent. - with graphql_branch_context(request): - try: - live_schema = get_live_schema() - if live_schema is not None: - self.schema = live_schema - except Exception: # noqa: BLE001 - fall back to the static schema - logger.warning("Failed to load live GraphQL schema; using static schema", exc_info=True) - return _original_dispatch(self, request, *args, **kwargs) + from netbox_custom_objects.graphql.live import get_live_schema + + # netbox-branching has already activated the request's branch (from the + # X-NetBox-Branch header, ?_branch=, or the active_branch cookie) — the plugin + # imposes no GraphQL-specific branch policy. get_live_schema builds the + # schema for whichever branch is active, so the schema and the data the query + # returns agree, and core models resolve exactly as they do without the plugin. + try: + live_schema = get_live_schema() + if live_schema is not None: + self.schema = live_schema + except Exception: # noqa: BLE001 - fall back to the static schema + logger.warning("Failed to load live GraphQL schema; using static schema", exc_info=True) + return _original_dispatch(self, request, *args, **kwargs) NetBoxGraphQLView.dispatch = _patched_dispatch _graphql_view_patched = True diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index b7cad8af..981c5f65 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -20,20 +20,19 @@ visible to every process on its next request — no restart, no cross-process messaging. -By default GraphQL resolves against **main**: GraphiQL has no concept of branches, -so an implicit UI branch (a branch cookie or ``?_branch=`` query param) must not -leak into it. A client may opt a single request into a branch with the -``X-NetBox-Branch`` header (the REST/GraphQL API convention — see -``netbox_branching.utilities.get_active_branch``); that request then gets that -branch's schema *and* data. :func:`graphql_branch_context` enforces this at the -view, and the schema cache is keyed per branch so each branch's custom object types -are reflected independently of main. +GraphQL resolves against whatever branch netbox-branching activated for the +request — the ``X-NetBox-Branch`` header, the ``?_branch=`` query param, or the +``active_branch`` cookie — exactly like the REST API and the rest of the UI (the +plugin imposes no GraphQL-specific branch policy of its own); with no branch active +it is main. The schema cache is keyed per branch (see :func:`_active_branch_key`) +so each branch's custom object types are reflected independently, and +:func:`get_live_schema` builds the schema for whichever branch is active so the +schema and the data the query returns always agree. The view patch in ``__init__.py`` calls :func:`get_live_schema` per request and assigns the result to the view before it executes the operation. """ -import contextlib import logging import threading @@ -45,41 +44,16 @@ # stale) schema instead of blocking. _rebuild_lock = threading.Lock() # Per-branch cache of (signature, schema), keyed by branch identifier (None = main). -# GraphQL honours a branch only when a request explicitly selects one via the -# X-NetBox-Branch header (see graphql_branch_context); each branch gets its own -# schema reflecting that branch's custom object types. Each value is an atomically -# swapped (signature, schema) tuple, so a lockless reader can never see a schema -# that doesn't match its signature. +# GraphQL reflects whichever branch netbox-branching activated for the request, so +# each branch gets its own schema reflecting that branch's custom object types. Each +# value is an atomically swapped (signature, schema) tuple, so a lockless reader can +# never see a schema that doesn't match its signature. _schema_cache = {} # Signature cache keys this process has populated, so reset_cache (tests) can clear # every per-branch entry. _signature_keys_seen = set() -@contextlib.contextmanager -def main_branch_context(): - """ - Run the enclosed block against the main database, ignoring any active branch. - - GraphQL is main-only: both its schema and the data it returns always reflect - main, never a branch. The signature check and the rebuild must not see a - branch's custom object types (a COT created or edited inside a branch must never - change the schema), and the request's data resolution must likewise read main. - Delegates to netbox-branching's own ``deactivate_branch`` context manager - (``activate_branch(None)``) so the meaning of "main" stays owned upstream - rather than reimplemented here. A no-op when netbox-branching is not - installed. - """ - try: - from netbox_branching.utilities import deactivate_branch - except ImportError: - yield - return - - with deactivate_branch(): - yield - - def _active_branch_key(): """ Identifier for the active branch (``None`` for main), used to key the per-branch @@ -93,37 +67,6 @@ def _active_branch_key(): return branch.pk if branch is not None else None -@contextlib.contextmanager -def graphql_branch_context(request): - """ - Scope a GraphQL request to a branch only when one is explicitly requested via - the ``X-NetBox-Branch`` header (the REST/GraphQL API convention — see - ``netbox_branching.utilities.get_active_branch``). - - Browser GraphiQL has no concept of branches, but a UI session may carry a branch - cookie or ``?_branch=`` query param that netbox-branching's middleware would - activate for the request. Letting that leak into GraphQL would silently resolve - the schema and data against a branch the GraphiQL user never chose, so without - the header we force main. With the header, the middleware has already activated - the branch and we leave it active for both the schema rebuild and the query's - data resolution. A no-op when netbox-branching is not installed. - """ - try: - from netbox_branching.constants import BRANCH_HEADER - except ImportError: - yield - return - - headers = getattr(request, "headers", None) or {} - if BRANCH_HEADER in headers: - # Explicit branch selection — honour the active branch the middleware set. - yield - else: - # No explicit selection: force main so an implicit UI branch can't leak in. - with main_branch_context(): - yield - - def schema_signature(): """ Return a cheap, comparable fingerprint of the custom object type schema. @@ -329,10 +272,10 @@ def get_live_schema(): Return the schema for the current request's active branch, rebuilding it if the database has changed since this process last built it for that branch. - The caller (the view patch, via :func:`graphql_branch_context`) has already - scoped the active branch: main unless the request carried the X-NetBox-Branch - header. The signature check, the rebuild, and the query's data resolution - therefore all run against that same branch. + netbox-branching has already scoped the active branch for the request (from the + X-NetBox-Branch header, the ``?_branch=`` query param, or the active_branch + cookie; main if none). The signature check, the rebuild, and the query's data + resolution therefore all run against that same branch. Returns ``None`` when dynamic models are unavailable (migrations/tests) or if the very first build fails — the caller then falls back to NetBox's static diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index b744df32..db6ca51b 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -2820,11 +2820,11 @@ def test_sync_then_branch_edit_then_merge_lifecycle(self): @unittest.skipUnless(HAS_BRANCHING, 'netbox-branching is not installed') class GraphQLBranchIsolationTestCase(BranchingTestBase, TransactionTestCase): """ - GraphQL resolves against main unless a request explicitly selects a branch with - the X-NetBox-Branch header. An implicit UI branch (cookie / ``?_branch=``) must - never leak into GraphQL, but a request that does carry the header gets that - branch's schema and data. These tests cover both halves: the header-only branch - gating, and the schema reflecting whichever branch is active. + GraphQL resolves against whichever branch netbox-branching activated for the + request (X-NetBox-Branch header, ``?_branch=``, or the active_branch cookie), + exactly like the REST API and the UI; with no branch active it is main. The + schema is built per-branch, so a branch's custom object types appear only for + requests scoped to that branch and never leak into main. """ def setUp(self): @@ -2843,13 +2843,13 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) - def test_schema_signature_ignores_branch_changes(self): - # The signature drives rebuilds; forced to main it must not move when a - # COT is created inside a branch, even though the raw branch-context - # signature does. + def test_schema_signature_is_branch_aware(self): + # The signature drives rebuilds; computed under a branch it must see that + # branch's custom object types (so the branch gets its own schema), while + # main — unaffected by the unmerged branch — keeps its own. main_sig = self.live.schema_signature() - branch = _provision_branch('GraphQL Iso Sig', user=self.user) + branch = _provision_branch('GraphQL Sig Branch', user=self.user) branch_request = _make_request(self.user) with activate_branch(branch), event_tracking(branch_request): cot = CustomObjectType.objects.create( @@ -2859,11 +2859,10 @@ def test_schema_signature_ignores_branch_changes(self): custom_object_type=cot, name='name', label='Name', type='text', primary=True, required=True, ) - # Raw branch-context signature sees the new COT... self.assertNotEqual(main_sig, self.live.schema_signature()) - # ...but forced to main (as the no-header GraphQL path does) it does not. - with self.live.main_branch_context(): - self.assertEqual(main_sig, self.live.schema_signature()) + + # Back on main, the signature is unchanged by the unmerged branch. + self.assertEqual(main_sig, self.live.schema_signature()) def test_live_schema_reflects_active_branch(self): # With a branch active (as the X-NetBox-Branch header path leaves it), the @@ -2893,29 +2892,6 @@ def test_live_schema_reflects_active_branch(self): 'custom_objects_branch_only', str(self.live.get_live_schema()) ) - def test_graphql_branch_context_honors_only_header(self): - # A GraphQL request honours a branch only when the X-NetBox-Branch header is - # present; an implicit branch (e.g. a GraphiQL cookie, modelled here by an - # already-active branch with no header) is forced back to main. - from netbox_branching.constants import BRANCH_HEADER - from netbox_branching.contextvars import active_branch - - branch = _provision_branch('GraphQL Ctx', user=self.user) - - # Header present → the active branch is honoured. - request = _make_request(self.user) - request.headers = {BRANCH_HEADER: branch.schema_id} - with activate_branch(branch): - with self.live.graphql_branch_context(request): - self.assertEqual(active_branch.get(), branch) - - # No header → forced to main even though a branch is active. - request_no_header = _make_request(self.user) - request_no_header.headers = {} - with activate_branch(branch): - with self.live.graphql_branch_context(request_no_header): - self.assertIsNone(active_branch.get()) - def test_branch_deletion_evicts_cached_schema(self): # Building a branch's schema caches it under the branch pk; deleting the # branch must evict that entry so it doesn't leak for the process lifetime. @@ -2942,10 +2918,11 @@ def test_branch_deletion_evicts_cached_schema(self): @override_settings(LOGIN_REQUIRED=True) class GraphQLBranchEndpointTestCase(BranchingTestBase, TransactionTestCase): """ - End-to-end against the real ``/graphql/`` endpoint: it serves main by default - and the branch named by the ``X-NetBox-Branch`` header otherwise — correct - schema AND data in each — and a schema change made inside a branch never leaks - into main (the branch is not merged). + End-to-end against the real ``/graphql/`` endpoint: it serves whichever branch + netbox-branching activated for the request (the ``X-NetBox-Branch`` header or the + ``?_branch=`` query param) and main otherwise — correct schema AND data in each — + and a schema change made inside a branch never leaks into main (the branch is not + merged). """ def setUp(self): @@ -2972,19 +2949,25 @@ def setUp(self): self.header = {'HTTP_AUTHORIZATION': f'Token {token_key}'} self.url = reverse('graphql') - def _post(self, query, branch=None): + def _post(self, query, branch=None, via='header'): headers = dict(self.header) + url = self.url if branch is not None: - # Django maps HTTP_X_NETBOX_BRANCH → the X-NetBox-Branch request header - # netbox-branching reads to activate the branch for an API request. - headers['HTTP_X_NETBOX_BRANCH'] = branch.schema_id + if via == 'header': + # Django maps HTTP_X_NETBOX_BRANCH → the X-NetBox-Branch request + # header netbox-branching reads to activate the branch. + headers['HTTP_X_NETBOX_BRANCH'] = branch.schema_id + elif via == 'query': + # The ?_branch= query param the UI uses — another branch source + # netbox-branching honours for any request. + url = f'{self.url}?_branch={branch.schema_id}' response = self.client.post( - self.url, data={'query': query}, format='json', **headers + url, data={'query': query}, format='json', **headers ) return json.loads(response.content) - def _data(self, query, branch=None): - payload = self._post(query, branch=branch) + def _data(self, query, branch=None, via='header'): + payload = self._post(query, branch=branch, via=via) self.assertNotIn('errors', payload, msg=str(payload.get('errors'))) return payload['data'] @@ -3049,6 +3032,17 @@ def test_main_and_branch_isolated_schema_and_data(self): [r['name'] for r in data['custom_objects_widget_list']], ['branch-widget'], ) + # --- BRANCH via ?_branch= query param (no header): the UI's branch source + # selects the same branch. GraphQL honours every branch source NetBox does, + # not just the header — if it didn't, this would fall back to main and reject + # the branch-only 'widget' field. --- + via_param = self._data( + '{ custom_objects_widget_list { name } }', branch=branch, via='query', + ) + self.assertEqual( + [r['name'] for r in via_param['custom_objects_widget_list']], ['branch-widget'], + ) + # --- MAIN again: still unchanged by the unmerged branch --- main_again = self._data('{ custom_objects_server_list { name } }') self.assertEqual( From 7c687d7c5e6791f7d9969919a0d75ba5b1b5392b Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 11:52:30 -0700 Subject: [PATCH 080/115] cleanup --- netbox_custom_objects/__init__.py | 13 ++-- netbox_custom_objects/graphql/schema.py | 24 ++++++- netbox_custom_objects/graphql/types.py | 26 ++++++- netbox_custom_objects/tests/base.py | 19 ++++++ netbox_custom_objects/tests/test_api.py | 18 +---- netbox_custom_objects/tests/test_branching.py | 24 ++----- netbox_custom_objects/tests/test_graphql.py | 67 ++++++++++++++----- .../tests/test_polymorphic_fields.py | 17 ++--- 8 files changed, 137 insertions(+), 71 deletions(-) diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index c5ccf2b4..31e9e578 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -267,11 +267,14 @@ class CustomObjectsPluginConfig(PluginConfig): # Resolves dynamic CO models (table{n}model) to on-the-fly serializers — # they have no importable path at the conventional location. serializer_resolver = "api.serializers.serializer_resolver" - # Contributes a GraphQL Query (one field pair per custom object type) to - # NetBox's schema. Built at startup from the existing types; see - # graphql/types.py for the restart-on-new-type caveat. The path resolves as - # module ``graphql.schema`` + attribute ``schema`` (NetBox loads resources - # via ``import_string``, which treats the final component as an attribute). + # Registers the plugin's GraphQL schema contribution with NetBox. The export + # is intentionally an empty list: custom object types are created/deleted at + # runtime, so a query class built once at startup would be immediately stale. + # The live, per-request schema is served instead by the GraphQL view patch in + # ``_patch_graphql_view`` (see the graphql/schema.py and graphql/live.py module + # docstrings). The path resolves as module ``graphql.schema`` + attribute + # ``schema`` (NetBox loads resources via ``import_string``, which treats the + # final component as an attribute). graphql_schema = "graphql.schema.schema" @staticmethod diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 61f86c7c..9b9fd7d4 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -22,7 +22,7 @@ import strawberry import strawberry_django -from .types import build_object_type, clear_type_cache, graphql_safe_name, reset_build_state +from .types import build_object_type, clear_type_cache, graphql_safe_name, reset_build_state, set_cot_map logger = logging.getLogger("netbox_custom_objects.graphql") @@ -96,7 +96,9 @@ class is avoided. if CustomObjectsPluginConfig.should_skip_dynamic_model_creation(): return [] - from netbox_custom_objects.models import CustomObjectType + from django.db.models import Prefetch + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField # Start each rebuild from an empty per-type cache. The cache is keyed by # (cot id, cache_timestamp), but a type that embeds another COT's type via a @@ -112,11 +114,27 @@ class is avoided. reset_build_state() try: - custom_object_types = list(CustomObjectType.objects.all()) + # Preload every type's fields once, with each field's related type(s) + # joined/prefetched, so the per-type build issues no per-field queries: + # without this the inner loop hits the DB for each type's fields and again + # for each field's related object type(s) — O(N + N·M) queries per rebuild. + fields_qs = ( + CustomObjectTypeField.objects + .select_related("related_object_type") + .prefetch_related("related_object_types") + ) + custom_object_types = list( + CustomObjectType.objects.prefetch_related(Prefetch("fields", queryset=fields_qs)) + ) except Exception: # noqa: BLE001 - DB may be unavailable at import time logger.debug("Could not load custom object types for GraphQL schema", exc_info=True) return [] + # Register the preloaded types by pk so a relationship field pointing at another + # custom object resolves its target from the prefetched instance instead of + # re-querying it (build_object_type then reuses the per-rebuild type cache). + set_cot_map({cot.pk: cot for cot in custom_object_types}) + annotations = {} attrs = {} used_names = set() diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index 6aa0a2a1..06d7f251 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -49,6 +49,7 @@ "clear_type_cache", "graphql_safe_name", "reset_build_state", + "set_cot_map", ) # Per-rebuild memoization of built GraphQL types, keyed by (cot id, @@ -94,10 +95,25 @@ def reset_build_state(): (pooled) thread cannot suppress caching or corrupt cycle detection on the next one. ``build_object_type`` only clears a type's taint on the success path, so a build that raises after a cyclic edge tainted an ancestor would - otherwise leave that taint set on the thread indefinitely. + otherwise leave that taint set on the thread indefinitely. Also drops the + preloaded COT map (see :func:`set_cot_map`). """ _building.cot_stack = [] _building.cycle_tainted = set() + _building.cot_map = None + + +def set_cot_map(cot_map): + """ + Register a ``{pk: CustomObjectType}`` map for the current rebuild. + + ``build_query_classes`` preloads every custom object type once with its fields + (and their related types) prefetched, then registers them here so a relationship + field pointing at another custom object resolves its target from the prefetched + instance — via :func:`_custom_object_graphql_type` — instead of issuing a fresh + query per reference. Cleared by :func:`reset_build_state`. + """ + _building.cot_map = cot_map RELATIONSHIP_TYPES = ( @@ -299,7 +315,13 @@ def _custom_object_graphql_type(model_name): # (A -> B -> C -> A), permanently freezing the stub into their subtree. _cycle_tainted_set().update(stack) return None - cot = CustomObjectType.objects.filter(pk=cot_id).first() + # Resolve from the rebuild's preloaded (prefetched) map when available, so a + # cross-COT reference doesn't issue a query per edge; fall back to a lookup for + # direct callers (e.g. tests) that didn't register a map. + cot_map = getattr(_building, "cot_map", None) + cot = cot_map.get(cot_id) if cot_map else None + if cot is None: + cot = CustomObjectType.objects.filter(pk=cot_id).first() if cot is None: return None try: diff --git a/netbox_custom_objects/tests/base.py b/netbox_custom_objects/tests/base.py index f4e6cb5d..a7dc89f8 100644 --- a/netbox_custom_objects/tests/base.py +++ b/netbox_custom_objects/tests/base.py @@ -20,6 +20,25 @@ logger = logging.getLogger(__name__) +def create_token(user): + """Create an API token for ``user`` and return its plaintext key. + + Handles the NetBox 4.5 token-versioning change (V1 tokens expose ``.token``; + older NetBox exposes ``.key``). + """ + try: + # NetBox >= 4.5 + from users.choices import TokenVersionChoices + token = Token(version=TokenVersionChoices.V1, user=user) + token.save() + return token.token + except ImportError: + # NetBox < 4.5 + token = Token(user=user) + token.save() + return token.key + + def _recreate_contenttypes(): """Recreate ContentType rows for all installed apps using get_or_create. diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index 3757e7aa..f96bb179 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -10,27 +10,13 @@ from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField from netbox_custom_objects.template_content import CustomObjectLink, LinkedCustomObject -from .base import CustomObjectsTestCase +from .base import CustomObjectsTestCase, create_token from core.models import ObjectType from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, Site -from users.models import ObjectPermission, Token +from users.models import ObjectPermission from virtualization.models import Cluster, ClusterType -def create_token(user): - try: - # NetBox >= 4.5 - from users.choices import TokenVersionChoices - token = Token(version=TokenVersionChoices.V1, user=user) - token.save() - return token.token - except ImportError: - # NetBox < 4.5 - token = Token(user=user) - token.save() - return token.key - - class CustomObjectAPITestCaseMixin: """ Base test class for custom object API endpoints. diff --git a/netbox_custom_objects/tests/test_branching.py b/netbox_custom_objects/tests/test_branching.py index db6ca51b..f8c8efc3 100644 --- a/netbox_custom_objects/tests/test_branching.py +++ b/netbox_custom_objects/tests/test_branching.py @@ -28,7 +28,6 @@ from django.urls import reverse from extras.models import CustomFieldChoiceSet from rest_framework.test import APIClient -from users.models import Token try: from netbox.context_managers import event_tracking @@ -40,7 +39,11 @@ HAS_BRANCHING = False from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField -from netbox_custom_objects.tests.base import TransactionCleanupMixin, _recreate_contenttypes +from netbox_custom_objects.tests.base import ( + TransactionCleanupMixin, + _recreate_contenttypes, + create_token, +) logger = logging.getLogger(__name__) User = get_user_model() @@ -54,21 +57,6 @@ def _make_request(user): return request -def _create_token(user): - """Create an API token (plaintext key) across NetBox token versions.""" - try: - # NetBox >= 4.5 - from users.choices import TokenVersionChoices - token = Token(version=TokenVersionChoices.V1, user=user) - token.save() - return token.token - except ImportError: - # NetBox < 4.5 - token = Token(user=user) - token.save() - return token.key - - # Provisioning timeout for branch tests. Override via the # ``NETBOX_CO_BRANCH_PROVISION_TIMEOUT`` env var (seconds) when CI flakes. BRANCH_PROVISION_TIMEOUT = float( @@ -2945,7 +2933,7 @@ def setUp(self): self.user.is_superuser = True self.user.save() self.client = APIClient() - token_key = _create_token(self.user) + token_key = create_token(self.user) self.header = {'HTTP_AUTHORIZATION': f'Token {token_key}'} self.url = reverse('graphql') diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index 31f06755..a7b141f4 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -26,29 +26,14 @@ from core.models import ObjectType from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Region, Site -from users.models import ObjectPermission, Token +from users.models import ObjectPermission from netbox_custom_objects.graphql import live as live_module from netbox_custom_objects.graphql import schema as schema_module from netbox_custom_objects.graphql.schema import build_query_classes, _query_field_name from netbox_custom_objects.graphql.types import build_object_type -from .base import CustomObjectsTestCase - - -def create_token(user): - """Create an API token, plaintext key, across NetBox token versions.""" - try: - # NetBox >= 4.5 - from users.choices import TokenVersionChoices - token = Token(version=TokenVersionChoices.V1, user=user) - token.save() - return token.token - except ImportError: - # NetBox < 4.5 - token = Token(user=user) - token.save() - return token.key +from .base import CustomObjectsTestCase, create_token class GraphQLSchemaGenerationTestCase(CustomObjectsTestCase, TestCase): @@ -172,6 +157,54 @@ def test_self_referential_object_field_does_not_recurse(self): # intentionally not cached — a later top-level query rebuilds it. self.assertIsNot(gql_type, build_object_type(cot)) + def test_rebuild_prefetch_covers_field_access(self): + # Regression for the schema-rebuild N+1: build_query_classes preloads types + # with their fields and each field's related type(s). Loading a type via + # that same prefetch must make the rebuild's per-field accesses + # (types.py: the fields iteration, the FK target, and the polymorphic M2M + # targets) issue zero further queries. + from django.db import connection + from django.db.models import Prefetch + from django.test.utils import CaptureQueriesContext + + from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField + + cot = self.create_custom_object_type(name="Pf", slug="pf") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + # Non-polymorphic object field → exercises the related_object_type FK. + self.create_custom_object_type_field( + cot, name="single", label="Single", type="object", + related_object_type=self.get_site_object_type(), + ) + # Polymorphic object field → exercises the related_object_types M2M. + self.create_polymorphic_field( + cot, [self.get_site_object_type(), self.get_device_object_type()], + name="poly", type="object", + ) + + # The exact prefetch build_query_classes uses. + fields_qs = ( + CustomObjectTypeField.objects + .select_related("related_object_type") + .prefetch_related("related_object_types") + ) + loaded = CustomObjectType.objects.prefetch_related( + Prefetch("fields", queryset=fields_qs) + ).get(pk=cot.pk) + + with CaptureQueriesContext(connection) as ctx: + for field in loaded.fields.all(): # fields iteration (no query) + if field.related_object_type_id: + _ = field.related_object_type # FK — select_related + if field.is_polymorphic: + list(field.related_object_types.all()) # M2M — prefetch_related + self.assertEqual( + len(ctx.captured_queries), 0, + f"prefetch missed an access: {[q['sql'] for q in ctx.captured_queries]}", + ) + class GraphQLLiveSchemaTestCase(CustomObjectsTestCase, TestCase): """ diff --git a/netbox_custom_objects/tests/test_polymorphic_fields.py b/netbox_custom_objects/tests/test_polymorphic_fields.py index 47daed83..85e81b1b 100644 --- a/netbox_custom_objects/tests/test_polymorphic_fields.py +++ b/netbox_custom_objects/tests/test_polymorphic_fields.py @@ -16,24 +16,21 @@ from dcim.models import Site from ipam.models import Prefix, IPAddress from ipam.choices import PrefixStatusChoices -from users.models import ObjectPermission, Token +from users.models import ObjectPermission from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.models import CustomObjectType, CustomObjectTypeField -from netbox_custom_objects.tests.base import CustomObjectsTestCase, TransactionCleanupMixin +from netbox_custom_objects.tests.base import ( + CustomObjectsTestCase, + TransactionCleanupMixin, + create_token, +) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _create_token(user): - from users.choices import TokenVersionChoices - t = Token(version=TokenVersionChoices.V1, user=user) - t.save() - return t.token # plaintext for V1 tokens - - def _grant_perm(user, action, model_class, name=None): perm = ObjectPermission(name=name or f"poly-test-{action}", actions=[action]) perm.save() @@ -57,7 +54,7 @@ def setUp(self): from django.test import Client as DjangoClient from utilities.testing import create_test_user self.user = create_test_user("poly-api-user") - token_key = _create_token(self.user) + token_key = create_token(self.user) self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} # Reset client to clear the session cookie set by CustomObjectsTestCase.setUp() # (force_login causes SessionAuthentication to take priority over TokenAuthentication) From b4d6691a85224551b1a9798028bc795c7c24e044 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 13:20:24 -0700 Subject: [PATCH 081/115] fix perms --- netbox_custom_objects/graphql/types.py | 7 ++++++- netbox_custom_objects/tests/test_graphql.py | 23 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index 06d7f251..3af945b1 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -196,9 +196,14 @@ def _filter_viewable(user, objects): than one ``.exists()`` per object (an N+1 explosion on multi-object fields): the related objects are grouped by model and each model's permission-restricted queryset is evaluated once with ``pk__in``. + + The user (anonymous or ``None`` included) is passed straight to the model + manager's ``restrict(user, "view")``, mirroring NetBox's + ``BaseObjectType.get_queryset`` — superuser bypass, ``EXEMPT_VIEW_PERMISSIONS`` + and anonymous handling are all left to ``restrict``. """ objects = [obj for obj in objects if obj is not None] - if user is None or not objects: + if not objects: return [] if getattr(user, "is_superuser", False): return objects diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index a7b141f4..866f1801 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -571,3 +571,26 @@ def test_multiobject_related_filtered_by_permission(self): devices = payload["data"]["custom_objects_group_list"][0]["devices"] self.assertEqual(len(devices), 1) self.assertEqual(devices[0]["id"], str(device.pk)) + + @override_settings(EXEMPT_VIEW_PERMISSIONS=["*"]) + def test_related_anonymous_access_honors_exempt_view_permissions(self): + # Anonymous/None traversal must defer to restrict(), exactly like NetBox's + # BaseObjectType.get_queryset: when the view is exempt, related objects are + # visible, not silently denied. + from django.contrib.auth.models import AnonymousUser + + from netbox_custom_objects.graphql.types import _filter_viewable + + self.assertEqual(_filter_viewable(AnonymousUser(), [self.site]), [self.site]) + self.assertEqual(_filter_viewable(None, [self.site]), [self.site]) + + @override_settings(EXEMPT_VIEW_PERMISSIONS=[]) + def test_related_anonymous_access_denied_without_exemption(self): + # And when the view is not exempt, an unauthenticated user sees nothing — + # again matching restrict()'s anonymous handling. + from django.contrib.auth.models import AnonymousUser + + from netbox_custom_objects.graphql.types import _filter_viewable + + self.assertEqual(_filter_viewable(AnonymousUser(), [self.site]), []) + self.assertEqual(_filter_viewable(None, [self.site]), []) From 6230cb05469aae30a99890f78801b248bbeb8cde Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 13:24:17 -0700 Subject: [PATCH 082/115] clarify config cache --- netbox_custom_objects/graphql/live.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index 981c5f65..27d3dfa3 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -263,6 +263,14 @@ def build_full_schema(): return strawberry.Schema( query=query_cls, config=config, + # Built fresh per rebuild on purpose — unlike ``config`` (a read-only + # settings object), get_schema_extensions() returns extension *instances* + # that strawberry mutates per request (``extension.execution_context = …``). + # Each Schema must own its own set; sharing one set across our several live + # schemas (main + per branch, plus an old schema still serving in-flight + # requests during a rebuild) would let concurrent requests on different + # schemas clobber each other's execution context. This mirrors what NetBox + # gets for its own schema, and rebuilds are rare, so the cost is irrelevant. extensions=ngs.get_schema_extensions(), ) From b3eb4e96b53ebff196716193d53cf61dc05dd921 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 9 Jun 2026 13:38:21 -0700 Subject: [PATCH 083/115] fixes --- netbox_custom_objects/graphql/live.py | 22 +++++++++++++++---- netbox_custom_objects/graphql/schema.py | 3 +++ netbox_custom_objects/graphql/types.py | 9 +++++++- netbox_custom_objects/tests/test_graphql.py | 24 +++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index 27d3dfa3..5b934a01 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -38,10 +38,14 @@ logger = logging.getLogger("netbox_custom_objects.graphql") -# Single-flight rebuild lock: only one thread rebuilds at a time (across all -# branches, so the shared per-rebuild type cache in graphql.types is never built -# by two threads at once); concurrent requests serve the current (possibly slightly -# stale) schema instead of blocking. +# Single-flight rebuild lock. Two roles: +# 1. Correctness (required): build_query_classes() clears and repopulates the +# *process-global* type cache in graphql.types (clear_type_cache()). Two +# rebuilds running at once — e.g. for different branches — would race on that +# cache, one wiping the other's freshly built entries mid-build. This single +# lock (global, not per-branch) serialises ALL rebuilds so that can't happen. +# 2. Performance: concurrent requests that find a stale schema serve it instead +# of blocking on the (potentially expensive) rebuild a peer is already doing. _rebuild_lock = threading.Lock() # Per-branch cache of (signature, schema), keyed by branch identifier (None = main). # GraphQL reflects whichever branch netbox-branching activated for the request, so @@ -259,6 +263,16 @@ def build_full_schema(): static_bases, config = _get_static_query_parts() bases = static_bases + tuple(build_query_classes()) + # Every rebuild creates fresh classes with names ("Query", "TableModelType", + # …) that were used by previous rebuilds' schemas. This relies on Strawberry + # building a *per-schema* type map at strawberry.Schema(...) time — same-named + # types in different Schema objects don't collide; only duplicates *within* one + # schema do (which build_query_classes avoids by uniquifying field/type names). + # That behaviour is version-dependent and Strawberry isn't pinned here (it's + # whatever NetBox ships); it's exercised by the multi-rebuild tests + # (test_live_schema_drops_deleted_type, test_get_live_schema_rebuilds_on_new_type). + # If a future Strawberry raised on cross-schema name reuse, get_live_schema would + # log the rebuild failure and keep serving the prior schema (not silently break). query_cls = strawberry.type(type("Query", bases, {})) return strawberry.Schema( query=query_cls, diff --git a/netbox_custom_objects/graphql/schema.py b/netbox_custom_objects/graphql/schema.py index 9b9fd7d4..b1b627ae 100644 --- a/netbox_custom_objects/graphql/schema.py +++ b/netbox_custom_objects/graphql/schema.py @@ -107,6 +107,9 @@ class is avoided. # Clearing here makes every type fresh per rebuild (rebuilds only happen on an # actual structural change), while the cache still memoizes within this single # rebuild pass so shared and recursive references reuse one built type. + # NOTE: this clear+repopulate of the process-global cache is why rebuilds must + # be serialised — concurrent rebuilds (e.g. different branches) would race here. + # live.get_live_schema holds _rebuild_lock around the whole rebuild to enforce it. clear_type_cache() # Also drop any in-progress build state (stack / cycle taint) leaked by an # exception during a previous rebuild on this pooled thread, so it can't diff --git a/netbox_custom_objects/graphql/types.py b/netbox_custom_objects/graphql/types.py index 3af945b1..33e89cd0 100644 --- a/netbox_custom_objects/graphql/types.py +++ b/netbox_custom_objects/graphql/types.py @@ -238,10 +238,17 @@ def _related_repr(obj): url = obj.get_absolute_url() except Exception: # noqa: BLE001 - URL resolution is best-effort url = None + try: + display = str(obj) + except Exception: # noqa: BLE001 - a broken __str__ must not fail the whole field + # Degrade to a stable identifier so one unrenderable object becomes a + # placeholder rather than erroring the entire (possibly multi-object) field; + # pk and _meta are safe even when __str__ raises. + display = f"{obj._meta.label_lower}:{obj.pk}" return CustomObjectRelatedObjectType( id=obj.pk, object_type=obj._meta.label_lower, - display=str(obj), + display=display, url=url, ) diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index 866f1801..12a77043 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -205,6 +205,30 @@ def test_rebuild_prefetch_covers_field_access(self): f"prefetch missed an access: {[q['sql'] for q in ctx.captured_queries]}", ) + def test_related_repr_degrades_on_broken_str(self): + # A referenced object whose __str__ (or get_absolute_url) raises must not + # propagate — that would error the whole relationship field instead of just + # this one object. It degrades to a stable placeholder display. + from netbox_custom_objects.graphql.types import _related_repr + + class Broken: + pk = 7 + + class _meta: + label_lower = "app.broken" + + def __str__(self): + raise ValueError("boom") + + def get_absolute_url(self): + raise ValueError("no url") + + rep = _related_repr(Broken()) + self.assertEqual(rep.id, 7) + self.assertEqual(rep.object_type, "app.broken") + self.assertEqual(rep.display, "app.broken:7") # fallback, not an exception + self.assertIsNone(rep.url) + class GraphQLLiveSchemaTestCase(CustomObjectsTestCase, TestCase): """ From 1650f9667a3d82d45871bf393ff7368ff432d22f Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 10 Jun 2026 12:59:28 -0700 Subject: [PATCH 084/115] reviwe feedback --- netbox_custom_objects/tests/test_graphql.py | 80 +++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index 12a77043..c8816e65 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -15,6 +15,7 @@ ``__init__.py``) is built and bound per request, exactly as in production. """ +import decimal import json from unittest import mock @@ -383,6 +384,47 @@ def test_scalar_fields_query(self): self.assertTrue(rows[0]["active"]) self.assertEqual(rows[0]["status"], "choice1") + def test_decimal_field_round_trips(self): + # A decimal field is annotated with the stdlib ``decimal.Decimal``, which + # Strawberry maps to its built-in Decimal scalar. Confirm the value + # survives the round trip through the GraphQL endpoint with full + # precision (Strawberry serializes the scalar to a string). + cot = self.create_custom_object_type(name="Product", slug="product") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + cot, name="price", label="Price", type="decimal" + ) + model = cot.get_model() + model.objects.create(name="Widget", price=decimal.Decimal("19.99")) + + data = self._gql("{ custom_objects_product_list { name price } }") + rows = data["custom_objects_product_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "Widget") + self.assertEqual(decimal.Decimal(str(rows[0]["price"])), decimal.Decimal("19.99")) + + def test_json_field_round_trips(self): + # A JSON field is annotated with Strawberry's JSON scalar; confirm a + # structured value survives the round trip through the endpoint intact. + cot = self.create_custom_object_type(name="Config", slug="config") + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True + ) + self.create_custom_object_type_field( + cot, name="data", label="Data", type="json" + ) + model = cot.get_model() + payload_value = {"enabled": True, "ports": [80, 443]} + model.objects.create(name="web", data=payload_value) + + data = self._gql("{ custom_objects_config_list { name data } }") + rows = data["custom_objects_config_list"] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["name"], "web") + self.assertEqual(rows[0]["data"], payload_value) + def test_single_object_query_by_id(self): cot = self.create_simple_custom_object_type(name="Note", slug="note") model = cot.get_model() @@ -596,6 +638,44 @@ def test_multiobject_related_filtered_by_permission(self): self.assertEqual(len(devices), 1) self.assertEqual(devices[0]["id"], str(device.pk)) + def test_multiobject_related_filtered_to_permitted_subset(self): + # The previous tests grant view on the whole model (all-or-nothing). This + # exercises the core security claim at object level: a *constrained* + # ObjectPermission must filter related objects to just the permitted subset. + manufacturer = Manufacturer.objects.create(name="Mfr", slug="mfr") + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model="Model", slug="model" + ) + role = DeviceRole.objects.create(name="Role", slug="role") + site = Site.objects.create(name="DevSite", slug="devsite") + allowed = Device.objects.create( + name="allowed", device_type=device_type, role=role, site=site + ) + denied = Device.objects.create( + name="denied", device_type=device_type, role=role, site=site + ) + cot = self.create_multi_object_custom_object_type(name="Fleet", slug="fleet") + model = cot.get_model() + instance = model.objects.create(name="F1") + instance.devices.add(allowed, denied) + + self._grant(model, "view-fleet") + # View on Device constrained to only the "allowed" device. + perm = ObjectPermission( + name="view-one-device", actions=["view"], constraints={"name": "allowed"} + ) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(Device)) + + payload = self._post( + "{ custom_objects_fleet_list { name devices { id name } } }" + ) + self.assertNotIn("errors", payload, msg=str(payload.get("errors"))) + devices = payload["data"]["custom_objects_fleet_list"][0]["devices"] + self.assertEqual([d["name"] for d in devices], ["allowed"]) + self.assertEqual(devices[0]["id"], str(allowed.pk)) + @override_settings(EXEMPT_VIEW_PERMISSIONS=["*"]) def test_related_anonymous_access_honors_exempt_view_permissions(self): # Anonymous/None traversal must defer to restrict(), exactly like NetBox's From 03b9f85b2b721abca680f0af65ce0909ec826aec Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 11 Jun 2026 10:52:52 -0700 Subject: [PATCH 085/115] #569 - add check for minimum netbox/branching versions for branching support --- docs/branching.md | 4 ++-- netbox_custom_objects/__init__.py | 5 +++++ pyproject.toml | 5 +++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/branching.md b/docs/branching.md index c68cf78f..a260abc0 100644 --- a/docs/branching.md +++ b/docs/branching.md @@ -3,9 +3,9 @@ When using Custom Objects together with NetBox Branching, the following minimum versions are required: - NetBox >= 4.6.2 -- netbox-branching >= 1.1.0 +- netbox-branching >= 1.0.4 -If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. +These requirements are only enforced when `netbox_branching` is present in `PLUGINS`. If you do not use branching, the standard compatibility matrix in `COMPATIBILITY.md` applies. A Django system check (`netbox_custom_objects.E001` / `E002`) will fail at startup if the combination is misconfigured. As of version 0.4.0, Custom Objects is _compatible_ with [NetBox Branching](https://netboxlabs.com/docs/extensions/branching/), but not yet fully supported. Users can safely run both plugins together, but there are some caveats to be aware of. See below for how each Custom Objects model interacts with NetBox Branching. diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 31e9e578..8309df12 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -373,6 +373,11 @@ def ready(self): # model is registered (must happen exactly once, before get_model() runs). install_clear_cache_suppressor() + # Register Django system checks (import triggers @register). These + # enforce the conditional NetBox/netbox-branching version floors that + # PluginConfig's static min_version/max_version can't express. + from . import checks # noqa: F401 + from .models import CustomObjectType from netbox_custom_objects.api.serializers import get_serializer_class diff --git a/pyproject.toml b/pyproject.toml index ff7422b5..bec778db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,11 @@ dependencies = [ [project.optional-dependencies] dev = ["check-manifest", "mkdocs", "mkdocs-material", "ruff"] test = ["coverage", "pytest", "pytest-cov"] +# Install with `pip install "netboxlabs-netbox-custom-objects[branching]"` when +# pairing this plugin with netbox-branching. Note: this extra also implies a +# stricter NetBox version requirement (>= 4.6.2), which is enforced at startup +# by netbox_custom_objects.checks rather than declared here. +branching = ["netbox-branching>=1.0.4"] [project.urls] "Homepage" = "https://netboxlabs.com/" From 045af32a1b4583ef012defc7d93ce2f1e080f4de Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 11 Jun 2026 10:54:44 -0700 Subject: [PATCH 086/115] #569 - add check for minimum netbox/branching versions for branching support --- netbox_custom_objects/checks.py | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 netbox_custom_objects/checks.py diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py new file mode 100644 index 00000000..0a5bde0c --- /dev/null +++ b/netbox_custom_objects/checks.py @@ -0,0 +1,62 @@ +"""Django system checks. + +Enforces conditional version floors that PluginConfig's static +``min_version``/``max_version`` can't express: when netbox-branching is +installed, NetBox and netbox-branching versions are pinned tighter because +the branching integration relies on APIs that only landed in those releases: + +- ``serializer_resolver`` / ``register_serializer_resolver`` — NetBox 4.6.2 +- ``register_branching_resolver``, ``register_objectchange_field_migrator``, + and the ``squash_dependency_graph_built`` signal — netbox-branching 1.0.4 + +Users who never install netbox-branching keep the broader compatibility +window declared by PluginConfig. +""" + +from importlib.metadata import PackageNotFoundError, version as _pkg_version + +from django.apps import apps +from django.conf import settings +from django.core.checks import Error, register +from packaging.version import InvalidVersion, Version + + +# Version floors enforced only when netbox-branching is installed. +REQUIRED_NETBOX_VERSION = '4.6.2' +REQUIRED_BRANCHING_VERSION = '1.0.4' + + +@register() +def check_branching_compatibility(app_configs, **kwargs): + """Enforce branching-only version floors; no-op without netbox-branching.""" + if not apps.is_installed('netbox_branching'): + return [] + + errors = [] + + try: + netbox_version = Version(settings.RELEASE.version) + if netbox_version < Version(REQUIRED_NETBOX_VERSION): + errors.append(Error( + f'netbox-custom-objects requires NetBox >= {REQUIRED_NETBOX_VERSION} ' + f'when netbox-branching is installed (detected {netbox_version}).', + hint='Upgrade NetBox, or remove netbox-branching from PLUGINS ' + 'if you do not need branching support for custom objects.', + id='netbox_custom_objects.E001', + )) + except (AttributeError, InvalidVersion): + pass # settings.RELEASE missing/unparseable — other checks surface it + + try: + branching_version = Version(_pkg_version('netbox-branching')) + if branching_version < Version(REQUIRED_BRANCHING_VERSION): + errors.append(Error( + f'netbox-custom-objects requires netbox-branching >= ' + f'{REQUIRED_BRANCHING_VERSION} (detected {branching_version}).', + hint=f'Upgrade with: pip install "netbox-branching>={REQUIRED_BRANCHING_VERSION}"', + id='netbox_custom_objects.E002', + )) + except (PackageNotFoundError, InvalidVersion): + pass # editable install without dist-info — skip rather than warn + + return errors From 50c61ff1f3fbee1db8220efae8b194ed08742ce5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 11 Jun 2026 11:09:08 -0700 Subject: [PATCH 087/115] claud review changes --- netbox_custom_objects/checks.py | 19 ++++- netbox_custom_objects/tests/test_checks.py | 87 ++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 netbox_custom_objects/tests/test_checks.py diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py index 0a5bde0c..cc682ba4 100644 --- a/netbox_custom_objects/checks.py +++ b/netbox_custom_objects/checks.py @@ -17,7 +17,7 @@ from django.apps import apps from django.conf import settings -from django.core.checks import Error, register +from django.core.checks import Error, Warning, register from packaging.version import InvalidVersion, Version @@ -56,7 +56,20 @@ def check_branching_compatibility(app_configs, **kwargs): hint=f'Upgrade with: pip install "netbox-branching>={REQUIRED_BRANCHING_VERSION}"', id='netbox_custom_objects.E002', )) - except (PackageNotFoundError, InvalidVersion): - pass # editable install without dist-info — skip rather than warn + except PackageNotFoundError: + # netbox-branching is an installed app but its distribution metadata + # isn't discoverable (e.g. an editable checkout without dist-info), so + # the version floor can't be enforced. Warn rather than silently pass + # so an incompatible checkout doesn't slip through unnoticed. + errors.append(Warning( + 'netbox-branching is installed but its version could not be ' + f'determined, so the >= {REQUIRED_BRANCHING_VERSION} requirement ' + 'cannot be verified.', + hint='If using an editable install, ensure its dist-info metadata ' + 'is present (reinstall with `pip install -e`).', + id='netbox_custom_objects.W001', + )) + except InvalidVersion: + pass # unparseable version string — skip rather than emit a confusing error return errors diff --git a/netbox_custom_objects/tests/test_checks.py b/netbox_custom_objects/tests/test_checks.py new file mode 100644 index 00000000..6069588c --- /dev/null +++ b/netbox_custom_objects/tests/test_checks.py @@ -0,0 +1,87 @@ +""" +Tests for the conditional netbox-branching version system check +(``netbox_custom_objects.checks.check_branching_compatibility``). + +The check is pure version-comparison logic gated on whether netbox-branching +is installed, so every branch is exercised by mocking its three inputs: +``apps.is_installed``, ``settings.RELEASE``, and the importlib.metadata +``version`` lookup. No database is required, hence ``SimpleTestCase``. +""" +from importlib.metadata import PackageNotFoundError +from types import SimpleNamespace +from unittest.mock import patch + +from django.test import SimpleTestCase + +from netbox_custom_objects import checks + + +def _run(*, branching_installed=True, netbox_version='4.6.2', branching_version='1.0.4'): + """Invoke the check with each input mocked. + + ``netbox_version`` / ``branching_version`` may be a version string, or an + Exception instance/class to simulate the failure paths (a missing + ``settings.RELEASE``, an unparseable version, or ``PackageNotFoundError``). + """ + if isinstance(netbox_version, str): + release = SimpleNamespace(version=netbox_version) + else: + # No RELEASE attribute -> accessing settings.RELEASE raises AttributeError. + release = None + + def fake_pkg_version(_name): + if isinstance(branching_version, str): + return branching_version + raise branching_version # Exception instance or class + + settings_stub = SimpleNamespace() + if release is not None: + settings_stub.RELEASE = release + + with patch.object(checks.apps, 'is_installed', return_value=branching_installed), \ + patch.object(checks, 'settings', settings_stub), \ + patch.object(checks, '_pkg_version', fake_pkg_version): + return checks.check_branching_compatibility(app_configs=None) + + +def _ids(errors): + return {e.id for e in errors} + + +class CheckBranchingCompatibilityTest(SimpleTestCase): + def test_no_branching_is_noop(self): + """Without netbox-branching installed, the check returns no messages.""" + self.assertEqual(_run(branching_installed=False), []) + + def test_both_versions_satisfied(self): + """Versions at the floor produce no errors.""" + self.assertEqual(_run(netbox_version='4.6.2', branching_version='1.0.4'), []) + + def test_newer_versions_satisfied(self): + self.assertEqual(_run(netbox_version='4.7.0', branching_version='1.1.0'), []) + + def test_netbox_too_old(self): + errors = _run(netbox_version='4.6.1', branching_version='1.0.4') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E001'}) + + def test_branching_too_old(self): + errors = _run(netbox_version='4.6.2', branching_version='1.0.3') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E002'}) + + def test_both_too_old(self): + errors = _run(netbox_version='4.5.0', branching_version='0.9.0') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E001', 'netbox_custom_objects.E002'}) + + def test_missing_release_skips_netbox_check(self): + """A missing settings.RELEASE skips E001 but still evaluates branching.""" + errors = _run(netbox_version=AttributeError, branching_version='1.0.3') + self.assertEqual(_ids(errors), {'netbox_custom_objects.E002'}) + + def test_unparseable_netbox_version_skips_netbox_check(self): + errors = _run(netbox_version='not-a-version', branching_version='1.0.4') + self.assertEqual(errors, []) + + def test_unresolvable_branching_version_warns(self): + """An installed-but-unmeasurable branching dist warns rather than passing silently.""" + errors = _run(netbox_version='4.6.2', branching_version=PackageNotFoundError()) + self.assertEqual(_ids(errors), {'netbox_custom_objects.W001'}) From 26a723ab4cb9b93d7a841feff4ff9fb682e74b89 Mon Sep 17 00:00:00 2001 From: bctiemann Date: Fri, 12 Jun 2026 11:36:11 -0400 Subject: [PATCH 088/115] Fixes #572: Use apps.is_installed() to guard netbox-branching import in live.py (#573) --- netbox_custom_objects/graphql/live.py | 9 +-- netbox_custom_objects/tests/test_graphql.py | 67 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/netbox_custom_objects/graphql/live.py b/netbox_custom_objects/graphql/live.py index 5b934a01..6f8ccbfe 100644 --- a/netbox_custom_objects/graphql/live.py +++ b/netbox_custom_objects/graphql/live.py @@ -36,6 +36,8 @@ import logging import threading +from django.apps import apps as django_apps + logger = logging.getLogger("netbox_custom_objects.graphql") # Single-flight rebuild lock. Two roles: @@ -206,11 +208,10 @@ def connect_signature_invalidation(): ) # Evict a branch's cached schema when the branch itself is deleted. No-op when - # netbox-branching is not installed (there are then no branch-keyed entries). - try: - from netbox_branching.models import Branch - except ImportError: + # netbox-branching is not installed or not in INSTALLED_APPS. + if not django_apps.is_installed('netbox_branching'): return + from netbox_branching.models import Branch post_delete.connect( _evict_branch_schema, sender=Branch, diff --git a/netbox_custom_objects/tests/test_graphql.py b/netbox_custom_objects/tests/test_graphql.py index c8816e65..3fb1cbce 100644 --- a/netbox_custom_objects/tests/test_graphql.py +++ b/netbox_custom_objects/tests/test_graphql.py @@ -300,6 +300,73 @@ def test_live_schema_drops_deleted_type(self): self.assertNotIn("custom_objects_temp_type", str(live_module.get_live_schema())) +class GraphQLSignalRegistrationTestCase(TestCase): + """ + Regression tests for connect_signature_invalidation() guard logic. + + The function must not crash when netbox-branching is installed in the Python + environment but absent from INSTALLED_APPS (which raises RuntimeError, not + ImportError, when Django model classes are imported). + """ + + def _assert_evict_handler_not_registered(self): + """ + Assert that the branch-eviction handler is not registered, using + Signal.disconnect() rather than inspecting the undocumented receivers + list. disconnect() is the public API and returns True only when it + actually removed something — so if it returns False the handler was + never connected (which is the desired outcome). Calling disconnect() + also serves as cleanup: if a prior test or ready() left a stale + registration, this removes it without affecting the assertion. + """ + from django.db.models.signals import post_delete + was_registered = post_delete.disconnect(dispatch_uid="nco_graphql_evict_branch") + self.assertFalse( + was_registered, + "Branch eviction handler must not be registered when branching is not enabled", + ) + + def test_branching_installed_but_not_enabled_does_not_crash(self): + # Simulate netbox-branching installed but not in INSTALLED_APPS by + # patching apps.is_installed to return False for it, then verify that + # connect_signature_invalidation() returns without registering the + # branch-eviction handler. + from django.db.models.signals import post_delete + + from netbox_custom_objects.graphql.live import connect_signature_invalidation + + # Remove any pre-existing registration (e.g. from ready()) so the test + # is self-contained regardless of the environment. + post_delete.disconnect(dispatch_uid="nco_graphql_evict_branch") + + with mock.patch( + "netbox_custom_objects.graphql.live.django_apps.is_installed", + side_effect=lambda app: app != "netbox_branching", + ): + # Must not raise — previously raised RuntimeError here. + connect_signature_invalidation() + + self._assert_evict_handler_not_registered() + + def test_branching_not_installed_does_not_crash(self): + # When netbox-branching is entirely absent, is_installed returns False + # for all apps and the function must return cleanly without registering + # the branch-eviction handler. + from django.db.models.signals import post_delete + + from netbox_custom_objects.graphql.live import connect_signature_invalidation + + post_delete.disconnect(dispatch_uid="nco_graphql_evict_branch") + + with mock.patch( + "netbox_custom_objects.graphql.live.django_apps.is_installed", + side_effect=lambda app: app != "netbox_branching", + ): + connect_signature_invalidation() # must not raise + + self._assert_evict_handler_not_registered() + + @override_settings(LOGIN_REQUIRED=True) class GraphQLEndpointTestCase(CustomObjectsTestCase, TestCase): """ From 8f502240f3d5894ee1e38e247cc0aa0dc00193cf Mon Sep 17 00:00:00 2001 From: bctiemann Date: Tue, 16 Jun 2026 11:55:35 -0400 Subject: [PATCH 089/115] Closes #286: Add ContactsMixin to CustomObject (#571) --- docs/index.md | 1 + netbox_custom_objects/models.py | 2 + .../netbox_custom_objects/customobject.html | 7 ++-- .../object_contacts.html | 11 +++++ netbox_custom_objects/tests/test_views.py | 3 ++ netbox_custom_objects/urls.py | 5 +++ netbox_custom_objects/views.py | 42 +++++++++++++++++++ 7 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html diff --git a/docs/index.md b/docs/index.md index febc7945..6cb4483b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,6 +41,7 @@ This documentation covers: * Full-text search * Change logging * Bookmarks + * Contacts * Custom Links * Cloning * Import/Export diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 4ba45076..e802b691 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -43,6 +43,7 @@ BookmarksMixin, ChangeLoggingMixin, CloningMixin, + ContactsMixin, CustomLinksMixin, CustomValidationMixin, EventRulesMixin, @@ -574,6 +575,7 @@ class CustomObject( BookmarksMixin, ChangeLoggingMixin, CloningMixin, + ContactsMixin, CustomLinksMixin, CustomValidationMixin, ExportTemplatesMixin, diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html index 18faab0e..c7315a4f 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html @@ -10,9 +10,6 @@ {% load i18n %} {% load custom_object_utils %} -{% block extra_controls %} -{% endblock %} - {% block breadcrumbs %}

  • {{ object.custom_object_type.get_verbose_name_plural }}
  • {{ object.custom_object_type }}
    • @@ -38,6 +35,7 @@ {% block controls %}
    {% block control-buttons %} + {% block extra_controls %}{% endblock %} {# Default buttons #} {% if perms.extras.add_bookmark and object.bookmarks %} {% custom_object_bookmark_button object %} @@ -73,6 +71,9 @@
  • {{ object.custom_object_type.get_verbose_name }}
    • +
  • + {% trans "Contacts" %} +
  • {% trans "Journal" %}
    • diff --git a/netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html b/netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html new file mode 100644 index 00000000..c9b7f663 --- /dev/null +++ b/netbox_custom_objects/templates/netbox_custom_objects/object_contacts.html @@ -0,0 +1,11 @@ +{% extends 'generic/object_children.html' %} +{% load helpers %} +{% load i18n %} + +{% block extra_controls %} + {% if perms.tenancy.add_contactassignment %} + + {% trans "Add a contact" %} + + {% endif %} +{% endblock %} \ No newline at end of file diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 405504d4..08a92e70 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -190,6 +190,9 @@ def test_get_object_changelog(self): def test_export_objects(self): ... + def test_export_objects_anonymous(self): + ... + def test_bulk_edit_objects_with_permission(self): ... diff --git a/netbox_custom_objects/urls.py b/netbox_custom_objects/urls.py index a95a001d..04831bdc 100644 --- a/netbox_custom_objects/urls.py +++ b/netbox_custom_objects/urls.py @@ -79,4 +79,9 @@ views.CustomObjectChangeLogView.as_view(), name="customobject_changelog", ), + path( + "//contacts/", + views.CustomObjectContactsView.as_view(), + name="customobject_contacts", + ), ] diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 7c9a581f..e46ffac8 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -1412,3 +1412,45 @@ def get(self, request, custom_object_type, **kwargs): "tab": "changelog", }, ) + + +class CustomObjectContactsView(ConditionalLoginRequiredMixin, View): + """ + Custom contacts view for CustomObject instances. + Shows all contacts assigned to a custom object. + """ + + tab = ViewTab( + label=_("Contacts"), + badge=lambda obj: obj.get_contacts().count(), + permission="tenancy.view_contactassignment", + weight=4500, + ) + + def get(self, request, custom_object_type, **kwargs): + from tenancy.tables import ContactAssignmentTable + + object_type = get_object_or_404(CustomObjectType, slug=custom_object_type) + model = object_type.get_model_with_serializer() + + lookup_kwargs = {k: v for k, v in kwargs.items() if k != "custom_object_type"} + obj = get_object_or_404(model.objects.all(), **lookup_kwargs) + + contacts = ( + obj.get_contacts() + .restrict(request.user, "view") + .order_by("priority", "contact", "role") + ) + table = ContactAssignmentTable(data=contacts, orderable=False) + table.configure(request) + + return render( + request, + "netbox_custom_objects/object_contacts.html", + { + "object": obj, + "table": table, + "base_template": "netbox_custom_objects/customobject.html", + "tab": "contacts", + }, + ) From 3f1a6dc37186f7f57ba2952b627dd50842c3faee Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 17 Jun 2026 10:34:07 -0700 Subject: [PATCH 090/115] change constant name --- netbox_custom_objects/checks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox_custom_objects/checks.py b/netbox_custom_objects/checks.py index cc682ba4..47585501 100644 --- a/netbox_custom_objects/checks.py +++ b/netbox_custom_objects/checks.py @@ -22,7 +22,7 @@ # Version floors enforced only when netbox-branching is installed. -REQUIRED_NETBOX_VERSION = '4.6.2' +REQUIRED_NETBOX_VERSION_FOR_BRANCHING = '4.6.2' REQUIRED_BRANCHING_VERSION = '1.0.4' @@ -36,9 +36,9 @@ def check_branching_compatibility(app_configs, **kwargs): try: netbox_version = Version(settings.RELEASE.version) - if netbox_version < Version(REQUIRED_NETBOX_VERSION): + if netbox_version < Version(REQUIRED_NETBOX_VERSION_FOR_BRANCHING): errors.append(Error( - f'netbox-custom-objects requires NetBox >= {REQUIRED_NETBOX_VERSION} ' + f'netbox-custom-objects requires NetBox >= {REQUIRED_NETBOX_VERSION_FOR_BRANCHING} ' f'when netbox-branching is installed (detected {netbox_version}).', hint='Upgrade NetBox, or remove netbox-branching from PLUGINS ' 'if you do not need branching support for custom objects.', From ee2ad34185c80ce6d992ad634d26e5cc763f0567 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 17 Jun 2026 11:02:30 -0700 Subject: [PATCH 091/115] stub out test --- netbox_custom_objects/tests/test_views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 405504d4..08a92e70 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -190,6 +190,9 @@ def test_get_object_changelog(self): def test_export_objects(self): ... + def test_export_objects_anonymous(self): + ... + def test_bulk_edit_objects_with_permission(self): ... From d6405968b1a4e67480f022067d29230d88157cf6 Mon Sep 17 00:00:00 2001 From: bctiemann Date: Wed, 17 Jun 2026 17:22:18 -0400 Subject: [PATCH 092/115] Closes #254: Quick Add support for custom object object/multiobject fields (#585) --- netbox_custom_objects/field_types.py | 20 +++++ .../htmx/co_quick_add.html | 27 +++++++ netbox_custom_objects/tests/test_views.py | 79 +++++++++++++++++++ netbox_custom_objects/views.py | 77 ++++++++++++++++++ 4 files changed, 203 insertions(+) create mode 100644 netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 4af2aff1..b675fd49 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -19,6 +19,7 @@ from django.db.models.fields.related import ForeignKey, ManyToManyDescriptor from django.db.models.manager import Manager from django.db.models.signals import m2m_changed +from django.urls import reverse from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -785,6 +786,17 @@ def get_form_field(self, field, for_csv_import=False, **kwargs): ) if has_context: form_field.widget.attrs['ts-parent-field'] = '_context' + # Enable quick-add for custom object targets: set quick_add_context + # directly on the widget (bypassing DynamicModelChoiceMixin.get_bound_field + # which uses get_action_url, a tag that can't resolve COT URLs). + if content_type.app_label == APP_LABEL: + form_field.widget.quick_add_context = { + 'url': reverse( + 'plugins:netbox_custom_objects:customobject_add', + kwargs={'custom_object_type': custom_object_type.slug}, + ), + 'params': {}, + } return form_field def get_filterform_field(self, field, **kwargs): @@ -1371,6 +1383,14 @@ def get_form_field(self, field, for_csv_import=False, **kwargs): ) if has_context: form_field.widget.attrs['ts-parent-field'] = '_context' + if content_type.app_label == APP_LABEL: + form_field.widget.quick_add_context = { + 'url': reverse( + 'plugins:netbox_custom_objects:customobject_add', + kwargs={'custom_object_type': custom_object_type.slug}, + ), + 'params': {}, + } return form_field def get_display_value(self, instance, field_name): diff --git a/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html b/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html new file mode 100644 index 00000000..19fdd82a --- /dev/null +++ b/netbox_custom_objects/templates/netbox_custom_objects/htmx/co_quick_add.html @@ -0,0 +1,27 @@ +{% load form_helpers %} +{% load helpers %} +{% load i18n %} +
    +

    + {% trans "Quick Add" %} {{ model|meta:"verbose_name"|bettertitle }} +

    + +
    +
    +
    + {% csrf_token %} + {% include 'htmx/form.html' %} +
    + + +
    +
    +
    \ No newline at end of file diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 08a92e70..6a798f48 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -904,3 +904,82 @@ def test_object_selector_search(self): 'q': 'Alpha', }) self.assertEqual(response.status_code, 200) + + +class QuickAddViewTestCase(CustomObjectsTestCase, TestCase): + """ + Tests for the quick-add flow in CustomObjectEditView. + + Covers GET (modal renders), POST success (object created, quick_add_created.html + returned), and POST validation failure (errors re-rendered in our custom template). + """ + + def setUp(self): + super().setUp() + self.user.is_superuser = True + self.user.save() + + # Target COT: objects of this type will be quick-added. + self.target_cot = self.create_simple_custom_object_type( + name='Target', slug='target', + ) + target_ot = ObjectType.objects.get( + app_label='netbox_custom_objects', + model=self.target_cot.get_table_model_name(self.target_cot.id).lower(), + ) + + # Source COT: has an object field pointing at Target. + self.source_cot = self.create_custom_object_type(name='Source', slug='source') + self.create_custom_object_type_field( + self.source_cot, name='name', label='Name', type='text', + primary=True, required=True, + ) + self.create_custom_object_type_field( + self.source_cot, name='ref', label='Ref', type='object', + related_object_type=target_ot, + ) + + self.add_url = reverse( + 'plugins:netbox_custom_objects:customobject_add', + kwargs={'custom_object_type': self.target_cot.slug}, + ) + + def test_quick_add_get_returns_200(self): + """GET ?_quickadd=True renders the custom quick-add modal without errors.""" + response = self.client.get( + self.add_url, + {'_quickadd': 'True', 'target': 'id_ref'}, + ) + self.assertEqual(response.status_code, 200) + # The custom template (not the core one) is used. + self.assertContains(response, 'hx-post=') + self.assertContains(response, f'/plugins/custom-objects/{self.target_cot.slug}/add/') + + def test_quick_add_post_success_creates_object(self): + """POST with _quickadd in POST data creates the object and returns quick_add_created template.""" + model = self.target_cot.get_model() + count_before = model.objects.count() + + response = self.client.post( + f'{self.add_url}?_quickadd=True&target=id_ref', + data={'quickadd-name': 'quick-created', '_quickadd': ''}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(model.objects.count(), count_before + 1) + # The success template contains the object PK for JS auto-selection. + self.assertContains(response, 'quick-add-object') + self.assertTrue(model.objects.filter(name='quick-created').exists()) + + def test_quick_add_post_validation_failure_rerenders(self): + """POST with missing required field re-renders the quick-add form with errors.""" + response = self.client.post( + f'{self.add_url}?_quickadd=True&target=id_ref', + # name is required but omitted + data={'_quickadd': ''}, + ) + self.assertEqual(response.status_code, 200) + # Error re-render uses our custom template, not a redirect. + self.assertContains(response, 'hx-post=') + # No new object created. + model = self.target_cot.get_model() + self.assertFalse(model.objects.exists()) diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index e46ffac8..bfb09101 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -1,13 +1,19 @@ import logging from core.models import ObjectChange +from core.signals import clear_events from core.tables import ObjectChangeTable from django.apps import apps as django_apps +from django.contrib import messages from django.contrib.contenttypes.models import ContentType +from django.db import router, transaction from django.db.models import ProtectedError, Q, RestrictedError from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django.utils.html import escape +from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ +from utilities.exceptions import AbortRequest, PermissionsViolation from django.views.generic import View from extras.choices import CustomFieldUIVisibleChoices from extras.forms import JournalEntryForm @@ -590,6 +596,7 @@ def get_extra_context(self, request, instance): class CustomObjectEditView(generic.ObjectEditView): template_name = "netbox_custom_objects/customobject_edit.html" htmx_template_name = "netbox_custom_objects/htmx/edit_fields.html" + _CO_QUICK_ADD_TEMPLATE = 'netbox_custom_objects/htmx/co_quick_add.html' form = None queryset = None object = None @@ -613,6 +620,76 @@ def get_queryset(self, request): model = self.object._meta.model return model.objects.all() + def get(self, request, *args, **kwargs): + if not request.GET.get('_quickadd'): + return super().get(request, *args, **kwargs) + # Quick-add GET: the core htmx/quick_add.html uses {% action_url model 'add' %} + # which cannot resolve custom object URLs (they require a COT slug). + # Use our own template that builds the form action URL from the slug instead. + obj = self.object # already populated by setup() + form = self.form( + instance=obj, + initial=normalize_querydict(request.GET), + prefix='quickadd', + ) + restrict_form_fields(form, request.user) + return render(request, self._CO_QUICK_ADD_TEMPLATE, { + 'model': obj._meta.model, + 'object': obj, + 'form': form, + }) + + def post(self, request, *args, **kwargs): + if '_quickadd' not in request.POST: + return super().post(request, *args, **kwargs) + # Quick-add POST: mirrors the parent's save logic but re-renders validation + # errors with our custom template instead of the core htmx/quick_add.html + # (which uses {% action_url model 'add' %} and fails for COT models). + _qa_logger = logging.getLogger('netbox.views.ObjectEditView') + obj = self.object # already populated by setup() + model = self.queryset.model + + if obj.pk and hasattr(obj, 'snapshot'): + obj.snapshot() + + obj = self.alter_object(obj, request, args, kwargs) + + form = self.form( + data=request.POST, + files=request.FILES, + instance=obj, + prefix='quickadd', + ) + restrict_form_fields(form, request.user) + + if form.is_valid(): + obj._changelog_message = form.cleaned_data.pop('changelog_message', '') + try: + with transaction.atomic(using=router.db_for_write(model)): + object_created = form.instance.pk is None + obj = form.save() + if not self.queryset.filter(pk=obj.pk).exists(): + raise PermissionsViolation() + msg = '{} {}'.format( + 'Created' if object_created else 'Modified', + model._meta.verbose_name, + ) + _qa_logger.info(f"{msg} {obj} (PK: {obj.pk})") + if hasattr(obj, 'get_absolute_url'): + msg = mark_safe(f'{msg} {escape(obj)}') + messages.success(request, msg) + return render(request, 'htmx/quick_add_created.html', {'object': obj}) + except (AbortRequest, PermissionsViolation) as e: + _qa_logger.debug(e.message) + form.add_error(None, e.message) + clear_events.send(sender=self) + + return render(request, self._CO_QUICK_ADD_TEMPLATE, { + 'model': obj._meta.model, + 'object': obj, + 'form': form, + }) + def get_object(self, **kwargs): if self.object: return self.object From 3b86c4387ff61e8be3d108fcb6a4403a2f23d05f Mon Sep 17 00:00:00 2001 From: bctiemann Date: Fri, 26 Jun 2026 08:04:23 -0400 Subject: [PATCH 093/115] Closes #376: Add first-class ownership to Custom Object instances (#597) * Add first-class ownership to Custom Object instances (#376) Adds OwnerMixin to CustomObject so every CO instance has an owner FK. Existing CO tables pick up owner_id automatically via heal_cot (field is nullable). Owner fields appear in the edit form, list filter, REST API, and detail page subtitle. Co-Authored-By: Claude Sonnet 4.6 * Fix CI failures and add owner field tests for #376 - Replace OwnerFilterMixin in filterset with explicit filters to avoid ValueError from get_additional_lookups() rejecting cross-FK field_name traversals (owner__group); method-based filter bypasses the validation - Fix test_create_with_nested_serializers: grant view_device permission inline using ObjectPermission (add_permissions not available here) - Fix test_patch_updates_cross_cot_m2m_field: grant view on model_target so the feature-branch restrict_queryset enforcement passes for FK resolution - Update test_custom_object_type_get_model_without_fields: OwnerMixin adds owner_id column so base field count is now 4, not 3 - Add OwnerAPITest covering null default, API response shape, create/patch/ clear owner via PK, and filter_by_owner_id / filter_by_owner_group_id Co-Authored-By: Claude Sonnet 4.6 * Fix three CI failures introduced in owner field implementation (#376) - Remove non-existent register_model() call from OwnerAPITest.setUp(); COT models are retrieved directly via cot.get_model() - Guard owner FK serializer field when a COT user-field is named 'owner': Django silently lets the CharField shadow the OwnerMixin FK, causing OwnerSerializer .to_representation() to receive a string and crash on .pk - Add SET CONSTRAINTS ALL IMMEDIATE before schema_editor.add_field() in _schema_add_field(); Django creates FK constraints as DEFERRABLE INITIALLY DEFERRED so within a TestCase outer transaction the owner_id deferred trigger blocks subsequent ALTER TABLE ADD COLUMN calls with "pending trigger events" Co-Authored-By: Claude Sonnet 4.6 * Address PR review: reserved owner name, heal_cot constraint flush, compatibility - Add 'owner' to RESERVED_FIELD_NAMES so users cannot create a COT field that shadows the OwnerMixin FK - Add SET CONSTRAINTS ALL IMMEDIATE before editor.add_field() in heal_cot() in mixin_migration.py, same pattern as _schema_add_field() - Add test_custom_object_type_field_reserved_name_rejected covering 'owner' and other reserved names - Add 0.6.x row to COMPATIBILITY.md; OwnerMixin requires NetBox >= 4.5.0 which is within the plugin supported range (>= 4.5.2) Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- COMPATIBILITY.md | 1 + netbox_custom_objects/api/serializers.py | 13 ++ netbox_custom_objects/constants.py | 1 + netbox_custom_objects/dynamic_forms.py | 3 +- netbox_custom_objects/filtersets.py | 17 ++ netbox_custom_objects/mixin_migration.py | 3 + netbox_custom_objects/models.py | 5 + .../netbox_custom_objects/customobject.html | 4 + netbox_custom_objects/tests/test_api.py | 162 +++++++++++++++++- netbox_custom_objects/tests/test_models.py | 15 +- netbox_custom_objects/views.py | 3 +- 11 files changed, 222 insertions(+), 5 deletions(-) diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index d016cb29..c83a5feb 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -2,6 +2,7 @@ | Release | Minimum NetBox Version | Maximum NetBox Version | |---------|------------------------|------------------------| +| 0.6.x | 4.5.2 | 4.6.x | | 0.5.1 | 4.5.2 | 4.6.x | | 0.5.0 | 4.5.2 | 4.6.x | | 0.4.10 | 4.4.0 | 4.5.x | diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index 97407361..999138e1 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -14,6 +14,8 @@ from rest_framework.reverse import reverse from rest_framework.utils import model_meta +from users.api.serializers_.owners import OwnerSerializer + from netbox_custom_objects import constants, field_types from netbox_custom_objects.models import (CustomObject, CustomObjectType, CustomObjectTypeField) @@ -457,8 +459,16 @@ def get_serializer_class(model, skip_object_fields=False): if isinstance(attr, field_types.PolymorphicM2MDescriptor) ) + # If a COT field is named 'owner', it shadows the OwnerMixin FK on the dynamic model + # (Django silently lets child attrs override abstract parent fields). The serializer + # must skip the FK owner field to avoid OwnerSerializer.to_representation() being + # called on a string value and crashing on .pk. + has_owner_field_conflict = any(f.name == 'owner' for f in model_fields) + # Create field list including all necessary fields base_fields = ["id", "url", "display", "created", "last_updated", "tags"] + if not has_owner_field_conflict: + base_fields.insert(3, "owner") # Include _context field when the model has designated context fields has_context_fields = bool(getattr(model, '_context_field_ids', [])) @@ -653,6 +663,9 @@ def validate(self, data): "validate": validate, } + if not has_owner_field_conflict: + attrs["owner"] = OwnerSerializer(nested=True, required=False, allow_null=True) + if has_context_fields: attrs["_context"] = serializers.SerializerMethodField() attrs["get__context"] = get__context diff --git a/netbox_custom_objects/constants.py b/netbox_custom_objects/constants.py index d7ac7fc3..e34ee91e 100644 --- a/netbox_custom_objects/constants.py +++ b/netbox_custom_objects/constants.py @@ -38,6 +38,7 @@ "last_updated", "model", "objects", + "owner", "pk", "refresh_from_db", "save", diff --git a/netbox_custom_objects/dynamic_forms.py b/netbox_custom_objects/dynamic_forms.py index 153f6d86..6fb227a9 100644 --- a/netbox_custom_objects/dynamic_forms.py +++ b/netbox_custom_objects/dynamic_forms.py @@ -1,6 +1,7 @@ import logging from netbox.forms import NetBoxModelFilterSetForm +from netbox.forms.mixins import OwnerFilterMixin as OwnerFilterFormMixin from utilities.forms.fields import TagFilterField from . import field_types @@ -35,6 +36,6 @@ def build_filterset_form_class(model): return type( f"{model._meta.object_name}FilterForm", - (NetBoxModelFilterSetForm,), + (OwnerFilterFormMixin, NetBoxModelFilterSetForm,), attrs, ) diff --git a/netbox_custom_objects/filtersets.py b/netbox_custom_objects/filtersets.py index da96dbd0..a1358807 100644 --- a/netbox_custom_objects/filtersets.py +++ b/netbox_custom_objects/filtersets.py @@ -11,6 +11,7 @@ from extras.choices import CustomFieldTypeChoices from netbox.filtersets import NetBoxModelFilterSet +from users.models import Owner, OwnerGroup from .constants import APP_LABEL from .models import CustomObjectType @@ -399,10 +400,26 @@ def search(self, queryset, name, value): return queryset.none() return queryset.filter(q) + def filter_owner_group_id(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(owner__group__in=value) + attrs = { "Meta": meta, "__module__": "netbox_custom_objects.filtersets", "search": search, + "filter_owner_group_id": filter_owner_group_id, + "owner_id": django_filters.ModelMultipleChoiceFilter( + field_name='owner', + queryset=Owner.objects.all(), + label='Owner (ID)', + ), + "owner_group_id": django_filters.ModelMultipleChoiceFilter( + queryset=OwnerGroup.objects.all(), + method='filter_owner_group_id', + label='Owner Group (ID)', + ), } # For each custom field, add a corresponding filter (dict of name → Filter). diff --git a/netbox_custom_objects/mixin_migration.py b/netbox_custom_objects/mixin_migration.py index 14e99a8f..fe46d336 100644 --- a/netbox_custom_objects/mixin_migration.py +++ b/netbox_custom_objects/mixin_migration.py @@ -157,6 +157,9 @@ def heal_cot(cot, verbosity=1, dry_run=False): try: with connection.schema_editor() as editor: + # Flush pending DEFERRABLE FK trigger events before ALTER TABLE; + # PostgreSQL rejects ADD COLUMN when deferred triggers are pending. + editor.execute('SET CONSTRAINTS ALL IMMEDIATE') editor.add_field(model, field) added.append(col_name) if verbosity >= 1: diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index e802b691..43d36f6b 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -39,6 +39,7 @@ from extras.utils import is_taggable, run_validators from netbox.config import get_config from netbox.models import ChangeLoggedModel, NetBoxModel +from netbox.models.mixins import OwnerMixin from netbox.models.features import ( BookmarksMixin, ChangeLoggingMixin, @@ -317,6 +318,9 @@ def _schema_add_field(fi, model, schema_editor, schema_conn): mf._to_model_name, ) + # Flush any pending DEFERRABLE FK trigger events (e.g. owner_id from OwnerMixin) + # before ALTER TABLE; PostgreSQL rejects ADD COLUMN when deferred triggers are pending. + schema_editor.execute('SET CONSTRAINTS ALL IMMEDIATE') schema_editor.add_field(model, mf) if fi.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: ft.create_m2m_table(fi, model, fi.name, schema_conn=schema_conn) @@ -572,6 +576,7 @@ def _set_with_collision_preference(result, key, value): class CustomObject( + OwnerMixin, BookmarksMixin, ChangeLoggingMixin, CloningMixin, diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html index c7315a4f..7137cad0 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html @@ -24,6 +24,10 @@ {% block subtitle %}
    + {% if object.owner %} + {{ object.owner|linkify }} + · + {% endif %} {% trans "Created" %} {{ object.created|isodatetime:"minutes" }} {% if object.last_updated %} · diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index f96bb179..0f86c6b8 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -371,6 +371,10 @@ def test_create_with_nested_serializers(self): obj_perm.save() obj_perm.users.add(self.user) obj_perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + view_device_perm = ObjectPermission(name='view-device-for-nested', actions=['view']) + view_device_perm.save() + view_device_perm.users.add(self.user) + view_device_perm.object_types.add(ObjectType.objects.get_for_model(Device)) devices = Device.objects.all() @@ -850,7 +854,7 @@ def test_detail_response_contains_netbox_standard_fields(self): response = self.client.get(self._detail_url(instance), **self.header) self.assertEqual(response.status_code, status.HTTP_200_OK) - for field in ('id', 'url', 'display', 'created', 'last_updated', 'tags'): + for field in ('id', 'url', 'display', 'owner', 'created', 'last_updated', 'tags'): self.assertIn(field, response.data, f"Standard field '{field}' missing from response") def test_detail_response_contains_custom_fields(self): @@ -1503,6 +1507,7 @@ def _add_perm(self, action, model): def test_patch_updates_cross_cot_m2m_field(self): """#443 – PATCH with a list of target PKs must update the M2M field.""" self._add_perm('change', self.model_source) + self._add_perm('view', self.model_target) # Confirm initial state. self.assertSetEqual( @@ -1601,3 +1606,158 @@ def test_get_response_includes_cross_cot_m2m_field(self): self.assertIn('refs', response.data, 'Response must include the refs M2M field.') ref_ids = [r['id'] for r in response.data['refs']] self.assertIn(self.obj_target1.pk, ref_ids) + + +class OwnerAPITest(CustomObjectsTestCase, TestCase): + """Tests for the owner field on custom object instances (#376).""" + + def setUp(self): + super().setUp() + from netbox_custom_objects.models import CustomObjectType + self.cot = CustomObjectType.objects.create(name='OwnedThing', slug='owned-thing') + self.model = self.cot.get_model() + token = create_token(self.user) + self.header = {'HTTP_AUTHORIZATION': f'Token {token}'} + self.client = APIClient() + + def _list_url(self): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-list', + kwargs={'custom_object_type': self.cot.slug}, + ) + + def _detail_url(self, pk): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-detail', + kwargs={'pk': pk, 'custom_object_type': self.cot.slug}, + ) + + def _add_perm(self, action, model): + perm = ObjectPermission(name=f'{action}-{model._meta.model_name}', actions=[action]) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model)) + return perm + + def _grant_owner_view(self, suffix=''): + from users.models import Owner + perm = ObjectPermission(name=f'view-owner{suffix}', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(Owner)) + return perm + + def test_owner_null_by_default(self): + """New CO instances must have owner=None when not explicitly set.""" + obj = self.model.objects.create() + self.assertIsNone(obj.owner) + + def test_owner_present_in_api_response(self): + """GET response must contain an 'owner' key (null when unset).""" + obj = self.model.objects.create() + self._add_perm('view', self.model) + + response = self.client.get(self._detail_url(obj.pk), **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('owner', response.data) + self.assertIsNone(response.data['owner']) + + def test_create_with_owner(self): + """POST with owner= must persist the owner FK.""" + from users.models import Owner + owner = Owner.objects.create(name='create-owner') + self._add_perm('add', self.model) + self._grant_owner_view('-create') + + response = self.client.post( + self._list_url(), + {'owner': owner.pk}, + format='json', + **self.header, + ) + self.assertEqual( + response.status_code, + status.HTTP_201_CREATED, + f'Expected 201; got {response.status_code}: {getattr(response, "data", response.content)}', + ) + obj = self.model.objects.get(pk=response.data['id']) + self.assertEqual(obj.owner_id, owner.pk) + + def test_patch_sets_owner(self): + """PATCH with owner= must update the owner FK.""" + from users.models import Owner + obj = self.model.objects.create() + owner = Owner.objects.create(name='patch-owner') + self._add_perm('change', self.model) + self._grant_owner_view('-patch') + + response = self.client.patch( + self._detail_url(obj.pk), + {'owner': owner.pk}, + format='json', + **self.header, + ) + self.assertEqual( + response.status_code, + status.HTTP_200_OK, + f'Expected 200; got {response.status_code}: {getattr(response, "data", response.content)}', + ) + obj.refresh_from_db() + self.assertEqual(obj.owner_id, owner.pk) + + def test_patch_clears_owner(self): + """PATCH with owner=null must clear the owner FK.""" + from users.models import Owner + owner = Owner.objects.create(name='clear-owner') + obj = self.model.objects.create(owner=owner) + self._add_perm('change', self.model) + + response = self.client.patch( + self._detail_url(obj.pk), + {'owner': None}, + format='json', + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + obj.refresh_from_db() + self.assertIsNone(obj.owner) + + def test_filter_by_owner_id(self): + """?owner_id= must return only instances with that owner.""" + from users.models import Owner + owner_a = Owner.objects.create(name='filter-owner-a') + owner_b = Owner.objects.create(name='filter-owner-b') + obj_a = self.model.objects.create(owner=owner_a) + obj_b = self.model.objects.create(owner=owner_b) + self._add_perm('view', self.model) + + response = self.client.get( + self._list_url(), + {'owner_id': owner_a.pk}, + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + ids = [r['id'] for r in response.data['results']] + self.assertIn(obj_a.pk, ids) + self.assertNotIn(obj_b.pk, ids) + + def test_filter_by_owner_group_id(self): + """?owner_group_id= must return only instances whose owner belongs to that group.""" + from users.models import Owner, OwnerGroup + group_x = OwnerGroup.objects.create(name='group-x') + group_y = OwnerGroup.objects.create(name='group-y') + owner_x = Owner.objects.create(name='owner-in-x', group=group_x) + owner_y = Owner.objects.create(name='owner-in-y', group=group_y) + obj_x = self.model.objects.create(owner=owner_x) + obj_y = self.model.objects.create(owner=owner_y) + self._add_perm('view', self.model) + + response = self.client.get( + self._list_url(), + {'owner_group_id': group_x.pk}, + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + ids = [r['id'] for r in response.data['results']] + self.assertIn(obj_x.pk, ids) + self.assertNotIn(obj_y.pk, ids) diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 9e1df4ed..2dbe2458 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -125,8 +125,8 @@ def test_custom_object_type_get_model_without_fields(self): custom_object_type = self.create_custom_object_type(name="TestObject") model = custom_object_type.get_model() - # Base fields: id, created, last_updated - self.assertEqual(len(model._meta.fields), 3) + # Base fields: id, created, last_updated, owner + self.assertEqual(len(model._meta.fields), 4) def test_custom_object_type_get_model_with_primary_field(self): """Test get_model method with a primary field.""" @@ -431,6 +431,17 @@ def test_custom_object_type_field_name_validation(self): ) field.full_clean() + def test_custom_object_type_field_reserved_name_rejected(self): + """Field names in RESERVED_FIELD_NAMES must be rejected with ValidationError.""" + for reserved in ("owner", "tags", "id", "created", "last_updated"): + with self.assertRaises(ValidationError, msg=f"Expected ValidationError for reserved name={reserved!r}"): + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name=reserved, + type="text", + ) + field.full_clean() + def test_custom_object_type_field_unique_name_per_type(self): """Test that field names must be unique within a custom object type.""" self.create_custom_object_type_field( diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index bfb09101..63e9ca8d 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -23,6 +23,7 @@ NetBoxModelBulkEditForm, NetBoxModelImportForm, ) +from netbox.forms.mixins import OwnerMixin as OwnerFormMixin from netbox.views import generic from netbox.views.generic.mixins import TableMixin from utilities.forms import ConfirmationForm, DeleteForm, restrict_form_fields @@ -807,7 +808,7 @@ def get_form(self, model): form_class = type( f"{model._meta.object_name}Form", - (forms.NetBoxModelForm,), + (OwnerFormMixin, forms.NetBoxModelForm,), attrs, ) From eb06183269367ba6808f4302820c7d5696a4cf4e Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 10:55:18 -0700 Subject: [PATCH 094/115] #532 - move integer BigIntegerField --- netbox_custom_objects/field_types.py | 5 +++- .../tests/test_field_types.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 0de2bf65..0f70e556 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -366,7 +366,10 @@ def get_model_field(self, field, **kwargs): # TODO: handle all args for IntegerField field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) - return models.IntegerField(null=True, blank=True, **field_kwargs) + # Use a 64-bit column (bigint) so values are not capped at the 32-bit + # signed range. Matches NetBox core's convention for semantic integers + # (e.g. ASNField, L2VPN identifier). See issue #532. + return models.BigIntegerField(null=True, blank=True, **field_kwargs) def get_filterform_field(self, field, **kwargs): return forms.IntegerField( diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 1fdb734e..96e77eef 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -6,6 +6,7 @@ from datetime import date, datetime from decimal import Decimal from django.core.exceptions import FieldDoesNotExist, ValidationError +from django.db import models from django.test import TestCase from core.models import ObjectType @@ -189,6 +190,28 @@ def test_integer_field_model_generation(self): self.assertEqual(instance.count, 25) + def test_integer_field_is_64_bit(self): + """Integer fields use a 64-bit (bigint) column, not 32-bit (issue #532).""" + self.create_custom_object_type_field( + self.custom_object_type, + name="count", + label="Count", + type="integer", + ) + + model = self.custom_object_type.get_model() + + # The generated model field must be a BigIntegerField. + self.assertIsInstance( + model._meta.get_field("count"), models.BigIntegerField + ) + + # A value beyond the signed 32-bit range must round-trip through the DB. + big_value = 9_000_000_000 # > 2**31 - 1 (2_147_483_647) + instance = model.objects.create(name="Test", count=big_value) + instance.refresh_from_db() + self.assertEqual(instance.count, big_value) + class DecimalFieldTypeTestCase(FieldTypeTestCase): """Test cases for decimal field type.""" From 059323cdbc5a2ff859f44c041e0002c9819c9049 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 10:56:28 -0700 Subject: [PATCH 095/115] #532 - move integer BigIntegerField --- netbox_custom_objects/field_types.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 0f70e556..4a46028e 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -366,9 +366,6 @@ def get_model_field(self, field, **kwargs): # TODO: handle all args for IntegerField field_kwargs = self._safe_kwargs(**kwargs) field_kwargs.update({"default": field.default, "unique": field.unique}) - # Use a 64-bit column (bigint) so values are not capped at the 32-bit - # signed range. Matches NetBox core's convention for semantic integers - # (e.g. ASNField, L2VPN identifier). See issue #532. return models.BigIntegerField(null=True, blank=True, **field_kwargs) def get_filterform_field(self, field, **kwargs): From d1b9b4319d7b9cacc72c32fe29570e074a94d0f7 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 11:45:11 -0700 Subject: [PATCH 096/115] #551 Implement Location Field --- docs/field-attributes.md | 22 +++ netbox_custom_objects/api/serializers.py | 25 +++ netbox_custom_objects/choices.py | 14 ++ netbox_custom_objects/field_types.py | 151 +++++++++++++++++- netbox_custom_objects/filtersets.py | 13 ++ netbox_custom_objects/models.py | 57 ++++++- .../netbox_custom_objects/customobject.html | 19 ++- .../templatetags/custom_object_utils.py | 30 +++- netbox_custom_objects/tests/test_api.py | 85 ++++++++++ .../tests/test_field_types.py | 93 +++++++++++ .../tests/test_filtersets.py | 60 +++++++ netbox_custom_objects/tests/test_views.py | 59 +++++++ netbox_custom_objects/utilities.py | 37 +++++ netbox_custom_objects/views.py | 35 ++++ 14 files changed, 685 insertions(+), 15 deletions(-) diff --git a/docs/field-attributes.md b/docs/field-attributes.md index 2050780b..075e0d3f 100644 --- a/docs/field-attributes.md +++ b/docs/field-attributes.md @@ -19,6 +19,7 @@ The following attributes are available when creating or editing a Custom Object | `multiselect` | Multiple selections from a choice set | | `object` | Reference to a single object (built-in NetBox object or Custom Object) | | `multiobject` | Reference to multiple objects of the same type | +| `coordinates` | A geographic latitude/longitude pair (with a map link) | ## Common Attributes @@ -85,3 +86,24 @@ Field types: `object`, `multiobject` !!! note To reference another Custom Object Type, choose `Custom Objects > ` in the **Related object type** dropdown. To create a polymorphic field that may reference objects of multiple types, enable **Polymorphic** and select the allowed types under **Related object types**. + +## Coordinates Fields + +Field type: `coordinates` + +A single `coordinates` field stores a geographic latitude/longitude pair, mirroring NetBox's +native Site/Device coordinates (plain decimal columns — PostGIS is not required). Adding one +`coordinates` field named `location` creates two backing columns and two form inputs: + +- `location_latitude` — decimal, range −90 to 90, up to 6 decimal places +- `location_longitude` — decimal, range −180 to 180, up to 6 decimal places + +Behaviour: + +- **Both-or-neither.** Latitude and longitude must either both be set or both be empty; setting + only one is rejected in forms and via the REST API. +- **REST API.** The pair is exposed as two flat fields, `_latitude` and `_longitude` + (matching NetBox core's serializers), not as a nested object. +- **Map link.** Detail views render the coordinates with a **Map** button that opens the location + using NetBox's `MAPS_URL` configuration parameter (Google Maps by default). +- `Must be unique` and `Default` are not supported for `coordinates` fields. diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index be3249ed..b0446dd1 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -16,6 +16,7 @@ from rest_framework.utils import model_meta from netbox_custom_objects import constants, field_types +from netbox_custom_objects.choices import CustomObjectFieldTypeChoices from netbox_custom_objects.models import (CustomObject, CustomObjectType, CustomObjectTypeField) @@ -469,6 +470,11 @@ def get_serializer_class(model, skip_object_fields=False): # Only include custom field names that will actually be added to the serializer custom_field_names = [] for field in model_fields: + # Coordinates fields expand into two real columns; expose them flat, mirroring + # NetBox core's Site/Device latitude/longitude serializer fields. + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + custom_field_names += [f"{field.name}_latitude", f"{field.name}_longitude"] + continue if field.name not in model_field_names: continue # excluded during model generation (e.g. broken FK) if skip_object_fields and field.type in [ @@ -524,6 +530,12 @@ def get_display(self, obj): f.name for f in model_fields if f.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT and f.is_polymorphic } + # (latitude_column, longitude_column) pairs for coordinates fields. + _coordinate_fields = [ + (f"{f.name}_latitude", f"{f.name}_longitude") + for f in model_fields + if f.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES + ] def get__context(self, obj): """Return context field values as a nested display object for APISelect secondary text.""" @@ -661,6 +673,19 @@ def validate(self, data): saved[field_name] = data.pop(field_name) data = NetBoxModelSerializer.validate(self, data) data.update(saved) + + # Coordinates: latitude and longitude must both be set or both be empty. + # On partial updates, only enforce when at least one of the pair is supplied. + for lat_field, lon_field in _coordinate_fields: + if lat_field not in data and lon_field not in data: + continue + latitude = data.get(lat_field) + longitude = data.get(lon_field) + if (latitude is None) != (longitude is None): + raise serializers.ValidationError( + _("Latitude and longitude must both be set or both be empty.") + ) + return data # Create basic attributes for the serializer diff --git a/netbox_custom_objects/choices.py b/netbox_custom_objects/choices.py index 8f66351d..59a8f4f4 100644 --- a/netbox_custom_objects/choices.py +++ b/netbox_custom_objects/choices.py @@ -1,7 +1,21 @@ from django.utils.translation import gettext_lazy as _ +from extras.choices import CustomFieldTypeChoices from utilities.choices import ChoiceSet +class CustomObjectFieldTypeChoices(CustomFieldTypeChoices): + """ + Extends NetBox's CustomFieldTypeChoices with field types specific to custom + objects. All existing ``CustomFieldTypeChoices.TYPE_*`` members remain valid. + """ + + TYPE_COORDINATES = "coordinates" + + CHOICES = CustomFieldTypeChoices.CHOICES + ( + (TYPE_COORDINATES, _("Coordinates")), + ) + + class ObjectFieldOnDeleteChoices(ChoiceSet): """Controls what happens to a Custom Object when the referenced object is deleted.""" CASCADE = "cascade" diff --git a/netbox_custom_objects/field_types.py b/netbox_custom_objects/field_types.py index 0de2bf65..efca8261 100644 --- a/netbox_custom_objects/field_types.py +++ b/netbox_custom_objects/field_types.py @@ -1,6 +1,7 @@ import hashlib import json import logging +from decimal import Decimal import django_tables2 as tables from django import forms @@ -8,8 +9,9 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import ArrayField -from django.core.exceptions import FieldDoesNotExist -from django.core.validators import RegexValidator +from django.core.exceptions import FieldDoesNotExist, ValidationError +from django.core.validators import (MaxValueValidator, MinValueValidator, + RegexValidator) from django.db import connection, models from django.db.utils import OperationalError, ProgrammingError from django.db.models.fields.related import ForeignKey, ManyToManyDescriptor @@ -35,7 +37,8 @@ from utilities.templatetags.builtins.filters import linkify, render_markdown from netbox.tables.columns import BooleanColumn -from netbox_custom_objects.choices import ObjectFieldOnDeleteChoices +from netbox_custom_objects.choices import (CustomObjectFieldTypeChoices, + ObjectFieldOnDeleteChoices) from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.utilities import extract_cot_id_from_model_name, generate_model @@ -1980,6 +1983,147 @@ def __get__(self, instance, owner=None): return PolymorphicMultiObjectReverseManager(instance, self.cot_pk, self.through_model_name) +class CoordinatesColumn(tables.Column): + """ + Table column for a coordinates field. Renders the latitude/longitude pair stored + in the two backing columns as ``", "`` (or a placeholder when unset). + """ + + def __init__(self, latitude_accessor, longitude_accessor, *args, **kwargs): + kwargs.setdefault("accessor", latitude_accessor) + kwargs.setdefault("orderable", True) + super().__init__(*args, **kwargs) + self.latitude_accessor = latitude_accessor + self.longitude_accessor = longitude_accessor + + def render(self, record): + latitude = getattr(record, self.latitude_accessor, None) + longitude = getattr(record, self.longitude_accessor, None) + if latitude is None or longitude is None: + return self.default + return f"{latitude}, {longitude}" + + +class CoordinatesFieldType(FieldType): + """ + A geographic coordinates field. A single field of this type expands into two real + DB columns, ``_latitude`` and ``_longitude``, mirroring NetBox's native + Site/Device coordinate handling (plain ``DecimalField``s; no PostGIS dependency). + """ + + @staticmethod + def latitude_field_name(field): + return f"{field.name}_latitude" + + @staticmethod + def longitude_field_name(field): + return f"{field.name}_longitude" + + def get_model_field(self, field, **kwargs): + return { + self.latitude_field_name(field): models.DecimalField( + verbose_name=_("latitude"), + null=True, + blank=True, + max_digits=8, + decimal_places=6, + validators=[ + MinValueValidator(Decimal("-90.0")), + MaxValueValidator(Decimal("90.0")), + ], + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ), + self.longitude_field_name(field): models.DecimalField( + verbose_name=_("longitude"), + null=True, + blank=True, + max_digits=9, + decimal_places=6, + validators=[ + MinValueValidator(Decimal("-180.0")), + MaxValueValidator(Decimal("180.0")), + ], + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ), + } + + def get_coordinate_values(self, instance, field): + """Return the (latitude, longitude) tuple stored on an instance.""" + return ( + getattr(instance, self.latitude_field_name(field), None), + getattr(instance, self.longitude_field_name(field), None), + ) + + def get_display_value(self, instance, field_name): + latitude = getattr(instance, f"{field_name}_latitude", None) + longitude = getattr(instance, f"{field_name}_longitude", None) + if latitude is None or longitude is None: + return None + return f"{latitude}, {longitude}" + + def get_form_fields(self, field): + """ + Return the two annotated form fields (latitude, longitude) keyed by their + backing column names. The keys match real model columns, so the generated + ModelForm binds and persists them natively. + """ + base_label = field.label or field.name.replace("_", " ").title() + latitude = forms.DecimalField( + label=f"{base_label} ({_('latitude')})", + required=field.required, + max_digits=8, + decimal_places=6, + min_value=Decimal("-90.0"), + max_value=Decimal("90.0"), + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ) + longitude = forms.DecimalField( + label=f"{base_label} ({_('longitude')})", + required=field.required, + max_digits=9, + decimal_places=6, + min_value=Decimal("-180.0"), + max_value=Decimal("180.0"), + help_text=_("GPS coordinate in decimal format (xx.yyyyyy)"), + ) + if field.ui_editable != CustomFieldUIEditableChoices.YES: + latitude.disabled = True + longitude.disabled = True + return { + self.latitude_field_name(field): latitude, + self.longitude_field_name(field): longitude, + } + + def get_filterform_field(self, field, **kwargs): + base_label = field.label or field.name.replace("_", " ").title() + return { + self.latitude_field_name(field): forms.DecimalField( + label=f"{base_label} ({_('latitude')})", required=False + ), + self.longitude_field_name(field): forms.DecimalField( + label=f"{base_label} ({_('longitude')})", required=False + ), + } + + def get_table_column_field(self, field, **kwargs): + return CoordinatesColumn( + self.latitude_field_name(field), + self.longitude_field_name(field), + verbose_name=str(field), + ) + + @staticmethod + def validate_pair(latitude, longitude): + """ + Enforce that latitude and longitude are either both set or both empty. + Raises ``django.core.exceptions.ValidationError`` on a half-populated pair. + """ + if (latitude is None) != (longitude is None): + raise ValidationError( + _("Latitude and longitude must both be set or both be empty.") + ) + + FIELD_TYPE_CLASS = { CustomFieldTypeChoices.TYPE_TEXT: TextFieldType, CustomFieldTypeChoices.TYPE_LONGTEXT: LongTextFieldType, @@ -1994,4 +2138,5 @@ def __get__(self, instance, owner=None): CustomFieldTypeChoices.TYPE_MULTISELECT: MultiSelectFieldType, CustomFieldTypeChoices.TYPE_OBJECT: ObjectFieldType, CustomFieldTypeChoices.TYPE_MULTIOBJECT: MultiObjectFieldType, + CustomObjectFieldTypeChoices.TYPE_COORDINATES: CoordinatesFieldType, } diff --git a/netbox_custom_objects/filtersets.py b/netbox_custom_objects/filtersets.py index 62b90173..e318a055 100644 --- a/netbox_custom_objects/filtersets.py +++ b/netbox_custom_objects/filtersets.py @@ -11,6 +11,7 @@ from extras.choices import CustomFieldTypeChoices from netbox.filtersets import NetBoxModelFilterSet +from .choices import CustomObjectFieldTypeChoices from .constants import APP_LABEL from .models import CustomObjectType @@ -339,6 +340,18 @@ def build_filter_for_field(field) -> dict: ): return _build_polymorphic_filters(field) + # Coordinates expand into two real columns; register a numeric filter for each. + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + base_label = field.label or field.name + return { + f"{field.name}_latitude": django_filters.NumberFilter( + field_name=f"{field.name}_latitude", label=f"{base_label} (latitude)" + ), + f"{field.name}_longitude": django_filters.NumberFilter( + field_name=f"{field.name}_longitude", label=f"{base_label} (longitude)" + ), + } + spec = FIELD_TYPE_FILTERS.get(field.type) if not spec: return {} diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index a31bc4f8..c49e16d5 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -60,7 +60,8 @@ from utilities.string import title from utilities.validators import validate_regex -from netbox_custom_objects.choices import ObjectFieldOnDeleteChoices +from netbox_custom_objects.choices import (CustomObjectFieldTypeChoices, + ObjectFieldOnDeleteChoices) from netbox_custom_objects.constants import APP_LABEL, RESERVED_FIELD_NAMES from netbox_custom_objects.field_types import ( FIELD_TYPE_CLASS, LazyForeignKey, safe_table_name, @@ -172,12 +173,20 @@ def clone_fields(self): if not hasattr(self, "custom_object_type_id"): return () - # Get all field names where is_cloneable=True for this custom object type + # Get all fields where is_cloneable=True for this custom object type cloneable_fields = self.custom_object_type.fields.filter( is_cloneable=True - ).values_list("name", flat=True) + ).values_list("name", "type") - return tuple(cloneable_fields) + names = [] + for name, field_type in cloneable_fields: + # Coordinates fields have no single column; clone the two backing columns. + if field_type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + names += [f"{name}_latitude", f"{name}_longitude"] + else: + names.append(name) + + return tuple(names) def get_absolute_url(self): return reverse( @@ -1090,7 +1099,7 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode type = models.CharField( verbose_name=_("type"), max_length=50, - choices=CustomFieldTypeChoices, + choices=CustomObjectFieldTypeChoices, default=CustomFieldTypeChoices.TYPE_TEXT, help_text=_("The type of data this custom object field holds"), ) @@ -1529,6 +1538,18 @@ def clean(self): {"unique": _("Uniqueness cannot be enforced for boolean or multiobject fields")} ) + # Coordinates fields expand into two columns and have no single value, so a + # number of single-value options do not apply. + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + if self.unique: + raise ValidationError( + {"unique": _("Uniqueness cannot be enforced for coordinates fields")} + ) + if self.default is not None: + raise ValidationError( + {"default": _("A default value cannot be set for coordinates fields")} + ) + # Check if uniqueness constraint can be applied when changing from non-unique to unique if ( self.pk @@ -2288,7 +2309,12 @@ def save(self, *args, **kwargs): with connection.schema_editor() as schema_editor: if self._state.adding: - if self.is_polymorphic: + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + # Coordinates expand into two concrete columns (latitude/longitude). + for column_name, model_field in field_type.get_model_field(self).items(): + model_field.contribute_to_class(model, column_name) + schema_editor.add_field(model, model_field) + elif self.is_polymorphic: # Polymorphic Object: add content_type + object_id columns + index # Polymorphic MultiObject: create through table with content_type + object_id if self.type == CustomFieldTypeChoices.TYPE_OBJECT: @@ -2323,11 +2349,21 @@ def save(self, *args, **kwargs): if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: field_type.create_m2m_table(self, model, self.name) else: + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + # Only a rename touches the schema; other attribute changes + # (label, description, …) are persisted by super().save() below. + if self.name != self._original_name: + new_fields = list(field_type.get_model_field(self).items()) + old_fields = list(field_type.get_model_field(self.original).items()) + for (old_col, old_field), (new_col, new_field) in zip(old_fields, new_fields): + old_field.contribute_to_class(model, old_col) + new_field.contribute_to_class(model, new_col) + schema_editor.alter_field(model, old_field, new_field) # Polymorphic fields: renames and type changes are rejected by clean(). # Non-schema attributes (label, description, …) may still change here. # If clean() was bypassed and a rename slipped through, raise rather # than silently leaving DB columns / through table out of sync. - if self.is_polymorphic or self._original_is_polymorphic: + elif self.is_polymorphic or self._original_is_polymorphic: if self.name != self._original_name: raise ValidationError( {"name": _("Cannot rename a polymorphic field after creation.")} @@ -2507,7 +2543,12 @@ def delete(self, *args, **kwargs): _unwire_polymorphic_reverse_descriptors(self) with connection.schema_editor() as schema_editor: - if self.is_polymorphic: + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + # Drop both backing columns (latitude/longitude). + for column_name, model_field in field_type.get_model_field(self).items(): + model_field.contribute_to_class(model, column_name) + schema_editor.remove_field(model, model_field) + elif self.is_polymorphic: if self.type == CustomFieldTypeChoices.TYPE_OBJECT: field_type.remove_polymorphic_object_columns(self, model, schema_editor) elif self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html index 0be4c39c..754d5746 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html @@ -123,7 +123,24 @@ {% endif %} - {% customfield_value field object|get_field_value:field %} + {% if field.type == 'coordinates' %} + {% with coords=object|get_field_value:field %} + {% if coords %} + {{ coords }} + {% with map_url=object|get_coordinate_map_url:field %} + {% if map_url %} + + {% trans "Map" %} + + {% endif %} + {% endwith %} + {% else %} + {{ ''|placeholder }} + {% endif %} + {% endwith %} + {% else %} + {% customfield_value field object|get_field_value:field %} + {% endif %} {% endif %} diff --git a/netbox_custom_objects/templatetags/custom_object_utils.py b/netbox_custom_objects/templatetags/custom_object_utils.py index bfeae6b2..e0c65911 100644 --- a/netbox_custom_objects/templatetags/custom_object_utils.py +++ b/netbox_custom_objects/templatetags/custom_object_utils.py @@ -1,7 +1,9 @@ from django import template -from extras.choices import CustomFieldTypeChoices, CustomFieldUIVisibleChoices +from extras.choices import CustomFieldUIVisibleChoices +from netbox_custom_objects.choices import CustomObjectFieldTypeChoices from netbox_custom_objects.models import CustomObjectTypeField +from netbox_custom_objects.utilities import build_map_url __all__ = ( "get_field_object_type", @@ -9,11 +11,12 @@ "get_field_value", "get_field_is_ui_visible", "get_child_relations", + "get_coordinate_map_url", ) register = template.Library() -custom_field_type_verbose_names = {c[0]: c[1] for c in CustomFieldTypeChoices.CHOICES} +custom_field_type_verbose_names = {c[0]: c[1] for c in CustomObjectFieldTypeChoices.CHOICES} @register.filter(name="get_field_object_type") @@ -28,15 +31,36 @@ def get_field_type_verbose_name(field: CustomObjectTypeField) -> str: @register.filter(name="get_field_value") def get_field_value(obj, field: CustomObjectTypeField): + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + latitude = getattr(obj, f"{field.name}_latitude", None) + longitude = getattr(obj, f"{field.name}_longitude", None) + if latitude is None or longitude is None: + return None + return f"{latitude}, {longitude}" return getattr(obj, field.name) +@register.filter(name="get_coordinate_map_url") +def get_coordinate_map_url(obj, field: CustomObjectTypeField): + """Return the external map URL for a coordinates field, or None.""" + if field.type != CustomObjectFieldTypeChoices.TYPE_COORDINATES: + return None + latitude = getattr(obj, f"{field.name}_latitude", None) + longitude = getattr(obj, f"{field.name}_longitude", None) + return build_map_url(latitude, longitude) + + @register.filter(name="get_field_is_ui_visible") def get_field_is_ui_visible(obj, field: CustomObjectTypeField) -> bool: if field.ui_visible == CustomFieldUIVisibleChoices.ALWAYS: return True - if field.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + if field.type == CustomObjectFieldTypeChoices.TYPE_MULTIOBJECT: field_value = getattr(obj, field.name).exists() + elif field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + field_value = ( + getattr(obj, f"{field.name}_latitude", None) is not None + and getattr(obj, f"{field.name}_longitude", None) is not None + ) else: field_value = getattr(obj, field.name) if field.ui_visible == CustomFieldUIVisibleChoices.IF_SET and field_value: diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index 3eb5a467..469cbee6 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -2,6 +2,7 @@ Tests for API code paths. """ import uuid +from decimal import Decimal from django.test import TestCase, RequestFactory from django.urls import reverse @@ -1853,3 +1854,87 @@ def test_post_null_required_object_field_rejected(self): format='json', ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + + +class CoordinatesFieldAPITest(CustomObjectsTestCase, NetBoxTestCase): + """REST API behaviour for the coordinates field type.""" + + @classmethod + def setUpTestData(cls): + cls.cot = CustomObjectType.objects.create( + name="GeoObject", + verbose_name_plural="Geo Objects", + slug="geo-objects", + ) + cls.create_custom_object_type_field( + cls.cot, name="name", type="text", primary=True, required=True + ) + cls.create_custom_object_type_field(cls.cot, name="location", type="coordinates") + cls.model = cls.cot.get_model() + + def setUp(self): + super().setUp() + self.user = create_test_user("geouser") + self.client = APIClient() + token_key = create_token(self.user) + self.header = {"HTTP_AUTHORIZATION": f"Token {token_key}"} + perm = ObjectPermission( + name="geo all", actions=["view", "add", "change", "delete"] + ) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + + def _list_url(self): + return reverse( + "plugins-api:netbox_custom_objects-api:customobject-list", + kwargs={"custom_object_type": self.cot.slug}, + ) + + def _detail_url(self, instance): + return reverse( + "plugins-api:netbox_custom_objects-api:customobject-detail", + kwargs={"pk": instance.pk, "custom_object_type": self.cot.slug}, + ) + + def test_serializer_exposes_flat_latitude_longitude(self): + """The coordinate columns are serialized as two flat fields (NetBox convention).""" + obj = self.model.objects.create( + name="Box", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + response = self.client.get(self._detail_url(obj), **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + self.assertEqual(Decimal(response.data["location_latitude"]), Decimal("40.712800")) + self.assertEqual(Decimal(response.data["location_longitude"]), Decimal("-74.006000")) + self.assertNotIn("location", response.data) + + def test_create_with_coordinates(self): + """Creating an object via the API persists both coordinate columns.""" + data = { + "name": "Created box", + "location_latitude": "41.000000", + "location_longitude": "-75.000000", + } + response = self.client.post(self._list_url(), data, format="json", **self.header) + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + obj = self.model.objects.get(pk=response.data["id"]) + self.assertEqual(obj.location_latitude, Decimal("41.000000")) + self.assertEqual(obj.location_longitude, Decimal("-75.000000")) + + def test_create_half_populated_pair_rejected(self): + """Setting only one of latitude/longitude is rejected.""" + data = {"name": "Bad box", "location_latitude": "41.000000"} + response = self.client.post(self._list_url(), data, format="json", **self.header) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + + def test_create_out_of_range_rejected(self): + """Latitude beyond ±90 is rejected by the model field validators.""" + data = { + "name": "Out of range", + "location_latitude": "120.000000", + "location_longitude": "10.000000", + } + response = self.client.post(self._list_url(), data, format="json", **self.header) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 1fdb734e..3d4a074e 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -11,6 +11,7 @@ from core.models import ObjectType from dcim.models import Device, DeviceType, ModuleType from netbox_custom_objects.field_types import ( + CoordinatesFieldType, MultiObjectFieldType, MultiSelectFieldType, ObjectFieldType, @@ -252,6 +253,98 @@ def test_decimal_field_model_generation(self): self.assertEqual(instance.price, Decimal("25.75")) +class CoordinatesFieldTypeTestCase(FieldTypeTestCase): + """Test cases for the coordinates field type.""" + + def test_coordinates_field_creation(self): + """A coordinates field is created with the expected type.""" + field = self.create_custom_object_type_field( + self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + self.assertEqual(field.type, "coordinates") + + def test_coordinates_field_model_generation(self): + """One coordinates field expands into two backing decimal columns.""" + self.create_custom_object_type_field( + self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + + model = self.custom_object_type.get_model() + column_names = {f.name for f in model._meta.local_fields} + self.assertIn("location_latitude", column_names) + self.assertIn("location_longitude", column_names) + # The logical field name itself is not a real column. + self.assertNotIn("location", column_names) + + # Verify precision/validators mirror NetBox core's Site coordinates. + latitude = model._meta.get_field("location_latitude") + longitude = model._meta.get_field("location_longitude") + self.assertEqual((latitude.max_digits, latitude.decimal_places), (8, 6)) + self.assertEqual((longitude.max_digits, longitude.decimal_places), (9, 6)) + + instance = model.objects.create( + name="Test", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + self.assertEqual(instance.location_latitude, Decimal("40.712800")) + self.assertEqual(instance.location_longitude, Decimal("-74.006000")) + + def test_coordinates_display_value(self): + """get_display_value combines both columns, or returns None when unset.""" + self.create_custom_object_type_field( + self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + model = self.custom_object_type.get_model() + field_type = CoordinatesFieldType() + + populated = model.objects.create( + name="A", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + self.assertEqual( + field_type.get_display_value(populated, "location"), + "40.712800, -74.006000", + ) + + empty = model.objects.create(name="B") + self.assertIsNone(field_type.get_display_value(empty, "location")) + + def test_coordinates_validate_pair(self): + """Latitude and longitude must both be set or both be empty.""" + field_type = CoordinatesFieldType() + # Both set / both empty are valid. + field_type.validate_pair(Decimal("1.0"), Decimal("2.0")) + field_type.validate_pair(None, None) + # Half-populated pairs are rejected. + with self.assertRaises(ValidationError): + field_type.validate_pair(Decimal("1.0"), None) + with self.assertRaises(ValidationError): + field_type.validate_pair(None, Decimal("2.0")) + + def test_coordinates_field_rejects_unique(self): + """A coordinates field cannot be marked unique.""" + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location", + label="Location", + type="coordinates", + unique=True, + ) + with self.assertRaises(ValidationError): + field.full_clean() + + class BooleanFieldTypeTestCase(FieldTypeTestCase): """Test cases for boolean field type.""" diff --git a/netbox_custom_objects/tests/test_filtersets.py b/netbox_custom_objects/tests/test_filtersets.py index 38058af7..ae4ddf5c 100644 --- a/netbox_custom_objects/tests/test_filtersets.py +++ b/netbox_custom_objects/tests/test_filtersets.py @@ -1386,3 +1386,63 @@ def test_form_fields_are_multi_choice(self): self.assertIsInstance(result, dict) for form_field in result.values(): self.assertIsInstance(form_field, DynamicModelMultipleChoiceField) + + +# --------------------------------------------------------------------------- +# CoordinatesFieldType — filter form fields and queryset filtering +# --------------------------------------------------------------------------- + + +class CoordinatesFieldFiltersetTestCase(CustomObjectsTestCase, TestCase): + """A coordinates field yields two numeric filters on its backing columns.""" + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + cls.cot = cls.create_custom_object_type(name="CoordFSTest", slug="coord-fs-test") + cls.create_custom_object_type_field( + cls.cot, name="name", label="Name", type="text", primary=True, required=True + ) + cls.field = cls.create_custom_object_type_field( + cls.cot, name="location", label="Location", type="coordinates" + ) + + model = cls.cot.get_model() + cls.obj1 = model.objects.create( + name="Box1", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + cls.obj2 = model.objects.create( + name="Box2", + location_latitude=Decimal("51.507400"), + location_longitude=Decimal("-0.127800"), + ) + + def _filterset(self, params): + model = self.cot.get_model() + return get_filterset_class(model)(params, model.objects.all()) + + def test_build_filter_for_field_returns_two_filters(self): + filters = build_filter_for_field(self.field) + self.assertEqual( + set(filters), {"location_latitude", "location_longitude"} + ) + self.assertIsInstance(filters["location_latitude"], django_filters.NumberFilter) + + def test_filter_by_latitude(self): + pks = list( + self._filterset({"location_latitude": "40.712800"}).qs.values_list("pk", flat=True) + ) + self.assertIn(self.obj1.pk, pks) + self.assertNotIn(self.obj2.pk, pks) + + def test_filter_by_longitude(self): + pks = list( + self._filterset({"location_longitude": "-0.127800"}).qs.values_list("pk", flat=True) + ) + self.assertIn(self.obj2.pk, pks) + self.assertNotIn(self.obj1.pk, pks) + + def test_no_filter_returns_all(self): + self.assertEqual(self._filterset({}).qs.count(), 2) diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 26da1e9f..162061d3 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -985,3 +985,62 @@ def test_object_selector_search(self): 'q': 'Alpha', }) self.assertEqual(response.status_code, 200) + + +class CoordinatesFieldViewTest(CustomObjectsTestCase, TestCase): + """UI form behaviour for the coordinates field type.""" + + @classmethod + def setUpTestData(cls): + cls.cot = CustomObjectType.objects.create( + name="GeoView", + verbose_name_plural="Geo Views", + slug="geo-views", + ) + CustomObjectTypeField.objects.create( + custom_object_type=cls.cot, name="name", type="text", primary=True, required=True + ) + CustomObjectTypeField.objects.create( + custom_object_type=cls.cot, name="location", type="coordinates" + ) + cls.model = cls.cot.get_model() + + def setUp(self): + super().setUp() + perm = ObjectPermission( + name="geo view all", actions=["view", "add", "change", "delete"] + ) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + + def _add_url(self): + return reverse( + "plugins:netbox_custom_objects:customobject_add", + kwargs={"custom_object_type": self.cot.slug}, + ) + + def test_add_form_renders_two_inputs(self): + response = self.client.get(self._add_url()) + self.assertEqual(response.status_code, 200) + self.assertContains(response, "location_latitude") + self.assertContains(response, "location_longitude") + + def test_create_valid_coordinates(self): + from decimal import Decimal + data = { + "name": "Box", + "location_latitude": "40.712800", + "location_longitude": "-74.006000", + } + response = self.client.post(self._add_url(), data) + self.assertEqual(response.status_code, 302, getattr(response, "content", b"")) + obj = self.model.objects.get(name="Box") + self.assertEqual(obj.location_latitude, Decimal("40.712800")) + self.assertEqual(obj.location_longitude, Decimal("-74.006000")) + + def test_create_half_populated_pair_rejected(self): + data = {"name": "Bad", "location_latitude": "40.712800"} + response = self.client.post(self._add_url(), data) + self.assertEqual(response.status_code, 200) + self.assertFalse(self.model.objects.filter(name="Bad").exists()) diff --git a/netbox_custom_objects/utilities.py b/netbox_custom_objects/utilities.py index 432b90bb..38ff1762 100644 --- a/netbox_custom_objects/utilities.py +++ b/netbox_custom_objects/utilities.py @@ -9,6 +9,7 @@ __all__ = ( "AppsProxy", + "build_map_url", "extract_cot_id_from_model_name", "generate_model", "get_viewname", @@ -16,6 +17,41 @@ "is_in_branch", ) + +def build_map_url(latitude, longitude): + """ + Build a map URL for a latitude/longitude pair, mirroring NetBox core's handling. + + Uses NetBox's ``MAPS_URL`` configuration parameter (default Google Maps). If that + value contains ``{lat}``/``{lon}`` placeholders they are substituted; otherwise the + coordinates are appended as a comma-separated pair. Returns ``None`` when either + coordinate is unset or no maps URL is configured. + + NetBox's own ``build_coords_url`` helper is used when available (NetBox >= 4.6.2); + on earlier supported versions we replicate its (simple) behaviour locally. + """ + if latitude is None or longitude is None: + return None + + try: + from netbox.config import get_config + + maps_url = get_config().MAPS_URL + except Exception: + maps_url = "https://maps.google.com/?q=" + + if not maps_url: + return None + + try: + from netbox.ui.utils import build_coords_url + + return build_coords_url(maps_url, latitude, longitude) + except ImportError: + if "{lat}" in maps_url or "{lon}" in maps_url: + return maps_url.replace("{lat}", str(latitude)).replace("{lon}", str(longitude)) + return f"{maps_url}{latitude},{longitude}" + # --------------------------------------------------------------------------- # Thread-safe apps.clear_cache suppression # --------------------------------------------------------------------------- @@ -31,6 +67,7 @@ # suppressed, and only for the duration of the critical window. # --------------------------------------------------------------------------- + _suppress_tl = threading.local() # thread-local suppression depth counter _real_clear_cache = None # set by install_clear_cache_suppressor() diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 66cc6a61..15faa3fe 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -35,6 +35,7 @@ from . import field_types, filtersets, forms, tables from .models import CustomObject, CustomObjectType, CustomObjectTypeField from extras.choices import CustomFieldTypeChoices +from netbox_custom_objects.choices import CustomObjectFieldTypeChoices from netbox_custom_objects.constants import APP_LABEL from netbox_custom_objects.dynamic_forms import build_filterset_form_class from netbox_custom_objects.utilities import extract_cot_id_from_model_name, is_in_branch @@ -692,6 +693,8 @@ def get_form(self, model): "custom_object_type_poly_obj_ct_names": set(), # Maps ct_sub → (obj_sub, field_label) for poly object pair rendering in the template "custom_object_type_poly_obj_pairs": {}, + # Maps coordinates field name → (latitude_field_name, longitude_field_name) + "custom_object_type_coordinates_fields": {}, } # Process custom object type fields (with grouping) @@ -701,6 +704,19 @@ def get_form(self, model): field_type = field_types.FIELD_TYPE_CLASS[field.type]() group_name = field.group_name or None + # Coordinates: one logical field rendered as two grouped latitude/longitude inputs + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + sub_fields = field_type.get_form_fields(field) + sub_names = list(sub_fields.keys()) + for sub_name, sub_field in sub_fields.items(): + attrs[sub_name] = sub_field + attrs["custom_object_type_rendered_names"].add(sub_name) + if group_name not in attrs["custom_object_type_field_groups"]: + attrs["custom_object_type_field_groups"][group_name] = [] + attrs["custom_object_type_field_groups"][group_name].extend(sub_names) + attrs["custom_object_type_coordinates_fields"][field.name] = tuple(sub_names) + continue + # Polymorphic single-object: type-selector + object-picker pair if field.is_polymorphic and field.type == CustomFieldTypeChoices.TYPE_OBJECT: ct_sub = f"{field.name}__ct" @@ -771,6 +787,7 @@ def custom_init(self, *args, **kwargs): self.custom_object_type_poly_obj_fields = attrs["custom_object_type_poly_obj_fields"] self.custom_object_type_poly_obj_ct_names = attrs["custom_object_type_poly_obj_ct_names"] self.custom_object_type_poly_obj_pairs = attrs["custom_object_type_poly_obj_pairs"] + self.custom_object_type_coordinates_fields = attrs["custom_object_type_coordinates_fields"] instance = kwargs.get('instance', None) @@ -938,6 +955,15 @@ def custom_clean(self): obj_sub, _("Please select an object of the chosen type."), ) + # Coordinates: latitude and longitude must both be set or both be empty. + for field_name, (lat_name, lon_name) in self.custom_object_type_coordinates_fields.items(): + latitude = self.cleaned_data.get(lat_name) + longitude = self.cleaned_data.get(lon_name) + if (latitude is None) != (longitude is None): + self.add_error( + lon_name if latitude is None else lat_name, + _("Latitude and longitude must both be set or both be empty."), + ) return self.cleaned_data form_class.__init__ = custom_init @@ -1106,6 +1132,15 @@ def get_form(self, queryset): for field in self.custom_object_type.fields.prefetch_related('related_object_types').all(): field_type = field_types.FIELD_TYPE_CLASS[field.type]() + # Coordinates: two optional latitude/longitude inputs in bulk edit + if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + for sub_name, sub_field in field_type.get_form_fields(field).items(): + sub_field.required = False + sub_field.widget.is_required = False + sub_field.initial = None + attrs[sub_name] = sub_field + continue + # Polymorphic single-object: scope-style type-selector + object-picker pair if field.is_polymorphic and field.type == CustomFieldTypeChoices.TYPE_OBJECT: ct_sub = f"{field.name}__ct" From 07c7ffa5680f0f114677d32137cb7b1fef4ace22 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 11:58:06 -0700 Subject: [PATCH 097/115] add migration --- .../migrations/0015_widen_integer_columns.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 netbox_custom_objects/migrations/0015_widen_integer_columns.py diff --git a/netbox_custom_objects/migrations/0015_widen_integer_columns.py b/netbox_custom_objects/migrations/0015_widen_integer_columns.py new file mode 100644 index 00000000..c83990e6 --- /dev/null +++ b/netbox_custom_objects/migrations/0015_widen_integer_columns.py @@ -0,0 +1,64 @@ +""" +Widen existing integer custom-object columns from 32-bit to 64-bit (bigint). + +Integer fields previously mapped to Django's IntegerField, i.e. a 32-bit signed +PostgreSQL ``integer`` column (max 2_147_483_647). They now map to BigIntegerField +(``bigint``). New columns are created as bigint automatically by the schema editor, +but columns on already-created custom_objects_* tables must be widened in place. + +``integer -> bigint`` is a lossless widening (every int32 value fits in int64), so +no USING clause or data transformation is needed. The conversion rewrites the table +under an ACCESS EXCLUSIVE lock; this is fine for typical custom-object table sizes. + +The reverse is intentionally a no-op: narrowing bigint back to integer could fail or +lose data for any value outside the 32-bit range. + +See issue #532. +""" + +from django.db import migrations + + +def widen_integer_columns(apps, schema_editor): + """ALTER every integer-typed custom-object column to bigint, in place.""" + CustomObjectTypeField = apps.get_model("netbox_custom_objects", "CustomObjectTypeField") + + # Drive off field metadata (not blind introspection of every custom_objects_* + # column) so we only touch user integer fields, never base-model columns + # inherited from NetBox mixins. After migration 0014 all field names are + # lowercase and the scalar column name equals the field name exactly. + integer_fields = CustomObjectTypeField.objects.filter(type="integer") + + with schema_editor.connection.cursor() as cursor: + for field in integer_fields: + table_name = f"custom_objects_{field.custom_object_type_id}" + column_name = field.name + + # Idempotent + safe: only act on a column that exists and is still a + # 32-bit integer. A no-op on fresh installs (already bigint) and on + # partial re-runs. + cursor.execute( + """ + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = %s + AND column_name = %s + AND data_type = 'integer' + """, + [table_name, column_name], + ) + if cursor.fetchone(): + cursor.execute( + f'ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE bigint' + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("netbox_custom_objects", "0014_fix_mixed_case_field_names"), + ] + + operations = [ + migrations.RunPython(widen_integer_columns, migrations.RunPython.noop), + ] From 1f515f9ef4e0a0e2fc7265e0c77cc950a8afec48 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 12:51:48 -0700 Subject: [PATCH 098/115] add test --- .../tests/test_field_types.py | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 96e77eef..430befd9 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -1,12 +1,14 @@ """ Tests for all the different field types supported by Custom Object Type Fields. """ +from importlib import import_module from unittest import skip from unittest.mock import Mock from datetime import date, datetime from decimal import Decimal +from django.apps import apps from django.core.exceptions import FieldDoesNotExist, ValidationError -from django.db import models +from django.db import connection, models from django.test import TestCase from core.models import ObjectType @@ -212,6 +214,63 @@ def test_integer_field_is_64_bit(self): instance.refresh_from_db() self.assertEqual(instance.count, big_value) + def test_integer_field_upgrade_widens_existing_32bit_column(self): + """The 0015 migration widens pre-#532 32-bit integer columns to bigint. + + New tables already get a bigint column (test_integer_field_is_64_bit), + but custom_objects_* tables created before issue #532 have a 32-bit + ``integer`` column that the field-type change alone does not alter -- + they are managed=False. This exercises that upgrade path: realize the + column, force it back to 32-bit to mimic a legacy install, run the + migration's data function, and confirm it is widened in place. + """ + field = self.create_custom_object_type_field( + self.custom_object_type, + name="legacy_count", + label="Legacy Count", + type="integer", + ) + + # Realize the backing model/table/column. + model = self.custom_object_type.get_model() + + table_name = f"custom_objects_{self.custom_object_type.pk}" + column_name = field.name + + def column_data_type(): + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT data_type FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = %s AND column_name = %s + """, + [table_name, column_name], + ) + row = cursor.fetchone() + return row[0] if row else None + + # Mimic a legacy install: force the column back to a 32-bit integer. + with connection.cursor() as cursor: + cursor.execute( + f'ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE integer' + ) + self.assertEqual(column_data_type(), "integer") + + # Run the migration's data function against the legacy column. + migration = import_module( + "netbox_custom_objects.migrations.0015_widen_integer_columns" + ) + with connection.schema_editor() as schema_editor: + migration.widen_integer_columns(apps, schema_editor) + + # Column is widened, and a value beyond the 32-bit range round-trips. + self.assertEqual(column_data_type(), "bigint") + big_value = 9_000_000_000 # > 2**31 - 1 (2_147_483_647) + instance = model.objects.create(name="Test", legacy_count=big_value) + instance.refresh_from_db() + self.assertEqual(instance.legacy_count, big_value) + class DecimalFieldTypeTestCase(FieldTypeTestCase): """Test cases for decimal field type.""" From 25ca383d6fff6dda3fa5b270801075d0c8aea84b Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 14:07:28 -0700 Subject: [PATCH 099/115] fixes --- .../migrations/0015_widen_integer_columns.py | 5 ++++- netbox_custom_objects/tests/test_field_types.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/migrations/0015_widen_integer_columns.py b/netbox_custom_objects/migrations/0015_widen_integer_columns.py index c83990e6..4a6e0b59 100644 --- a/netbox_custom_objects/migrations/0015_widen_integer_columns.py +++ b/netbox_custom_objects/migrations/0015_widen_integer_columns.py @@ -48,8 +48,11 @@ def widen_integer_columns(apps, schema_editor): [table_name, column_name], ) if cursor.fetchone(): + # quote_name() both identifiers (consistent with the %s-parameterised + # check above) so a field name containing a quote can't break out. cursor.execute( - f'ALTER TABLE "{table_name}" ALTER COLUMN "{column_name}" TYPE bigint' + f"ALTER TABLE {schema_editor.quote_name(table_name)} " + f"ALTER COLUMN {schema_editor.quote_name(column_name)} TYPE bigint" ) diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 430befd9..f331bc33 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -258,6 +258,10 @@ def column_data_type(): self.assertEqual(column_data_type(), "integer") # Run the migration's data function against the legacy column. + # widen_integer_columns only reads stable CustomObjectTypeField columns + # (type, custom_object_type_id, name), so the current app registry is + # equivalent to the historical one RunPython would pass. If the function + # ever introspects historical field structure, switch to MigrationLoader. migration = import_module( "netbox_custom_objects.migrations.0015_widen_integer_columns" ) From ae37f0a675a856a4ec231ccd20ad57925a095c08 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 14:33:27 -0700 Subject: [PATCH 100/115] fixes --- netbox_custom_objects/models.py | 4 +++ .../tests/test_field_types.py | 16 ++++++++++++ netbox_custom_objects/tests/test_views.py | 26 +++++++++++++++++++ netbox_custom_objects/utilities.py | 2 +- netbox_custom_objects/views.py | 25 +++++++++++++++++- 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index c49e16d5..ae5b2456 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1549,6 +1549,10 @@ def clean(self): raise ValidationError( {"default": _("A default value cannot be set for coordinates fields")} ) + # A coordinates field has no single column matching its name, so it can + # never be added to the search index. Force the weight to 0 rather than + # silently honouring a non-zero value that has no effect. + self.search_weight = 0 # Check if uniqueness constraint can be applied when changing from non-unique to unique if ( diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 3d4a074e..959f3ca4 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -344,6 +344,22 @@ def test_coordinates_field_rejects_unique(self): with self.assertRaises(ValidationError): field.full_clean() + def test_coordinates_field_search_weight_forced_to_zero(self): + """ + A coordinates field has no column matching its name, so it can never be + indexed for search; clean() forces search_weight to 0 rather than honour a + non-zero value that would silently have no effect. + """ + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location", + label="Location", + type="coordinates", + search_weight=500, + ) + field.full_clean() + self.assertEqual(field.search_weight, 0) + class BooleanFieldTypeTestCase(FieldTypeTestCase): """Test cases for boolean field type.""" diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 162061d3..905079d1 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -1044,3 +1044,29 @@ def test_create_half_populated_pair_rejected(self): response = self.client.post(self._add_url(), data) self.assertEqual(response.status_code, 200) self.assertFalse(self.model.objects.filter(name="Bad").exists()) + + def _bulk_edit_url(self): + return reverse( + "plugins:netbox_custom_objects:customobject_bulk_edit", + kwargs={"custom_object_type": self.cot.slug}, + ) + + def test_bulk_edit_half_populated_pair_rejected(self): + """Bulk-editing only one of latitude/longitude is rejected.""" + from decimal import Decimal + obj = self.model.objects.create( + name="Existing", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + data = { + "pk": [obj.pk], + "_apply": "Apply", + "location_latitude": "10.000000", + } + response = self.client.post(self._bulk_edit_url(), data) + self.assertEqual(response.status_code, 200) + obj.refresh_from_db() + # Values are unchanged because the form failed validation. + self.assertEqual(obj.location_latitude, Decimal("40.712800")) + self.assertEqual(obj.location_longitude, Decimal("-74.006000")) diff --git a/netbox_custom_objects/utilities.py b/netbox_custom_objects/utilities.py index 38ff1762..75333447 100644 --- a/netbox_custom_objects/utilities.py +++ b/netbox_custom_objects/utilities.py @@ -37,7 +37,7 @@ def build_map_url(latitude, longitude): from netbox.config import get_config maps_url = get_config().MAPS_URL - except Exception: + except (ImportError, AttributeError): maps_url = "https://maps.google.com/?q=" if not maps_url: diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 15faa3fe..997f5f53 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -961,7 +961,7 @@ def custom_clean(self): longitude = self.cleaned_data.get(lon_name) if (latitude is None) != (longitude is None): self.add_error( - lon_name if latitude is None else lat_name, + lat_name if latitude is None else lon_name, _("Latitude and longitude must both be set or both be empty."), ) return self.cleaned_data @@ -1127,6 +1127,8 @@ def get_form(self, queryset): "custom_object_type_poly_obj_pairs": {}, "custom_object_type_poly_m2m_groups": {}, "custom_object_type_rendered_names": set(), + # field_name → (latitude_field_name, longitude_field_name) + "custom_object_type_coordinates_fields": {}, } for field in self.custom_object_type.fields.prefetch_related('related_object_types').all(): @@ -1134,11 +1136,15 @@ def get_form(self, queryset): # Coordinates: two optional latitude/longitude inputs in bulk edit if field.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + sub_names = [] for sub_name, sub_field in field_type.get_form_fields(field).items(): sub_field.required = False sub_field.widget.is_required = False sub_field.initial = None attrs[sub_name] = sub_field + sub_names.append(sub_name) + # (latitude_name, longitude_name) for cross-field validation below. + attrs["custom_object_type_coordinates_fields"][field.name] = tuple(sub_names) continue # Polymorphic single-object: scope-style type-selector + object-picker pair @@ -1214,6 +1220,23 @@ def bulk_poly_init(self, *args, **kwargs): attrs["__init__"] = bulk_poly_init + coordinates_fields_ref = attrs["custom_object_type_coordinates_fields"] + + def bulk_clean(self): + cleaned_data = NetBoxModelBulkEditForm.clean(self) + # Coordinates: latitude and longitude must both be set or both be empty. + for field_name, (lat_name, lon_name) in coordinates_fields_ref.items(): + latitude = cleaned_data.get(lat_name) + longitude = cleaned_data.get(lon_name) + if (latitude is None) != (longitude is None): + self.add_error( + lat_name if latitude is None else lon_name, + _("Latitude and longitude must both be set or both be empty."), + ) + return cleaned_data + + attrs["clean"] = bulk_clean + form = type( f"{queryset.model._meta.object_name}BulkEditForm", (NetBoxModelBulkEditForm,), From 9ec2a25487c7edceb9e1aa13a86acc92681cc099 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 14:51:37 -0700 Subject: [PATCH 101/115] fixes --- netbox_custom_objects/api/serializers.py | 5 +- netbox_custom_objects/models.py | 46 ++++++++++++++ netbox_custom_objects/tests/test_api.py | 5 +- .../tests/test_field_types.py | 52 +++++++++++++++ .../tests/test_schema_operations.py | 63 +++++++++++++++++++ 5 files changed, 169 insertions(+), 2 deletions(-) diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index b0446dd1..c1ed4d63 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -682,8 +682,11 @@ def validate(self, data): latitude = data.get(lat_field) longitude = data.get(lon_field) if (latitude is None) != (longitude is None): + # Pin the error to the empty field rather than non_field_errors, + # mirroring the UI form's add_error() behaviour. + missing_field = lat_field if latitude is None else lon_field raise serializers.ValidationError( - _("Latitude and longitude must both be set or both be empty.") + {missing_field: [_("Latitude and longitude must both be set or both be empty.")]} ) return data diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index ae5b2456..7087ef05 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1700,6 +1700,52 @@ def clean(self): {"name": _("Cannot rename a polymorphic field after creation.")} ) + # Prevent converting an existing field to or from coordinates. + # + # A coordinates field occupies two concrete columns ("{name}_latitude" and + # "{name}_longitude") while every other field type occupies a single column + # named "{name}". The save() path has no logic to migrate between those two + # shapes, so allowing the conversion would either leave the original column + # orphaned (→ coordinates) or attempt to alter a column that doesn't exist + # (coordinates → other), raising an uncaught database error. Reject it here, + # mirroring the polymorphic-flag guard above. + is_coordinates = self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES + was_coordinates = self._original_type == CustomObjectFieldTypeChoices.TYPE_COORDINATES + if self.pk and not self._state.adding and is_coordinates != was_coordinates: + raise ValidationError( + {"type": _("Cannot change a field's type to or from coordinates after creation.")} + ) + + # Guard against backing-column name collisions. + # + # A coordinates field expands into "{name}_latitude"/"{name}_longitude". If a + # sibling field already occupies one of those column names (or vice versa: a + # plain field named "_latitude"), the schema editor would issue a + # duplicate-column ALTER and PostgreSQL would raise a ProgrammingError instead + # of a clean validation error. Detect the overlap here. + if self.custom_object_type_id: + def _occupied_columns(name, field_type): + if field_type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + return {f"{name}_latitude", f"{name}_longitude"} + return {name} + + # Only worth checking when coordinates columns are involved on either side. + own_columns = _occupied_columns(self.name, self.type) + siblings = self.custom_object_type.fields.all() + if self.pk: + siblings = siblings.exclude(pk=self.pk) + for sibling in siblings: + clash = own_columns & _occupied_columns(sibling.name, sibling.type) + if clash: + raise ValidationError( + { + "name": _( + "Field name conflicts with column '{column}' already used by field " + "'{other}' on this custom object type." + ).format(column=sorted(clash)[0], other=sibling.name) + } + ) + # related_name can only be set for object-type fields if self.related_name and self.type not in ( CustomFieldTypeChoices.TYPE_OBJECT, diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index 469cbee6..1867dbc3 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -1924,10 +1924,13 @@ def test_create_with_coordinates(self): self.assertEqual(obj.location_longitude, Decimal("-75.000000")) def test_create_half_populated_pair_rejected(self): - """Setting only one of latitude/longitude is rejected.""" + """Setting only one of latitude/longitude is rejected, keyed to the empty field.""" data = {"name": "Bad box", "location_latitude": "41.000000"} response = self.client.post(self._list_url(), data, format="json", **self.header) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + # The error is pinned to the missing field, not non_field_errors. + self.assertIn("location_longitude", response.data) + self.assertNotIn("non_field_errors", response.data) def test_create_out_of_range_rejected(self): """Latitude beyond ±90 is rejected by the model field validators.""" diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index 959f3ca4..4c6b6659 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -360,6 +360,58 @@ def test_coordinates_field_search_weight_forced_to_zero(self): field.full_clean() self.assertEqual(field.search_weight, 0) + def test_change_existing_field_to_coordinates_rejected(self): + """An existing non-coordinates field cannot be converted to coordinates.""" + field = self.create_custom_object_type_field( + self.custom_object_type, name="location", label="Location", type="text", + ) + field.type = "coordinates" + with self.assertRaises(ValidationError): + field.full_clean() + + def test_change_coordinates_field_to_other_type_rejected(self): + """An existing coordinates field cannot be converted to another type.""" + field = self.create_custom_object_type_field( + self.custom_object_type, name="location", label="Location", type="coordinates", + ) + field.type = "text" + with self.assertRaises(ValidationError): + field.full_clean() + + def test_coordinates_backing_column_collision_rejected(self): + """ + Adding a coordinates field whose backing column collides with an existing + field's column raises a ValidationError rather than a DB error. + """ + self.create_custom_object_type_field( + self.custom_object_type, name="location_latitude", label="Lat", type="text", + ) + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location", + label="Location", + type="coordinates", + ) + with self.assertRaises(ValidationError): + field.full_clean() + + def test_field_colliding_with_coordinates_backing_column_rejected(self): + """ + The reverse collision: a plain field named "_longitude" cannot be + added when a coordinates field "" already exists. + """ + self.create_custom_object_type_field( + self.custom_object_type, name="location", label="Location", type="coordinates", + ) + field = CustomObjectTypeField( + custom_object_type=self.custom_object_type, + name="location_longitude", + label="Lon", + type="text", + ) + with self.assertRaises(ValidationError): + field.full_clean() + class BooleanFieldTypeTestCase(FieldTypeTestCase): """Test cases for boolean field type.""" diff --git a/netbox_custom_objects/tests/test_schema_operations.py b/netbox_custom_objects/tests/test_schema_operations.py index db5f2e42..582cc9c6 100644 --- a/netbox_custom_objects/tests/test_schema_operations.py +++ b/netbox_custom_objects/tests/test_schema_operations.py @@ -8,9 +8,11 @@ from django.apps import apps from django.core.management import call_command +from django.db import connection from django.test import TransactionTestCase from netbox_custom_objects.constants import APP_LABEL +from netbox_custom_objects.models import CustomObjectTypeField from .base import CustomObjectsTestCase, TransactionCleanupMixin @@ -212,3 +214,64 @@ def test_collectstatic_without_database(self): stderr=err, ) # No uncaught exceptions reaching here means success. + + # ------------------------------------------------------------------ + # Coordinates fields expand into two backing columns; verify the schema + # editor manages both on rename and delete. + # ------------------------------------------------------------------ + + def _db_columns(self, model): + """Return the set of actual DB column names for a generated model's table.""" + with connection.cursor() as cursor: + return { + col.name + for col in connection.introspection.get_table_description( + cursor, model._meta.db_table + ) + } + + def test_coordinates_field_rename_renames_both_columns(self): + """Renaming a coordinates field renames both backing DB columns.""" + cot = self.create_custom_object_type(name='coordrename', slug='coord-rename') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, + ) + field = self.create_custom_object_type_field( + cot, name='location', label='Location', type='coordinates', + ) + + columns = self._db_columns(cot.get_model()) + self.assertIn('location_latitude', columns) + self.assertIn('location_longitude', columns) + + # Reload from DB so the rename path has the original snapshot (set in + # from_db) — this mirrors how the edit view loads the field before saving. + field = CustomObjectTypeField.objects.get(pk=field.pk) + field.name = 'geo' + field.save() + + columns = self._db_columns(cot.get_model()) + self.assertNotIn('location_latitude', columns) + self.assertNotIn('location_longitude', columns) + self.assertIn('geo_latitude', columns) + self.assertIn('geo_longitude', columns) + + def test_coordinates_field_delete_drops_both_columns(self): + """Deleting a coordinates field drops both backing DB columns.""" + cot = self.create_custom_object_type(name='coorddelete', slug='coord-delete') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, + ) + field = self.create_custom_object_type_field( + cot, name='location', label='Location', type='coordinates', + ) + + columns = self._db_columns(cot.get_model()) + self.assertIn('location_latitude', columns) + self.assertIn('location_longitude', columns) + + field.delete() + + columns = self._db_columns(cot.get_model()) + self.assertNotIn('location_latitude', columns) + self.assertNotIn('location_longitude', columns) From d9f0f41b2c52d7c18b92ec609b4babaa9a5d2060 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 29 Jun 2026 15:02:04 -0700 Subject: [PATCH 102/115] fixes --- netbox_custom_objects/api/serializers.py | 15 ++++++++-- netbox_custom_objects/models.py | 12 ++++++-- netbox_custom_objects/tests/test_api.py | 38 ++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index c1ed4d63..371754bd 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -677,10 +677,19 @@ def validate(self, data): # Coordinates: latitude and longitude must both be set or both be empty. # On partial updates, only enforce when at least one of the pair is supplied. for lat_field, lon_field in _coordinate_fields: - if lat_field not in data and lon_field not in data: + lat_in_data = lat_field in data + lon_in_data = lon_field in data + if not lat_in_data and not lon_in_data: continue - latitude = data.get(lat_field) - longitude = data.get(lon_field) + # For a PATCH that touches only one column, the other is absent from + # data; fall back to the instance's current DB value so clearing just + # one half of an already-populated pair is still rejected. + latitude = data.get(lat_field) if lat_in_data else ( + getattr(self.instance, lat_field, None) if self.instance else None + ) + longitude = data.get(lon_field) if lon_in_data else ( + getattr(self.instance, lon_field, None) if self.instance else None + ) if (latitude is None) != (longitude is None): # Pin the error to the empty field rather than non_field_errors, # mirroring the UI form's add_error() behaviour. diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 7087ef05..eccc0fa8 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1729,9 +1729,17 @@ def _occupied_columns(name, field_type): return {f"{name}_latitude", f"{name}_longitude"} return {name} - # Only worth checking when coordinates columns are involved on either side. own_columns = _occupied_columns(self.name, self.type) - siblings = self.custom_object_type.fields.all() + # A clash can only involve a coordinates field's expanded columns. If + # this field isn't coordinates, only sibling coordinates fields can + # collide with it — so skip the full sibling scan and query just those + # (usually zero) to avoid an extra queryset on every field's save. + if self.type == CustomObjectFieldTypeChoices.TYPE_COORDINATES: + siblings = self.custom_object_type.fields.all() + else: + siblings = self.custom_object_type.fields.filter( + type=CustomObjectFieldTypeChoices.TYPE_COORDINATES + ) if self.pk: siblings = siblings.exclude(pk=self.pk) for sibling in siblings: diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index 1867dbc3..fda165b5 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -1941,3 +1941,41 @@ def test_create_out_of_range_rejected(self): } response = self.client.post(self._list_url(), data, format="json", **self.header) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + + def test_patch_clearing_one_half_rejected(self): + """A PATCH that clears only one coordinate of a populated pair is rejected.""" + obj = self.model.objects.create( + name="Box", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + response = self.client.patch( + self._detail_url(obj), + {"location_latitude": None}, + format="json", + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.data) + self.assertIn("location_latitude", response.data) + # The DB row is left untouched. + obj.refresh_from_db() + self.assertEqual(obj.location_latitude, Decimal("40.712800")) + self.assertEqual(obj.location_longitude, Decimal("-74.006000")) + + def test_patch_clearing_both_halves_allowed(self): + """A PATCH clearing both coordinates at once is accepted.""" + obj = self.model.objects.create( + name="Box", + location_latitude=Decimal("40.712800"), + location_longitude=Decimal("-74.006000"), + ) + response = self.client.patch( + self._detail_url(obj), + {"location_latitude": None, "location_longitude": None}, + format="json", + **self.header, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK, response.data) + obj.refresh_from_db() + self.assertIsNone(obj.location_latitude) + self.assertIsNone(obj.location_longitude) From 64206f90be625b7920774c738a8f96ce359ad169 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 30 Jun 2026 15:43:27 -0700 Subject: [PATCH 103/115] #98 - Add optional ConfigContext to Custom Objects --- netbox_custom_objects/api/serializers.py | 7 ++ netbox_custom_objects/constants.py | 1 + netbox_custom_objects/forms.py | 15 +++- ...customobjecttype_config_context_enabled.py | 18 ++++ netbox_custom_objects/models.py | 39 ++++++++- .../customobjecttype.html | 4 + netbox_custom_objects/tests/test_api.py | 85 +++++++++++++++++++ netbox_custom_objects/tests/test_models.py | 78 ++++++++++++++++- 8 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index 999138e1..c343488c 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -8,6 +8,7 @@ from django.urls import NoReverseMatch from django.utils.translation import gettext_lazy as _ from extras.choices import CustomFieldTypeChoices +from extras.models import ConfigContextModel from netbox.api.serializers import NetBoxModelSerializer from rest_framework import serializers from rest_framework.exceptions import ValidationError @@ -470,6 +471,12 @@ def get_serializer_class(model, skip_object_fields=False): if not has_owner_field_conflict: base_fields.insert(3, "owner") + # Expose local_context_data when the type opted in to config context support + # (the generated model mixes in ConfigContextModel via + # CustomObjectConfigContextMixin). + if issubclass(model, ConfigContextModel): + base_fields.append("local_context_data") + # Include _context field when the model has designated context fields has_context_fields = bool(getattr(model, '_context_field_ids', [])) if has_context_fields: diff --git a/netbox_custom_objects/constants.py b/netbox_custom_objects/constants.py index e34ee91e..e6e2e435 100644 --- a/netbox_custom_objects/constants.py +++ b/netbox_custom_objects/constants.py @@ -36,6 +36,7 @@ "jobs", "journal_entries", "last_updated", + "local_context_data", "model", "objects", "owner", diff --git a/netbox_custom_objects/forms.py b/netbox_custom_objects/forms.py index d029ba70..1d9bd38a 100644 --- a/netbox_custom_objects/forms.py +++ b/netbox_custom_objects/forms.py @@ -59,7 +59,7 @@ class CustomObjectTypeForm(NetBoxModelForm): fieldsets = ( FieldSet( "name", "verbose_name", "verbose_name_plural", "slug", - "version", "description", "group_name", "tags", + "version", "description", "group_name", "config_context_enabled", "tags", ), ) comments = CommentField() @@ -68,9 +68,20 @@ class Meta: model = CustomObjectType fields = ( "name", "verbose_name", "verbose_name_plural", "slug", "version", "description", - "group_name", "comments", "tags", + "group_name", "config_context_enabled", "comments", "tags", ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # config_context_enabled controls whether the generated model carries a + # local_context_data column, which is created once at type creation. It + # cannot be toggled afterwards, so lock it when editing an existing type. + if self.instance.pk: + self.fields["config_context_enabled"].disabled = True + self.fields["config_context_enabled"].help_text = _( + "Config context support cannot be changed after creation." + ) + class CustomObjectTypeBulkEditForm(NetBoxModelBulkEditForm): description = forms.CharField( diff --git a/netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py b/netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py new file mode 100644 index 00000000..e3a3b15b --- /dev/null +++ b/netbox_custom_objects/migrations/0015_customobjecttype_config_context_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.6 on 2026-06-30 22:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("netbox_custom_objects", "0014_fix_mixed_case_field_names"), + ] + + operations = [ + migrations.AddField( + model_name="customobjecttype", + name="config_context_enabled", + field=models.BooleanField(default=False), + ), + ] diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 43d36f6b..982c5878 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -34,7 +34,7 @@ CustomFieldUIEditableChoices, CustomFieldUIVisibleChoices, ) -from extras.models import CustomField +from extras.models import ConfigContextModel, CustomField from extras.models.customfields import SEARCH_TYPES from extras.utils import is_taggable, run_validators from netbox.config import get_config @@ -971,6 +971,25 @@ def _get_action_url(cls, action=None, rest_api=False, kwargs=None): return reverse(cls._get_viewname(action, rest_api), kwargs=kwargs) +class CustomObjectConfigContextMixin(ConfigContextModel): + """ConfigContextModel variant for dynamically generated custom object models. + + Inherits ``local_context_data`` (a JSONField) and its ``clean()`` validator + from NetBox's ``ConfigContextModel``. However, custom objects have none of + the site/tenant/role dimensions that ``ConfigContext`` assignment rules + filter on, so the parent's ``get_config_context()`` — which calls + ``ConfigContext.objects.get_for_object()`` and dereferences ``obj.site`` / + ``obj.tenant`` / ``obj.role`` — would raise ``AttributeError``. We override + it to return only the locally-defined context data. + """ + + class Meta: + abstract = True + + def get_config_context(self): + return self.local_context_data or {} + + def validate_pep440(value): """Validate that *value* is a valid PEP 440 version string.""" if not value: @@ -1061,6 +1080,14 @@ class CustomObjectType(NetBoxModel): blank=True, editable=False ) + config_context_enabled = models.BooleanField( + default=False, + verbose_name=_("config context support"), + help_text=_( + "Enable local config context data on objects of this type. " + "Can only be set when the type is created." + ), + ) class Meta: verbose_name = "Custom Object Type" @@ -1578,10 +1605,18 @@ def wrapped_post_through_setup(self, cls): TM.post_through_setup = wrapped_post_through_setup + # Optionally mix in config context support (local_context_data) when + # the type opts in. The mixin contributes a concrete column, so the + # flag is only honoured at creation time (the table is built once via + # schema_editor.create_model() and is otherwise managed=False). + bases = (CustomObject, models.Model) + if self.config_context_enabled: + bases = (CustomObject, CustomObjectConfigContextMixin, models.Model) + try: model = generate_model( str(model_name), - (CustomObject, models.Model), + bases, attrs, ) finally: diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html index a6724902..ed0f7b59 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobjecttype.html @@ -37,6 +37,10 @@
    {% trans "Custom Object Type" %}
    {% trans "Description" %} {{ object.description|placeholder }} + + {% trans "Config context support" %} + {% checkmark object.config_context_enabled %} + {% trans "Last activity" %} diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index 0f86c6b8..cf5de120 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -1761,3 +1761,88 @@ def test_filter_by_owner_group_id(self): ids = [r['id'] for r in response.data['results']] self.assertIn(obj_x.pk, ids) self.assertNotIn(obj_y.pk, ids) + + +class ConfigContextAPITest(CustomObjectsTestCase, TestCase): + """REST API exposure of local_context_data for config-context-enabled types (#98).""" + + def setUp(self): + super().setUp() + self.cot = CustomObjectType.objects.create( + name='cc_api', slug='cc-api', config_context_enabled=True, + ) + self.create_custom_object_type_field( + self.cot, name='name', label='Name', type='text', primary=True, required=True, + ) + self.model = self.cot.get_model() + token = create_token(self.user) + self.header = {'HTTP_AUTHORIZATION': f'Token {token}'} + self.client = APIClient() + + def _list_url(self): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-list', + kwargs={'custom_object_type': self.cot.slug}, + ) + + def _detail_url(self, pk): + return reverse( + 'plugins-api:netbox_custom_objects-api:customobject-detail', + kwargs={'pk': pk, 'custom_object_type': self.cot.slug}, + ) + + def _add_perm(self, action): + perm = ObjectPermission(name=f'{action}-{self.model._meta.model_name}', actions=[action]) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(self.model)) + return perm + + def test_local_context_data_present_in_response(self): + obj = self.model.objects.create(name='obj-1', local_context_data={'ntp': ['10.0.0.1']}) + self._add_perm('view') + + response = self.client.get(self._detail_url(obj.pk), **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('local_context_data', response.data) + self.assertEqual(response.data['local_context_data'], {'ntp': ['10.0.0.1']}) + + def test_set_local_context_data_via_patch(self): + obj = self.model.objects.create(name='obj-2') + self._add_perm('change') + + response = self.client.patch( + self._detail_url(obj.pk), + {'local_context_data': {'role': 'edge'}}, + format='json', + **self.header, + ) + self.assertEqual( + response.status_code, + status.HTTP_200_OK, + f'Expected 200; got {response.status_code}: {getattr(response, "data", response.content)}', + ) + obj.refresh_from_db() + self.assertEqual(obj.local_context_data, {'role': 'edge'}) + + def test_local_context_data_absent_when_disabled(self): + """A type without config context support must not expose local_context_data.""" + cot = CustomObjectType.objects.create(name='cc_off', slug='cc-off') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + model = cot.get_model() + obj = model.objects.create(name='x') + + perm = ObjectPermission(name='view-cc-off', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model)) + + url = reverse( + 'plugins-api:netbox_custom_objects-api:customobject-detail', + kwargs={'pk': obj.pk, 'custom_object_type': cot.slug}, + ) + response = self.client.get(url, **self.header) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertNotIn('local_context_data', response.data) diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 2dbe2458..7e653aca 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -386,6 +386,82 @@ def test_stale_registry_entry_causes_relation_error_on_related_object_delete(sel django_apps.clear_cache() +class CustomObjectTypeConfigContextTestCase(CustomObjectsTestCase, TestCase): + """Config context support (issue #98) on custom object types.""" + + @staticmethod + def _table_columns(table_name): + with connection.cursor() as cursor: + return { + col.name + for col in connection.introspection.get_table_description(cursor, table_name) + } + + def test_enabled_model_is_config_context_subclass_with_column(self): + """A config-context-enabled type generates a ConfigContextModel subclass + whose backing table has a local_context_data column.""" + from extras.models import ConfigContextModel + + cot = self.create_custom_object_type( + name="cc_enabled", slug="cc-enabled", config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + self.assertTrue(issubclass(model, ConfigContextModel)) + self.assertIn("local_context_data", self._table_columns(cot.get_database_table_name())) + + def test_disabled_model_has_no_config_context(self): + """The default (flag False) type is not a ConfigContextModel and has no + local_context_data column.""" + from extras.models import ConfigContextModel + + cot = self.create_custom_object_type(name="cc_disabled", slug="cc-disabled") + self.assertFalse(cot.config_context_enabled) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + self.assertFalse(issubclass(model, ConfigContextModel)) + self.assertNotIn("local_context_data", self._table_columns(cot.get_database_table_name())) + + def test_local_context_data_round_trips_and_get_config_context(self): + """An instance stores local_context_data and get_config_context() returns + it without touching the org-dimension aggregation that custom objects lack.""" + cot = self.create_custom_object_type( + name="cc_roundtrip", slug="cc-roundtrip", config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + + obj = model(name="obj-1", local_context_data={"ntp_servers": ["10.0.0.1"]}) + obj.save() + obj.refresh_from_db() + self.assertEqual(obj.local_context_data, {"ntp_servers": ["10.0.0.1"]}) + # Safe override: returns the local data, never raises on missing site/tenant/role. + self.assertEqual(obj.get_config_context(), {"ntp_servers": ["10.0.0.1"]}) + + empty = model(name="obj-2") + empty.save() + self.assertEqual(empty.get_config_context(), {}) + + def test_clean_rejects_non_dict_local_context_data(self): + """The inherited ConfigContextModel.clean() validator rejects non-object JSON.""" + cot = self.create_custom_object_type( + name="cc_clean", slug="cc-clean", config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + obj = model(name="bad", local_context_data=["not", "a", "dict"]) + with self.assertRaises(ValidationError): + obj.clean() + + class CustomObjectTypeFieldTestCase(CustomObjectsTestCase, TestCase): """Test cases for CustomObjectTypeField model.""" @@ -433,7 +509,7 @@ def test_custom_object_type_field_name_validation(self): def test_custom_object_type_field_reserved_name_rejected(self): """Field names in RESERVED_FIELD_NAMES must be rejected with ValidationError.""" - for reserved in ("owner", "tags", "id", "created", "last_updated"): + for reserved in ("owner", "tags", "id", "created", "last_updated", "local_context_data"): with self.assertRaises(ValidationError, msg=f"Expected ValidationError for reserved name={reserved!r}"): field = CustomObjectTypeField( custom_object_type=self.custom_object_type, From 34bb1aa109f06722171620b5436ede95ffd5811e Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 30 Jun 2026 16:22:47 -0700 Subject: [PATCH 104/115] #98 - Add optional ConfigContext to Custom Objects --- .../netbox_custom_objects/customobject.html | 5 ++ .../object_configcontext.html | 19 ++++++ netbox_custom_objects/tests/test_views.py | 65 +++++++++++++++++++ netbox_custom_objects/urls.py | 5 ++ netbox_custom_objects/views.py | 55 ++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html diff --git a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html index 7137cad0..e31faa27 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/customobject.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/customobject.html @@ -75,6 +75,11 @@
  • {{ object.custom_object_type.get_verbose_name }}
    • + {% if object.custom_object_type.config_context_enabled %} +
  • + {% trans "Config Context" %} +
    • + {% endif %}
  • {% trans "Contacts" %}
    • diff --git a/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html new file mode 100644 index 00000000..1ebf3062 --- /dev/null +++ b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html @@ -0,0 +1,19 @@ +{% extends base_template %} +{% load helpers %} +{% load i18n %} + +{% block content %} +
    +
    +
    + {% include 'extras/inc/configcontext_data.html' with title="Local Context Data" data=object.local_context_data format=format copyid="local_context" %} +
    + + + {% trans "Custom objects cannot be assigned source config contexts, so the rendered context is the local context data shown here." %} + +
    +
    +
    +
    +{% endblock %} diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index 6a798f48..aa8995d3 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -983,3 +983,68 @@ def test_quick_add_post_validation_failure_rerenders(self): # No new object created. model = self.target_cot.get_model() self.assertFalse(model.objects.exists()) + + +class CustomObjectConfigContextViewTestCase(CustomObjectsTestCase, TestCase): + """Config context tab on custom object instances (#98).""" + + def _config_context_url(self, cot, instance): + return reverse( + 'plugins:netbox_custom_objects:customobject_configcontext', + kwargs={'custom_object_type': cot.slug, 'pk': instance.pk}, + ) + + def test_tab_returns_200_and_shows_local_data_when_enabled(self): + cot = self.create_custom_object_type( + name='cc_view', slug='cc-view', config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + model = cot.get_model() + obj = model.objects.create(name='obj-1', local_context_data={'ntp_servers': ['10.0.0.1']}) + + response = self.client.get(self._config_context_url(cot, obj)) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'ntp_servers') + self.assertContains(response, 'Config Context') + + def test_tab_link_present_only_when_enabled(self): + """The detail page shows the Config Context tab only for enabled types.""" + enabled = self.create_custom_object_type( + name='cc_on', slug='cc-on', config_context_enabled=True, + ) + self.create_custom_object_type_field( + enabled, name='name', label='Name', type='text', primary=True, required=True, + ) + disabled = self.create_custom_object_type(name='cc_no', slug='cc-no') + self.create_custom_object_type_field( + disabled, name='name', label='Name', type='text', primary=True, required=True, + ) + on_model = enabled.get_model() + off_model = disabled.get_model() + on_obj = on_model.objects.create(name='on') + off_obj = off_model.objects.create(name='off') + + # The detail view (generic.ObjectView) enforces object-level view perms. + perm = ObjectPermission(name='view-cc-detail', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(on_model)) + perm.object_types.add(ObjectType.objects.get_for_model(off_model)) + + on_detail = self.client.get(on_obj.get_absolute_url()) + self.assertContains(on_detail, self._config_context_url(enabled, on_obj)) + + off_detail = self.client.get(off_obj.get_absolute_url()) + self.assertNotContains(off_detail, 'config-context') + + def test_tab_404_when_disabled(self): + cot = self.create_custom_object_type(name='cc_off_view', slug='cc-off-view') + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + obj = cot.get_model().objects.create(name='x') + + response = self.client.get(self._config_context_url(cot, obj)) + self.assertEqual(response.status_code, 404) diff --git a/netbox_custom_objects/urls.py b/netbox_custom_objects/urls.py index 04831bdc..87e755dd 100644 --- a/netbox_custom_objects/urls.py +++ b/netbox_custom_objects/urls.py @@ -84,4 +84,9 @@ views.CustomObjectContactsView.as_view(), name="customobject_contacts", ), + path( + "//config-context/", + views.CustomObjectConfigContextView.as_view(), + name="customobject_configcontext", + ), ] diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 63e9ca8d..5bc1b142 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -8,6 +8,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import router, transaction from django.db.models import ProtectedError, Q, RestrictedError +from django.http import Http404 from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils.html import escape @@ -1532,3 +1533,57 @@ def get(self, request, custom_object_type, **kwargs): "tab": "contacts", }, ) + + +class CustomObjectConfigContextView(ConditionalLoginRequiredMixin, View): + """ + Config context view for CustomObject instances. + + Only available when the instance's CustomObjectType has + ``config_context_enabled=True`` (i.e. the generated model mixes in + ConfigContextModel). Custom objects have no site/tenant/role dimensions, + so there are no source contexts to aggregate — the rendered context is the + object's local_context_data (see CustomObjectConfigContextMixin). + """ + + base_template = None + tab = ViewTab( + label=_("Config Context"), + visible=lambda obj: obj.custom_object_type.config_context_enabled, + weight=2000, + ) + + def get(self, request, custom_object_type, **kwargs): + object_type = get_object_or_404(CustomObjectType, slug=custom_object_type) + if not object_type.config_context_enabled: + raise Http404(_("Config context support is not enabled for this type.")) + model = object_type.get_model_with_serializer() + + lookup_kwargs = {k: v for k, v in kwargs.items() if k != "custom_object_type"} + obj = get_object_or_404(model.objects.all(), **lookup_kwargs) + + # Determine the user's preferred output format (json/yaml), persisting + # an explicit choice the same way NetBox's ObjectConfigContextView does. + if request.GET.get("format") in ("json", "yaml"): + format = request.GET.get("format") + if request.user.is_authenticated: + request.user.config.set("data_format", format, commit=True) + elif request.user.is_authenticated: + format = request.user.config.get("data_format", "json") + else: + format = "json" + + if self.base_template is None: + self.base_template = "netbox_custom_objects/customobject.html" + + return render( + request, + "netbox_custom_objects/object_configcontext.html", + { + "object": obj, + "rendered_context": obj.get_config_context(), + "format": format, + "base_template": self.base_template, + "tab": "configcontext", + }, + ) From d5de3f24fba02b147fe42a53a1dd3d16f363f5c0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 30 Jun 2026 16:36:23 -0700 Subject: [PATCH 105/115] #98 - Add optional ConfigContext to Custom Objects --- netbox_custom_objects/api/serializers.py | 11 ++++ .../object_configcontext.html | 2 +- netbox_custom_objects/tests/test_api.py | 52 +++++++++++++++++++ netbox_custom_objects/tests/test_views.py | 24 +++++++++ netbox_custom_objects/views.py | 4 +- 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/netbox_custom_objects/api/serializers.py b/netbox_custom_objects/api/serializers.py index c343488c..727ba5f5 100644 --- a/netbox_custom_objects/api/serializers.py +++ b/netbox_custom_objects/api/serializers.py @@ -335,6 +335,7 @@ class Meta: "version", "group_name", "description", + "config_context_enabled", "tags", "created", "last_updated", @@ -346,6 +347,16 @@ class Meta: read_only_fields = ("schema_document",) brief_fields = ("id", "url", "name", "slug", "description") + def validate_config_context_enabled(self, value): + # Settable only at creation: the backing local_context_data column is + # created once when the type's table is built, so the flag is immutable + # afterwards (mirrors the disable-on-edit behaviour of the web form). + if self.instance is not None and value != self.instance.config_context_enabled: + raise serializers.ValidationError( + _("Config context support cannot be changed after creation.") + ) + return value + def get_table_model_name(self, obj): return obj.get_table_model_name(obj.id) diff --git a/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html index 1ebf3062..c2fe6678 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html @@ -6,7 +6,7 @@
    - {% include 'extras/inc/configcontext_data.html' with title="Local Context Data" data=object.local_context_data format=format copyid="local_context" %} + {% include 'extras/inc/configcontext_data.html' with title="Local Context Data" data=rendered_context format=format copyid="local_context" %}
    diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index cf5de120..b78430a9 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -678,6 +678,58 @@ def test_group_name_empty_by_default_in_api_response(self): self.assertIn('group_name', response.data) self.assertEqual(response.data['group_name'], '') + def _add_change_permission(self): + obj_perm = ObjectPermission(name='Change permission', actions=['change']) + obj_perm.save() + obj_perm.users.add(self.user) + obj_perm.object_types.add(ObjectType.objects.get_for_model(CustomObjectType)) + + def test_config_context_enabled_set_at_creation(self): + """config_context_enabled can be set to True when creating a type via the API.""" + data = {'name': 'cc_create', 'slug': 'cc-create', 'config_context_enabled': True} + url = reverse('plugins-api:netbox_custom_objects-api:customobjecttype-list') + response = self.client.post(url, data, format='json', **self.header) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.data) + self.assertTrue(response.data['config_context_enabled']) + cot = CustomObjectType.objects.get(pk=response.data['id']) + self.assertTrue(cot.config_context_enabled) + + def test_config_context_enabled_in_detail_response(self): + """The current value is readable via the API detail endpoint.""" + self._add_view_permission() + cot = CustomObjectType.objects.create( + name='cc_detail', slug='cc-detail', config_context_enabled=True, + ) + url = reverse( + 'plugins-api:netbox_custom_objects-api:customobjecttype-detail', + kwargs={'pk': cot.pk}, + ) + response = self.client.get(url, **self.header) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('config_context_enabled', response.data) + self.assertTrue(response.data['config_context_enabled']) + + def test_config_context_enabled_immutable_after_creation(self): + """Changing config_context_enabled via PATCH is rejected (create-only).""" + self._add_change_permission() + cot = CustomObjectType.objects.create( + name='cc_immut', slug='cc-immut', config_context_enabled=False, + ) + url = reverse( + 'plugins-api:netbox_custom_objects-api:customobjecttype-detail', + kwargs={'pk': cot.pk}, + ) + response = self.client.patch( + url, {'config_context_enabled': True}, format='json', **self.header, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn('config_context_enabled', response.data) + cot.refresh_from_db() + self.assertFalse(cot.config_context_enabled) + class CustomObjectTypeFieldObjectResolutionTest(CustomObjectsTestCase, TestCase): """ diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index aa8995d3..fa8f927d 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -994,6 +994,13 @@ def _config_context_url(self, cot, instance): kwargs={'custom_object_type': cot.slug, 'pk': instance.pk}, ) + def _grant_view(self, model): + perm = ObjectPermission(name=f'view-{model._meta.model_name}', actions=['view']) + perm.save() + perm.users.add(self.user) + perm.object_types.add(ObjectType.objects.get_for_model(model)) + return perm + def test_tab_returns_200_and_shows_local_data_when_enabled(self): cot = self.create_custom_object_type( name='cc_view', slug='cc-view', config_context_enabled=True, @@ -1003,12 +1010,29 @@ def test_tab_returns_200_and_shows_local_data_when_enabled(self): ) model = cot.get_model() obj = model.objects.create(name='obj-1', local_context_data={'ntp_servers': ['10.0.0.1']}) + self._grant_view(model) response = self.client.get(self._config_context_url(cot, obj)) self.assertEqual(response.status_code, 200) self.assertContains(response, 'ntp_servers') self.assertContains(response, 'Config Context') + def test_tab_403_without_view_permission(self): + """The tab must honour object-level RBAC, not leak local_context_data.""" + cot = self.create_custom_object_type( + name='cc_rbac', slug='cc-rbac', config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + model = cot.get_model() + obj = model.objects.create(name='secret', local_context_data={'k': 'v'}) + + # No ObjectPermission granted → restricted queryset yields nothing → 404. + response = self.client.get(self._config_context_url(cot, obj)) + self.assertIn(response.status_code, (403, 404)) + self.assertNotContains(response, 'secret', status_code=response.status_code) + def test_tab_link_present_only_when_enabled(self): """The detail page shows the Config Context tab only for enabled types.""" enabled = self.create_custom_object_type( diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 5bc1b142..812c5054 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -1560,7 +1560,9 @@ def get(self, request, custom_object_type, **kwargs): model = object_type.get_model_with_serializer() lookup_kwargs = {k: v for k, v in kwargs.items() if k != "custom_object_type"} - obj = get_object_or_404(model.objects.all(), **lookup_kwargs) + # Gate on object-level view permission so the config-context tab can't + # leak local_context_data past NetBox's RBAC (the detail view restricts too). + obj = get_object_or_404(model.objects.restrict(request.user, "view"), **lookup_kwargs) # Determine the user's preferred output format (json/yaml), persisting # an explicit choice the same way NetBox's ObjectConfigContextView does. From a915db97dcbc2d42f41a81b60949861a03a5fd70 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 30 Jun 2026 16:41:37 -0700 Subject: [PATCH 106/115] #98 - Add optional ConfigContext to Custom Objects --- netbox_custom_objects/tests/test_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index b78430a9..84ec1819 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -721,7 +721,9 @@ def test_config_context_enabled_immutable_after_creation(self): 'plugins-api:netbox_custom_objects-api:customobjecttype-detail', kwargs={'pk': cot.pk}, ) - response = self.client.patch( + # APIClient honours format='json' on PATCH (the plain Django client sends + # octet-stream → 415). + response = APIClient().patch( url, {'config_context_enabled': True}, format='json', **self.header, ) From 58b3fc35029db9f4dd4baf7b84befbabc5e0d039 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 30 Jun 2026 16:54:36 -0700 Subject: [PATCH 107/115] #98 - Add optional ConfigContext to Custom Objects --- netbox_custom_objects/models.py | 18 ++++++++++++++++++ netbox_custom_objects/tests/test_models.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index 982c5878..d66cab47 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1116,6 +1116,24 @@ def clean(self): {"slug": _("Slug field cannot be empty.")} ) + # config_context_enabled is immutable after creation: the local_context_data + # column is created once when the type's table is built (managed=False, no + # migration), so flipping the flag would desync the generated model from the + # physical schema (False→True makes every query raise "column does not exist"). + # Enforce on validated paths (forms, full_clean()); raw QuerySet.update()/save() + # bypass clean() as with any Django model. Skip when None — clean() runs during + # branch revert with attributes hydrated from serialized data (see custom_field_data above). + if self.pk and self.config_context_enabled is not None: + original = CustomObjectType.objects.filter(pk=self.pk).values_list( + "config_context_enabled", flat=True + ).first() + if original is not None and self.config_context_enabled != original: + raise ValidationError({ + "config_context_enabled": _( + "Config context support cannot be changed after creation." + ) + }) + # Enforce max number of COTs that may be created (max_custom_object_types) if not self.pk: max_cots = get_plugin_config("netbox_custom_objects", "max_custom_object_types") diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 7e653aca..0734dd20 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -461,6 +461,22 @@ def test_clean_rejects_non_dict_local_context_data(self): with self.assertRaises(ValidationError): obj.clean() + def test_config_context_enabled_immutable_in_clean(self): + """full_clean() rejects flipping config_context_enabled on an existing type.""" + cot = self.create_custom_object_type(name="cc_clean_immut", slug="cc-clean-immut") + self.assertFalse(cot.config_context_enabled) + cot.config_context_enabled = True + with self.assertRaises(ValidationError): + cot.full_clean() + + def test_config_context_enabled_clean_allows_unchanged(self): + """Re-validating without changing the flag must not raise.""" + cot = self.create_custom_object_type( + name="cc_clean_ok", slug="cc-clean-ok", config_context_enabled=True, + ) + cot.description = "edited" + cot.full_clean() # should not raise + class CustomObjectTypeFieldTestCase(CustomObjectsTestCase, TestCase): """Test cases for CustomObjectTypeField model.""" From 6b251e6ca23ce9d5523252979551669a96b46449 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Jul 2026 10:55:17 -0700 Subject: [PATCH 108/115] #98 - Add optional ConfigContext to Custom Objects --- docs/index.md | 33 +++++++++ netbox_custom_objects/constants.py | 15 ++++ netbox_custom_objects/models.py | 69 +++++++++++++++--- .../object_configcontext.html | 36 +++++++++- netbox_custom_objects/tests/test_models.py | 70 +++++++++++++++++++ netbox_custom_objects/tests/test_views.py | 31 ++++++++ netbox_custom_objects/views.py | 17 +++-- 7 files changed, 255 insertions(+), 16 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6cb4483b..95ad2f82 100644 --- a/docs/index.md +++ b/docs/index.md @@ -127,6 +127,39 @@ As with core NetBox objects, Custom Objects have their own list views. To see al You will see a standard NetBox list view for your new Custom Objects with the standard options including `Configure Table`, `+ Add`, `Import`, `Export`, and others. +### Config Context + +A Custom Object Type can opt in to NetBox's [config context](https://netboxlabs.com/docs/netbox/models/extras/configcontext/) support by checking **Config context support** when the type is created. (This can only be set at creation time — it adds a `local_context_data` column to the type's table, so it cannot be toggled afterwards.) + +When enabled, every object of that type gains: + +- A **Local Context Data** field (arbitrary JSON), stored per object and exposed in the REST API — useful for building configurations (e.g. with Ansible). +- A **Config Context** tab on the object's detail view, showing the rendered context, the local context, and any source contexts. + +#### Aggregating source config contexts + +Custom objects can also inherit the ConfigContexts that apply to NetBox objects they reference, exactly as a Device or VM would. This works **by field-naming convention** — no extra configuration: + +Give the Custom Object Type a single (non-polymorphic) **object** field named one of the following, pointing at the matching NetBox model: + +| Field name | Points at | +|------------|-----------| +| `site` | Site | +| `tenant` | Tenant | +| `role` | Device Role | +| `platform` | Platform | +| `location` | Location | +| `device_type` | Device Type | +| `cluster` | Cluster | + +Any active ConfigContext assigned to the referenced object (or to its region, site group, tenant group, cluster type/group, or matching tags) is then aggregated into the object's rendered context, ordered by weight, with the object's own **Local Context Data** applied last (it always wins). Assigning tags to the custom object also matches tag-scoped ConfigContexts automatically. + +Notes: + +- Field **names are unique per type**, so there is never ambiguity about which field feeds a dimension. +- A field must be an object field with both the **name and target model** above; a mis-named or mis-pointed field is simply ignored. +- If a type defines none of these fields, its rendered context is just its Local Context Data. + ### Deletions #### Deleting Custom Object Types diff --git a/netbox_custom_objects/constants.py b/netbox_custom_objects/constants.py index e6e2e435..15c72a42 100644 --- a/netbox_custom_objects/constants.py +++ b/netbox_custom_objects/constants.py @@ -13,6 +13,21 @@ APP_LABEL = "netbox_custom_objects" +# Convention for config-context aggregation (issue #98): a single, non-polymorphic +# OBJECT field named as the key and pointing at the given (app_label, model) feeds the +# matching ConfigContext assignment dimension read by +# ConfigContextQuerySet.get_for_object(). Region / site-group / tenant-group / +# cluster-type / cluster-group are derived from site/tenant/cluster and need no field. +CONFIG_CONTEXT_DIMENSION_FIELDS = { + "site": ("dcim", "site"), + "tenant": ("tenancy", "tenant"), + "role": ("dcim", "devicerole"), + "platform": ("dcim", "platform"), + "location": ("dcim", "location"), + "device_type": ("dcim", "devicetype"), + "cluster": ("virtualization", "cluster"), +} + # Field names that are reserved and cannot be used for custom object fields. # Keep in alphabetical order for ease of reading error message. RESERVED_FIELD_NAMES = [ diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index d66cab47..ad3842b5 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -34,7 +34,7 @@ CustomFieldUIEditableChoices, CustomFieldUIVisibleChoices, ) -from extras.models import ConfigContextModel, CustomField +from extras.models import ConfigContext, ConfigContextModel, CustomField from extras.models.customfields import SEARCH_TYPES from extras.utils import is_taggable, run_validators from netbox.config import get_config @@ -58,7 +58,7 @@ from netbox.registry import registry from netbox.search import SearchIndex from utilities import filters -from utilities.data import get_config_value_ci +from utilities.data import deepmerge, get_config_value_ci from utilities.datetime import datetime_from_timestamp from utilities.object_types import object_type_name from utilities.querysets import RestrictedQuerySet @@ -67,7 +67,11 @@ from utilities.validators import validate_regex from netbox_custom_objects.choices import ObjectFieldOnDeleteChoices -from netbox_custom_objects.constants import APP_LABEL, RESERVED_FIELD_NAMES +from netbox_custom_objects.constants import ( + APP_LABEL, + CONFIG_CONTEXT_DIMENSION_FIELDS, + RESERVED_FIELD_NAMES, +) from netbox_custom_objects.field_types import FIELD_TYPE_CLASS, LazyForeignKey, safe_table_name from netbox_custom_objects.jobs import ReindexCustomObjectTypeJob from netbox_custom_objects.utilities import ( @@ -975,19 +979,64 @@ class CustomObjectConfigContextMixin(ConfigContextModel): """ConfigContextModel variant for dynamically generated custom object models. Inherits ``local_context_data`` (a JSONField) and its ``clean()`` validator - from NetBox's ``ConfigContextModel``. However, custom objects have none of - the site/tenant/role dimensions that ``ConfigContext`` assignment rules - filter on, so the parent's ``get_config_context()`` — which calls - ``ConfigContext.objects.get_for_object()`` and dereferences ``obj.site`` / - ``obj.tenant`` / ``obj.role`` — would raise ``AttributeError``. We override - it to return only the locally-defined context data. + from NetBox's ``ConfigContextModel``. Custom objects have none of the fixed + site/tenant/role attributes that ``ConfigContext.get_for_object()`` reads, so + we can't hand a custom object to it directly. Instead, by convention (issue + #98), a single non-polymorphic OBJECT field named ``site`` / ``tenant`` / + ``role`` / ``platform`` / ``location`` / ``device_type`` / ``cluster`` and + pointing at the matching NetBox model feeds that assignment dimension. We + build a proxy from those fields and reuse ``get_for_object`` so all of + NetBox's matching, hierarchy walking, and weight ordering apply. When no + such field exists we fall back to local-data-only. """ class Meta: abstract = True + def _config_context_source(self): + """Proxy populated from convention-named OBJECT fields, or None if none match. + + Only honours a field whose *name* and *target model* both match the + convention, so a mistyped/mispointed field is silently ignored rather + than feeding the wrong dimension. + """ + from types import SimpleNamespace + + cot = self.custom_object_type + by_name = { + f.name: f + for f in cot.fields.filter( + type=CustomFieldTypeChoices.TYPE_OBJECT, + is_polymorphic=False, + name__in=CONFIG_CONTEXT_DIMENSION_FIELDS.keys(), + ) + } + dims = {name: None for name in CONFIG_CONTEXT_DIMENSION_FIELDS} + used = False + for name, (app_label, model_name) in CONFIG_CONTEXT_DIMENSION_FIELDS.items(): + f = by_name.get(name) + ct = getattr(f, "related_object_type", None) + if f and ct and (ct.app_label, ct.model) == (app_label, model_name): + dims[name] = getattr(self, name, None) + used = True + if not used: + return None + proxy = SimpleNamespace(**dims) + proxy.tags = self.tags # custom objects are taggable + return proxy + def get_config_context(self): - return self.local_context_data or {} + """Merge ConfigContexts applicable to referenced dimension objects, then + overlay ``local_context_data`` (which takes precedence).""" + data = {} + proxy = self._config_context_source() + if proxy is not None: + contexts = ConfigContext.objects.get_for_object(proxy, aggregate_data=True) or [] + for context in contexts: + data = deepmerge(data, context) + if self.local_context_data: + data = deepmerge(data, self.local_context_data) + return data def validate_pep440(value): diff --git a/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html index c2fe6678..aef91dc6 100644 --- a/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html +++ b/netbox_custom_objects/templates/netbox_custom_objects/object_configcontext.html @@ -6,14 +6,46 @@
    - {% include 'extras/inc/configcontext_data.html' with title="Local Context Data" data=rendered_context format=format copyid="local_context" %} + {% include 'extras/inc/configcontext_data.html' with title="Rendered Context" data=rendered_context format=format copyid="rendered_context" %} +
    +
    +
    +
    + {% include 'extras/inc/configcontext_data.html' with title="Local Context" data=object.local_context_data format=format copyid="local_context" %}
    - {% trans "Custom objects cannot be assigned source config contexts, so the rendered context is the local context data shown here." %} + {% trans "The local config context overwrites all source contexts" %}.
    +
    +

    + {% trans "Source Contexts" %} +
    + {% include 'extras/inc/format_toggle.html' %} +
    +

    + {% for context in source_contexts %} +
    +
    + {{ context.weight }} +
    + {{ context|linkify:"name" }} + {% if context.description %} +
    {{ context.description }} + {% endif %} + {% include 'extras/inc/configcontext_data.html' with data=context.data format=format %} +
    + {% empty %} +
    + {% trans "None found" %} +
    + {% trans "Config contexts are aggregated from objects this type references via fields named site, tenant, role, platform, location, device_type, or cluster." %} +
    +
    + {% endfor %} +
    {% endblock %} diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 0734dd20..9909ac96 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -477,6 +477,76 @@ def test_config_context_enabled_clean_allows_unchanged(self): cot.description = "edited" cot.full_clean() # should not raise + # --- Source-context aggregation via field-naming convention (#98 phase 2) --- + + def _site_cot(self, name, slug): + """Config-context-enabled COT with a primary name field and a `site` object field.""" + cot = self.create_custom_object_type(name=name, slug=slug, config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="site", label="Site", type="object", + related_object_type=self.get_site_object_type(), + ) + return cot + + def test_aggregates_referenced_site_config_context(self): + """A `site` field pulls in ConfigContexts assigned to the referenced Site, + with local_context_data merged on top.""" + from extras.models import ConfigContext + + site = Site.objects.create(name="CC Site", slug="cc-site") + cc = ConfigContext.objects.create(name="cc-site-ctx", weight=1000, is_active=True, + data={"ntp": "10.0.0.1", "dns": "8.8.8.8"}) + cc.sites.add(site) + + cot = self._site_cot("cc_agg_site", "cc-agg-site") + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", site=site, local_context_data={"dns": "1.1.1.1"}) + + rendered = obj.get_config_context() + self.assertEqual(rendered["ntp"], "10.0.0.1") # from source context + self.assertEqual(rendered["dns"], "1.1.1.1") # local overrides source + + def test_no_convention_field_returns_local_only(self): + """Without a convention-named dimension field, no source aggregation happens.""" + from extras.models import ConfigContext + + # A global (unassigned) active context that WOULD match a dimensionful object. + ConfigContext.objects.create(name="global-ctx", weight=1000, is_active=True, data={"g": 1}) + + cot = self.create_custom_object_type(name="cc_nodim", slug="cc-nodim", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", local_context_data={"local": 2}) + + self.assertEqual(obj.get_config_context(), {"local": 2}) + + def test_misnamed_field_is_ignored(self): + """A `site`-named field pointing at the wrong model must not feed the site dimension.""" + from extras.models import ConfigContext + + site = Site.objects.create(name="CC Site 2", slug="cc-site-2") + cc = ConfigContext.objects.create(name="cc-site-ctx-2", weight=1000, is_active=True, data={"x": 1}) + cc.sites.add(site) + + # Field named `site` but pointing at Prefix (wrong model) → ignored. + cot = self.create_custom_object_type(name="cc_wrong", slug="cc-wrong", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="site", label="Site", type="object", + related_object_type=self.get_prefix_object_type(), + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", local_context_data={"only": "local"}) + + self.assertEqual(obj.get_config_context(), {"only": "local"}) + class CustomObjectTypeFieldTestCase(CustomObjectsTestCase, TestCase): """Test cases for CustomObjectTypeField model.""" diff --git a/netbox_custom_objects/tests/test_views.py b/netbox_custom_objects/tests/test_views.py index fa8f927d..12d423ea 100644 --- a/netbox_custom_objects/tests/test_views.py +++ b/netbox_custom_objects/tests/test_views.py @@ -1017,6 +1017,37 @@ def test_tab_returns_200_and_shows_local_data_when_enabled(self): self.assertContains(response, 'ntp_servers') self.assertContains(response, 'Config Context') + def test_tab_lists_aggregated_source_contexts(self): + """A `site` field surfaces the referenced Site's ConfigContext in the tab.""" + from dcim.models import Site + from extras.models import ConfigContext + + site = Site.objects.create(name='Tab Site', slug='tab-site') + cc = ConfigContext.objects.create(name='tab-site-ctx', weight=1000, is_active=True, data={'a': 1}) + cc.sites.add(site) + + cot = self.create_custom_object_type( + name='cc_src', slug='cc-src', config_context_enabled=True, + ) + self.create_custom_object_type_field( + cot, name='name', label='Name', type='text', primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name='site', label='Site', type='object', + related_object_type=self.get_site_object_type(), + ) + model = cot.get_model() + obj = model.objects.create(name='o1', site=site) + self._grant_view(model) + # The Source Contexts panel restricts to ConfigContexts the user may view + # (matching NetBox's ObjectConfigContextView), so grant that too. + self._grant_view(ConfigContext) + + response = self.client.get(self._config_context_url(cot, obj)) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Source Contexts') + self.assertContains(response, 'tab-site-ctx') + def test_tab_403_without_view_permission(self): """The tab must honour object-level RBAC, not leak local_context_data.""" cot = self.create_custom_object_type( diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index 812c5054..bfcca069 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -18,7 +18,7 @@ from django.views.generic import View from extras.choices import CustomFieldUIVisibleChoices from extras.forms import JournalEntryForm -from extras.models import JournalEntry +from extras.models import ConfigContext, JournalEntry from extras.tables import JournalEntryTable from netbox.forms import ( NetBoxModelBulkEditForm, @@ -1541,9 +1541,9 @@ class CustomObjectConfigContextView(ConditionalLoginRequiredMixin, View): Only available when the instance's CustomObjectType has ``config_context_enabled=True`` (i.e. the generated model mixes in - ConfigContextModel). Custom objects have no site/tenant/role dimensions, - so there are no source contexts to aggregate — the rendered context is the - object's local_context_data (see CustomObjectConfigContextMixin). + ConfigContextModel). Source contexts are aggregated from NetBox objects the + custom object references via convention-named fields (site/tenant/role/...); + see CustomObjectConfigContextMixin. local_context_data overrides them. """ base_template = None @@ -1578,12 +1578,21 @@ def get(self, request, custom_object_type, **kwargs): if self.base_template is None: self.base_template = "netbox_custom_objects/customobject.html" + # List the source ConfigContexts feeding the rendered context (restricted + # to what the user may view), via the same proxy the mixin aggregates from. + proxy = obj._config_context_source() + if proxy is not None: + source_contexts = ConfigContext.objects.restrict(request.user, "view").get_for_object(proxy) + else: + source_contexts = ConfigContext.objects.none() + return render( request, "netbox_custom_objects/object_configcontext.html", { "object": obj, "rendered_context": obj.get_config_context(), + "source_contexts": source_contexts, "format": format, "base_template": self.base_template, "tab": "configcontext", From 63585c9ab36dd3e52a3379f233c0cd29026b4239 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Jul 2026 11:10:21 -0700 Subject: [PATCH 109/115] #98 - Add optional ConfigContext to Custom Objects --- docs/index.md | 2 +- netbox_custom_objects/models.py | 20 ++++++++++++++------ netbox_custom_objects/views.py | 7 ++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/index.md b/docs/index.md index 95ad2f82..eb175fac 100644 --- a/docs/index.md +++ b/docs/index.md @@ -158,7 +158,7 @@ Notes: - Field **names are unique per type**, so there is never ambiguity about which field feeds a dimension. - A field must be an object field with both the **name and target model** above; a mis-named or mis-pointed field is simply ignored. -- If a type defines none of these fields, its rendered context is just its Local Context Data. +- If a type defines **none** of these fields, aggregation is skipped entirely and its rendered context is just its Local Context Data. This is a deliberate difference from Devices/VMs: **global (unassigned) ConfigContexts are not applied** to such a type — enabling config context support alone never silently pulls in every global context. Add at least one convention field (e.g. `site`) to opt the type into source aggregation; global contexts then apply too (as they do for any object with a dimension). ### Deletions diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index ad3842b5..1e9b27c8 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -1028,10 +1028,18 @@ def _config_context_source(self): def get_config_context(self): """Merge ConfigContexts applicable to referenced dimension objects, then overlay ``local_context_data`` (which takes precedence).""" + return self._render_config_context(self._config_context_source()) + + def _render_config_context(self, source): + """Merge the ConfigContexts for *source* (a proxy from + ``_config_context_source()``, or ``None``), then overlay + ``local_context_data``. Takes a pre-built *source* so a caller that also + needs the source-context list (the detail tab) can build the proxy once + instead of paying for the ``cot.fields`` lookup twice. + """ data = {} - proxy = self._config_context_source() - if proxy is not None: - contexts = ConfigContext.objects.get_for_object(proxy, aggregate_data=True) or [] + if source is not None: + contexts = ConfigContext.objects.get_for_object(source, aggregate_data=True) or [] for context in contexts: data = deepmerge(data, context) if self.local_context_data: @@ -1170,9 +1178,9 @@ def clean(self): # migration), so flipping the flag would desync the generated model from the # physical schema (False→True makes every query raise "column does not exist"). # Enforce on validated paths (forms, full_clean()); raw QuerySet.update()/save() - # bypass clean() as with any Django model. Skip when None — clean() runs during - # branch revert with attributes hydrated from serialized data (see custom_field_data above). - if self.pk and self.config_context_enabled is not None: + # bypass clean() as with any Django model. (original is None only if the row was + # deleted concurrently — nothing to compare against, so skip.) + if self.pk: original = CustomObjectType.objects.filter(pk=self.pk).values_list( "config_context_enabled", flat=True ).first() diff --git a/netbox_custom_objects/views.py b/netbox_custom_objects/views.py index bfcca069..e9162837 100644 --- a/netbox_custom_objects/views.py +++ b/netbox_custom_objects/views.py @@ -1578,8 +1578,9 @@ def get(self, request, custom_object_type, **kwargs): if self.base_template is None: self.base_template = "netbox_custom_objects/customobject.html" - # List the source ConfigContexts feeding the rendered context (restricted - # to what the user may view), via the same proxy the mixin aggregates from. + # Build the proxy once and reuse it for both the rendered context and the + # (RBAC-restricted) source-context list, instead of letting get_config_context() + # rebuild it internally. proxy = obj._config_context_source() if proxy is not None: source_contexts = ConfigContext.objects.restrict(request.user, "view").get_for_object(proxy) @@ -1591,7 +1592,7 @@ def get(self, request, custom_object_type, **kwargs): "netbox_custom_objects/object_configcontext.html", { "object": obj, - "rendered_context": obj.get_config_context(), + "rendered_context": obj._render_config_context(proxy), "source_contexts": source_contexts, "format": format, "base_template": self.base_template, From b7be3249d1807f3bf3e02206534e7cb521af4c48 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Jul 2026 11:25:37 -0700 Subject: [PATCH 110/115] #98 - Add optional ConfigContext to Custom Objects --- netbox_custom_objects/tests/test_models.py | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/netbox_custom_objects/tests/test_models.py b/netbox_custom_objects/tests/test_models.py index 9909ac96..ca9320f8 100644 --- a/netbox_custom_objects/tests/test_models.py +++ b/netbox_custom_objects/tests/test_models.py @@ -547,6 +547,50 @@ def test_misnamed_field_is_ignored(self): self.assertEqual(obj.get_config_context(), {"only": "local"}) + def test_aggregates_referenced_tenant_config_context(self): + """The `tenant` dimension (a direct obj.tenant read in get_for_object) works.""" + from extras.models import ConfigContext + from tenancy.models import Tenant + + tenant = Tenant.objects.create(name="CC Tenant", slug="cc-tenant") + cc = ConfigContext.objects.create(name="cc-tenant-ctx", weight=1000, is_active=True, data={"t": 1}) + cc.tenants.add(tenant) + + cot = self.create_custom_object_type(name="cc_tenant", slug="cc-tenant", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="tenant", label="Tenant", type="object", + related_object_type=ObjectType.objects.get(app_label="tenancy", model="tenant"), + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", tenant=tenant) + + self.assertEqual(obj.get_config_context(), {"t": 1}) + + def test_aggregates_referenced_role_config_context(self): + """The `role` dimension (a direct obj.role read + get_ancestors) works.""" + from dcim.models import DeviceRole + from extras.models import ConfigContext + + role = DeviceRole.objects.create(name="CC Role", slug="cc-role", color="ff0000") + cc = ConfigContext.objects.create(name="cc-role-ctx", weight=1000, is_active=True, data={"r": 1}) + cc.roles.add(role) + + cot = self.create_custom_object_type(name="cc_role", slug="cc-role", config_context_enabled=True) + self.create_custom_object_type_field( + cot, name="name", label="Name", type="text", primary=True, required=True, + ) + self.create_custom_object_type_field( + cot, name="role", label="Role", type="object", + related_object_type=ObjectType.objects.get(app_label="dcim", model="devicerole"), + ) + model = cot.get_model(no_cache=True) + obj = model.objects.create(name="o1", role=role) + + self.assertEqual(obj.get_config_context(), {"r": 1}) + class CustomObjectTypeFieldTestCase(CustomObjectsTestCase, TestCase): """Test cases for CustomObjectTypeField model.""" From 234b4b8bc8cc9e698971d9d9de8334ffc982fd46 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 10:37:02 -0700 Subject: [PATCH 111/115] merge main into feature --- netbox_custom_objects/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_custom_objects/tests/test_api.py b/netbox_custom_objects/tests/test_api.py index bc05739b..08fccb55 100644 --- a/netbox_custom_objects/tests/test_api.py +++ b/netbox_custom_objects/tests/test_api.py @@ -16,7 +16,7 @@ from core.models import ObjectType from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, Site from extras.models import Tag -from users.models import ObjectPermission, Token +from users.models import ObjectPermission from virtualization.models import Cluster, ClusterType From 7f146da6808a8c64ee7f8ae55b7c2a3f3be52fdd Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 10:56:01 -0700 Subject: [PATCH 112/115] merge main into feature --- netbox_custom_objects/models.py | 13 ++++++++++--- netbox_custom_objects/tests/query_counts.json | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py index a8c8d511..0d1786b8 100644 --- a/netbox_custom_objects/models.py +++ b/netbox_custom_objects/models.py @@ -3552,11 +3552,18 @@ def delete(self, *args, **kwargs): pass field_type.drop_polymorphic_m2m_table(self, model, schema_editor) else: + # _schema_remove_field drops the through table (for MULTIOBJECT) but + # leaves the real through model in apps.all_models; capture it so the + # deregistration below removes it (issue #535). + if self.type == CustomFieldTypeChoices.TYPE_MULTIOBJECT: + try: + _dropped_through_model = model._meta.apps.get_model(APP_LABEL, self.through_model_name) + except LookupError: + pass _schema_remove_field(self, model, schema_editor, schema_conn=schema_conn) - # Deregister the dropped through model (polymorphic MULTIOBJECT case) so the - # cascade-delete collector no longer queries the missing table. Non-polymorphic - # fields are handled inside _schema_remove_field, which self-cleans its registry. + # Deregister the dropped through model (both polymorphic and plain MULTIOBJECT) + # so the cascade-delete collector no longer queries the now-missing table. if _dropped_through_model is not None: through_name = _dropped_through_model.__name__.lower() with CustomObjectType._global_lock: diff --git a/netbox_custom_objects/tests/query_counts.json b/netbox_custom_objects/tests/query_counts.json index eddfc29c..1c266b6b 100644 --- a/netbox_custom_objects/tests/query_counts.json +++ b/netbox_custom_objects/tests/query_counts.json @@ -1,6 +1,6 @@ { - "customobject-complex:list_objects_with_permission": 42, - "customobject-objectfields:list_objects_with_permission": 51, - "customobject-simple:list_objects_with_permission": 34, - "customobjecttype:list_objects_with_permission": 36 + "customobject-complex:list_objects_with_permission": 41, + "customobject-objectfields:list_objects_with_permission": 50, + "customobject-simple:list_objects_with_permission": 33, + "customobjecttype:list_objects_with_permission": 34 } From 1c9b8e1428ed4dbe8f216cec8589a0825ee31da6 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 2 Jul 2026 12:01:27 -0700 Subject: [PATCH 113/115] merge feature --- ...5_widen_integer_columns.py => 0016_widen_integer_columns.py} | 2 +- netbox_custom_objects/tests/test_field_types.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename netbox_custom_objects/migrations/{0015_widen_integer_columns.py => 0016_widen_integer_columns.py} (97%) diff --git a/netbox_custom_objects/migrations/0015_widen_integer_columns.py b/netbox_custom_objects/migrations/0016_widen_integer_columns.py similarity index 97% rename from netbox_custom_objects/migrations/0015_widen_integer_columns.py rename to netbox_custom_objects/migrations/0016_widen_integer_columns.py index 4a6e0b59..9e208eb5 100644 --- a/netbox_custom_objects/migrations/0015_widen_integer_columns.py +++ b/netbox_custom_objects/migrations/0016_widen_integer_columns.py @@ -59,7 +59,7 @@ def widen_integer_columns(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ("netbox_custom_objects", "0014_fix_mixed_case_field_names"), + ("netbox_custom_objects", "0015_customobjecttype_config_context_enabled"), ] operations = [ diff --git a/netbox_custom_objects/tests/test_field_types.py b/netbox_custom_objects/tests/test_field_types.py index de2f3810..9bf5c59d 100644 --- a/netbox_custom_objects/tests/test_field_types.py +++ b/netbox_custom_objects/tests/test_field_types.py @@ -263,7 +263,7 @@ def column_data_type(): # equivalent to the historical one RunPython would pass. If the function # ever introspects historical field structure, switch to MigrationLoader. migration = import_module( - "netbox_custom_objects.migrations.0015_widen_integer_columns" + "netbox_custom_objects.migrations.0016_widen_integer_columns" ) with connection.schema_editor() as schema_editor: migration.widen_integer_columns(apps, schema_editor) From 22967fd95b8590c813236003d44d92e180822e46 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 2 Jul 2026 18:29:36 -0400 Subject: [PATCH 114/115] Release v0.6.0 Co-Authored-By: Claude Sonnet 4.6 --- docs/releases.md | 20 ++++++++++++++++++++ netbox_custom_objects/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index f445998e..fe3a9b93 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,5 +1,25 @@ # Releases +## 0.6.0 + +### Enhancements + +- [#30](https://github.com/netboxlabs/netbox-custom-objects/issues/30) - GraphQL support for Custom Objects +- [#98](https://github.com/netboxlabs/netbox-custom-objects/issues/98) - Config context support for Custom Objects +- [#254](https://github.com/netboxlabs/netbox-custom-objects/issues/254) - Quick Add support for object/multiobject fields +- [#286](https://github.com/netboxlabs/netbox-custom-objects/issues/286) - Contacts support for Custom Objects +- [#376](https://github.com/netboxlabs/netbox-custom-objects/issues/376) - Ownership field for Custom Object instances +- [#404](https://github.com/netboxlabs/netbox-custom-objects/issues/404) - Full branching support for Custom Object Types and instances +- [#532](https://github.com/netboxlabs/netbox-custom-objects/issues/532) - Integer fields now use 64-bit `BigIntegerField` to support values exceeding the 32-bit range +- [#551](https://github.com/netboxlabs/netbox-custom-objects/issues/551) - Add `location` field type for latitude/longitude coordinate pairs +- [#569](https://github.com/netboxlabs/netbox-custom-objects/issues/569) - Add version check to gate branching features on required NetBox / netbox-branching versions + +### Bug Fixes + +- [#572](https://github.com/netboxlabs/netbox-custom-objects/issues/572) - Crash on startup when netbox-branching is installed but not listed in `PLUGINS` + +--- + ## 0.5.2 ### New Features diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py index 0845ad8c..a65e667e 100644 --- a/netbox_custom_objects/__init__.py +++ b/netbox_custom_objects/__init__.py @@ -251,7 +251,7 @@ class CustomObjectsPluginConfig(PluginConfig): name = "netbox_custom_objects" verbose_name = "Custom Objects" description = "A plugin to manage custom objects in NetBox" - version = "0.5.2" + version = "0.6.0" author = 'Netbox Labs' author_email = 'support@netboxlabs.com' base_url = "custom-objects" diff --git a/pyproject.toml b/pyproject.toml index cbecc877..33d7b948 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "netboxlabs-netbox-custom-objects" -version = "0.5.2" +version = "0.6.0" description = "A plugin to manage custom objects in NetBox" readme = "README.md" requires-python = ">=3.10" From d2c90e39e7e704be076f0b22a9891dabfda6a0de Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 2 Jul 2026 19:08:10 -0400 Subject: [PATCH 115/115] docs: elevate Branching and GraphQL to New Features subheadings in v0.6.0 notes Co-Authored-By: Claude Sonnet 4.6 --- docs/releases.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index 83812c7d..63b0dcd6 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,17 +2,29 @@ ## 0.6.0 -### Enhancements +### New Features + +**NetBox Branching Support** + +Custom Object Types and their instances now participate fully in NetBox branching workflows. COT schema changes (adding/removing fields, modifying field definitions) and instance creates/updates/deletes are all branch-aware. A version check ensures the minimum required NetBox and netbox-branching versions are present before enabling branching features. + +- [#404](https://github.com/netboxlabs/netbox-custom-objects/issues/404) - Full branching support for Custom Object Types and instances +- [#569](https://github.com/netboxlabs/netbox-custom-objects/issues/569) - Add version check to gate branching features on required NetBox / netbox-branching versions + +**GraphQL Support** + +Custom Object Types and their instances are now queryable and mutable via the NetBox GraphQL API. Queries, mutations, and filtering are supported for all field types. - [#30](https://github.com/netboxlabs/netbox-custom-objects/issues/30) - GraphQL support for Custom Objects + +### Enhancements + - [#98](https://github.com/netboxlabs/netbox-custom-objects/issues/98) - Config context support for Custom Objects - [#254](https://github.com/netboxlabs/netbox-custom-objects/issues/254) - Quick Add support for object/multiobject fields - [#286](https://github.com/netboxlabs/netbox-custom-objects/issues/286) - Contacts support for Custom Objects - [#376](https://github.com/netboxlabs/netbox-custom-objects/issues/376) - Ownership field for Custom Object instances -- [#404](https://github.com/netboxlabs/netbox-custom-objects/issues/404) - Full branching support for Custom Object Types and instances - [#532](https://github.com/netboxlabs/netbox-custom-objects/issues/532) - Integer fields now use 64-bit `BigIntegerField` to support values exceeding the 32-bit range - [#551](https://github.com/netboxlabs/netbox-custom-objects/issues/551) - Add `location` field type for latitude/longitude coordinate pairs -- [#569](https://github.com/netboxlabs/netbox-custom-objects/issues/569) - Add version check to gate branching features on required NetBox / netbox-branching versions ### Bug Fixes