From 21ab7ab9bc19872d68863c0f3457105a82d62da9 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 11 Sep 2025 14:07:01 -0500 Subject: [PATCH 01/17] Merge pull request #6308 from specify/issue-6266 run_key_migration_functions django command --- .env | 2 +- docker-entrypoint.sh | 1 + .../backend/businessrules/migration_utils.py | 71 + .../migrations/0004_catnum_uniquerule.py | 3 +- .../businessrules/migrations/0005_cojo.py | 2 +- .../0008_fix_global_default_rules.py | 18 + .../businessrules/uniqueness_rules.json | 4 +- .../backend/businessrules/uniqueness_rules.py | 117 +- specifyweb/backend/patches/migration_utils.py | 36 + .../migrations/0001_restore_separators.py | 1 + .../migrations/0002_fix_accepted_taxon.py | 10 +- .../migrations/0003_coordinate_fields_fix.py | 25 +- specifyweb/backend/permissions/initialize.py | 687 +++++---- ...add_dataset_create_recordset_permission.py | 26 +- .../0007_add_stats_edit_permission.py | 16 +- .../migrations/0008_attachment_import_role.py | 253 +--- .../backend/workbench/upload/auditlog.py | 11 +- .../__tests__/UploadAttachment.test.tsx | 27 +- .../WorkBench/WbAttachmentsPreview.tsx | 4 +- .../permissions/migration_utils/__init__.py | 0 .../migration_utils/edit_permissions.py | 59 + specifyweb/specify/api/utils.py | 11 + .../commands/run_key_migration_functions.py | 176 +++ .../specify/migration_utils/default_cots.py | 156 ++ .../migration_utils/misc_migrations.py | 8 + .../migration_utils/sp7_schemaconfig.py | 254 +++- .../specify/migration_utils/tectonic_ranks.py | 141 ++ .../migration_utils/update_schema_config.py | 1249 ++++++++++++++++- specifyweb/specify/migrations/0002_geo.py | 106 +- .../migrations/0003_cotype_picklist.py | 71 +- .../migrations/0004_stratigraphy_age.py | 61 +- .../migrations/0007_schema_config_update.py | 152 +- .../migrations/0008_ageCitations_fix.py | 48 +- .../specify/migrations/0009_tectonic_ranks.py | 131 +- .../0012_add_cojo_to_schema_config.py | 23 +- .../0013_collectionobjectgroup_parentcog.py | 33 +- .../migrations/0015_add_version_to_ages.py | 19 +- .../migrations/0017_schemaconfig_fixes.py | 78 +- .../migrations/0018_cot_catnum_schema.py | 62 +- ...add_tectonicunit_to_pc_in_schema_config.py | 23 +- .../0021_update_hidden_geo_tables.py | 94 +- .../migrations/0022_ensure_default_cots.py | 2 +- .../0023_update_schema_config_text.py | 108 +- .../0024_add_uniqueIdentifier_storage.py | 24 +- .../specify/migrations/0027_CO_children.py | 82 +- .../0029_remove_collectionobject_parentco.py | 130 +- .../0031_add_default_for_selectseries.py | 11 +- .../migrations/0032_add_quantities_gift.py | 83 +- .../migrations/0033_update_paleo_desc.py | 22 +- .../migrations/0034_accession_date_fields.py | 92 +- .../migrations/0035_version_required.py | 11 + .../test_create_default_collection_types.py | 8 +- specifyweb/specify/utils/field_change_info.py | 4 +- 53 files changed, 2948 insertions(+), 1898 deletions(-) create mode 100644 specifyweb/backend/businessrules/migration_utils.py create mode 100644 specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py create mode 100644 specifyweb/backend/patches/migration_utils.py create mode 100644 specifyweb/permissions/migration_utils/__init__.py create mode 100644 specifyweb/permissions/migration_utils/edit_permissions.py create mode 100644 specifyweb/specify/management/commands/run_key_migration_functions.py create mode 100644 specifyweb/specify/migration_utils/default_cots.py create mode 100644 specifyweb/specify/migration_utils/misc_migrations.py create mode 100644 specifyweb/specify/migration_utils/tectonic_ranks.py diff --git a/.env b/.env index 48b7d94cfc0..89dea676cc5 100644 --- a/.env +++ b/.env @@ -47,4 +47,4 @@ LOG_LEVEL=WARNING # should only be used during development and troubleshooting and not # during general use. Django applications leak memory when operated # continuously in debug mode. -SP7_DEBUG=true \ No newline at end of file +SP7_DEBUG=true diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e68a3ad3780..5976f21ef5d 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -12,6 +12,7 @@ if [ "$1" = 've/bin/gunicorn' ] || [ "$1" = 've/bin/python' ]; then set +e ve/bin/python manage.py base_specify_migration ve/bin/python manage.py migrate + # ve/bin/python manage.py run_key_migration_functions # Uncomment if you want the key migration functions to run on startup. set -e fi exec "$@" diff --git a/specifyweb/backend/businessrules/migration_utils.py b/specifyweb/backend/businessrules/migration_utils.py new file mode 100644 index 00000000000..38265a6b20c --- /dev/null +++ b/specifyweb/backend/businessrules/migration_utils.py @@ -0,0 +1,71 @@ +from typing import Tuple, List + +from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule + + +def catnum_rule_editable(apps, schema_editor=None): + """ Find any CollectionObject catalogNumber must be unique to Collection + rules which are readonly on the frontend (have isDatabaseConstraint=True) + and set their isDatabaseConstraint=False. + + Generally should be run only after migration businessrules/0003 has been + applied + """ + UniquenessRule = apps.get_model("businessrules", "UniquenessRule") + + model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", isDatabaseConstraint=True) + + catalog_number_rules: List[int] = [] + for rule in model_rules: + rule_fields = rule.uniquenessrulefield_set.all() + + fields = rule_fields.filter(isScope=False) + scopes = rule_fields.filter(isScope=True) + + # We're only interested in the rule "CollectionObject catalogNumber + # must be unique to Collection" + # We check for length of fields and scopes because get() raises an + # exception if more than one result is returned + if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): + catalog_number_rules.append(rule.id) + + rules_to_update = UniquenessRule.objects.filter(id__in=catalog_number_rules) + rules_to_update.update(isDatabaseConstraint=False) + + +def catnum_rule_uneditable(apps, schema_editor=None): + """ Find any CollectionObject catalogNumber must be unique to Collection + rules which are editable on the frontend (have isDatabaseConstraint=False) + and set their isDatabaseConstraint=True. + + Generally should be run when migration businessrules/0003 is being reverted + """ + Discipline = apps.get_model("specify", "Discipline") + UniquenessRule = apps.get_model("businessrules", "UniquenessRule") + + for discipline in Discipline.objects.all(): + model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", discipline_id=discipline.id, isDatabaseConstraint=False) + + has_catalognumber_rule = False + for rule in model_rules: + rule_fields = rule.uniquenessrulefield_set.all() + + fields = rule_fields.filter(isScope=False) + scopes = rule_fields.filter(isScope=True) + + # We're only interested in the rule "CollectionObject catalogNumber + # must be unique to Collection" + # We check for length of fields and scopes because get() raises an + # exception if more than one result is returned + if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): + has_catalognumber_rule = True + + if not has_catalognumber_rule: + create_uniqueness_rule( + "Collectionobject", + discipline=discipline, + is_database_constraint=True, + fields=["catalogNumber"], + scopes=["collection"], + registry=apps, + ) diff --git a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py index fd1e3fe66ec..33f2798cb81 100644 --- a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py +++ b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py @@ -1,7 +1,6 @@ -from typing import Tuple - from django.db import migrations +from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable, catnum_rule_uneditable from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule diff --git a/specifyweb/backend/businessrules/migrations/0005_cojo.py b/specifyweb/backend/businessrules/migrations/0005_cojo.py index e9440465fd8..c006d850af6 100644 --- a/specifyweb/backend/businessrules/migrations/0005_cojo.py +++ b/specifyweb/backend/businessrules/migrations/0005_cojo.py @@ -6,7 +6,7 @@ Applies the COJO uniqueness rule to the database. """ def apply_migration(apps, schema_editor): - cojo_rules = DEFAULT_UNIQUENESS_RULES["CollectionObjectGroupJoin"] + cojo_rules = DEFAULT_UNIQUENESS_RULES["Collectionobjectgroupjoin"] Discipline = apps.get_model('specify', 'Discipline') for rule in cojo_rules: diff --git a/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py b/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py new file mode 100644 index 00000000000..49208174a35 --- /dev/null +++ b/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py @@ -0,0 +1,18 @@ +import logging +from django.db import migrations +from specifyweb.backend.businessrules.uniqueness_rules import fix_global_default_rules + +logger = logging.getLogger(__name__) + +def apply_migration(apps, schema_editor): + fix_global_default_rules(apps) + +class Migration(migrations.Migration): + + dependencies = [ + ('businessrules', '0007_more_uniqueness_rules'), + ] + + operations = [ + migrations.RunPython(apply_migration, migrations.RunPython.noop, atomic=True) + ] diff --git a/specifyweb/backend/businessrules/uniqueness_rules.json b/specifyweb/backend/businessrules/uniqueness_rules.json index 8182b89dccd..575f4b9f31e 100644 --- a/specifyweb/backend/businessrules/uniqueness_rules.json +++ b/specifyweb/backend/businessrules/uniqueness_rules.json @@ -78,7 +78,7 @@ "Collectionobject": [ { "rule": [["catalogNumber"], ["collection"]], - "isDatabaseConstraint": true + "isDatabaseConstraint": false }, { "rule": [["uniqueIdentifier"], []], @@ -89,7 +89,7 @@ "isDatabaseConstraint": false } ], - "CollectionObjectGroupJoin": [ + "Collectionobjectgroupjoin": [ { "rule": [["childCo"], []], "isDatabaseConstraint": true diff --git a/specifyweb/backend/businessrules/uniqueness_rules.py b/specifyweb/backend/businessrules/uniqueness_rules.py index a02d4a8220d..fb3f055dbce 100644 --- a/specifyweb/backend/businessrules/uniqueness_rules.py +++ b/specifyweb/backend/businessrules/uniqueness_rules.py @@ -6,13 +6,14 @@ from collections.abc import Iterable from django.apps import apps -from django.db import connections -from django.db.models import Q, Count +from django.db import connections, transaction +from django.db.models import Q, Count, Exists, OuterRef from django.db.migrations.recorder import MigrationRecorder from django.core.exceptions import ObjectDoesNotExist from specifyweb.specify.api.crud import get_model from specifyweb.specify.datamodel import datamodel from specifyweb.middleware.general import serialize_django_obj +from specifyweb.specify.models import Discipline from specifyweb.specify.utils.scoping import in_same_scope from .orm_signal_handler import orm_signal_handler from .exceptions import BusinessRuleException @@ -73,7 +74,8 @@ def validate_unique(model, instance): rules = UniquenessRule.objects.filter(modelName=model_name) for rule in rules: rule_fields = UniquenessRuleField.objects.filter(uniquenessrule=rule) - if not rule_is_global(tuple(field.fieldPath for field in rule_fields.filter(isScope=True))) and not in_same_scope(rule, instance): + if not rule_is_global(tuple(field.fieldPath for field in rule_fields.filter(isScope=True))) \ + and not in_same_scope(rule, instance): continue field_names = [ @@ -152,7 +154,8 @@ class UniquenessCheck(TypedDict): fields: list[ViolatedUniquenessCheck] -def check_uniqueness(model_name: str, raw_fields: list[str], raw_scopes: list[str], registry=None) -> UniquenessCheck | None: +def check_uniqueness(model_name: str, raw_fields: list[str], raw_scopes: list[str], registry=None) \ + -> UniquenessCheck | None: """ Given a model, a list of fields, and a list of scopes, check whether there are models of model_name which have duplicate values of fields in scopes. @@ -180,16 +183,32 @@ def check_uniqueness(model_name: str, raw_fields: list[str], raw_scopes: list[st duplicates_field = '__duplicates' - duplicates = django_model.objects.values( - *all_fields).annotate(**{duplicates_field: Count('id')}).filter(strict_filters).filter(**{f"{duplicates_field}__gt": 1}).order_by(f'-{duplicates_field}') + duplicates = ( + django_model.objects + .values(*all_fields) + .annotate(**{duplicates_field: Count('id')}) + .filter(strict_filters) + .filter(**{f"{duplicates_field}__gt": 1}) + .order_by(f'-{duplicates_field}') + ) total_duplicates = sum(duplicate[duplicates_field] for duplicate in duplicates) final = { "totalDuplicates": total_duplicates, - "fields": [{"duplicates": duplicate[duplicates_field], "fields": {field: value for field, value in duplicate.items() if field != duplicates_field}} - for duplicate in duplicates]} + "fields": [ + { + "duplicates": duplicate[duplicates_field], + "fields": { + field: value + for field, value in duplicate.items() + if field != duplicates_field + }, + } + for duplicate in duplicates + ], + } return final @@ -224,7 +243,6 @@ def serialize_multiple_django(matchable, field_map, fields): def join_with_and(fields): return ' and '.join(fields) - def apply_default_uniqueness_rules(discipline, registry=None): for table, rules in DEFAULT_UNIQUENESS_RULES.items(): model_name = getattr(datamodel.get_table(table), "django_name", None) @@ -234,8 +252,7 @@ def apply_default_uniqueness_rules(discipline, registry=None): fields, scopes = rule["rule"] isDatabaseConstraint = rule["isDatabaseConstraint"] - create_uniqueness_rule( - model_name, discipline, isDatabaseConstraint, fields, scopes, registry) + create_uniqueness_rule(model_name, discipline, isDatabaseConstraint, fields, scopes, registry) def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, fields, scopes, registry=None): @@ -246,9 +263,11 @@ def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f discipline = None if rule_is_global(scopes) else raw_discipline - candidate_rules = UniquenessRule.objects.filter(modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) + candidate_rules = UniquenessRule.objects.filter(modelName=model_name, + isDatabaseConstraint=is_database_constraint, + discipline=discipline) - for rule in candidate_rules: + for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) @@ -256,16 +275,14 @@ def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): return + logger.info(f"Creating uniqueness rule on {model_name} with fields {fields} and scopes {scopes} for the discipline {discipline.name if discipline else 'Global'}") rule = UniquenessRule.objects.create( discipline=discipline, modelName=model_name, isDatabaseConstraint=is_database_constraint) for field in fields: - UniquenessRuleField.objects.create( - uniquenessrule=rule, fieldPath=field, isScope=False) + UniquenessRuleField.objects.create(uniquenessrule=rule, fieldPath=field, isScope=False) for scope in scopes: - UniquenessRuleField.objects.create( - uniquenessrule=rule, fieldPath=scope, isScope=True) - + UniquenessRuleField.objects.create(uniquenessrule=rule, fieldPath=scope, isScope=True) def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, fields, scopes, registry=None): UniquenessRule = registry.get_model( @@ -275,7 +292,8 @@ def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f discipline = None if rule_is_global(scopes) else raw_discipline - candidate_rules = UniquenessRule.objects.filter(modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) + candidate_rules = UniquenessRule.objects.filter( + modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) rule_ids = [] for rule in candidate_rules: @@ -297,6 +315,63 @@ def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f """ GLOBAL_RULE_FIELDS = ["division", 'institution'] - def rule_is_global(scopes: Iterable[str]) -> bool: - return len(scopes) == 0 or any(any(scope_field.lower() in GLOBAL_RULE_FIELDS for scope_field in scope.split('__')) for scope in scopes) + return len(scopes) == 0 \ + or any(any(scope_field.lower() in GLOBAL_RULE_FIELDS for scope_field in scope.split('__')) for scope in scopes) + +def fix_global_default_rules(registry=None): + UniquenessRule = registry.get_model('businessrules', 'UniquenessRule') \ + if registry \ + else models.UniquenessRule + UniquenessRuleField = registry.get_model('businessrules', 'UniquenessRuleField') \ + if registry \ + else models.UniquenessRuleField + + global_rule_fields = UniquenessRuleField.objects.filter( + uniquenessrule__discipline__isnull=True + ).values( + "uniquenessrule__modelName", + "uniquenessrule__isDatabaseConstraint", + "fieldPath", + "isScope", + ) + + global_rule_exists = UniquenessRule.objects.filter( + discipline__isnull=True, + modelName=OuterRef("modelName"), + isDatabaseConstraint=OuterRef("isDatabaseConstraint"), + ) + + discipline_ids = ( + UniquenessRule.objects.exclude(discipline__isnull=True) + .values_list("discipline_id", flat=True) + .distinct() + ) + + for discipline_id in discipline_ids: + with transaction.atomic(): + # Delete matching fields for this discipline + matching_fields_qs = UniquenessRuleField.objects.filter( + uniquenessrule__discipline_id=discipline_id + ).filter( + Exists( + global_rule_fields.filter( + **{ + "uniquenessrule__modelName": OuterRef("uniquenessrule__modelName"), + "uniquenessrule__isDatabaseConstraint": OuterRef("uniquenessrule__isDatabaseConstraint"), + "fieldPath": OuterRef("fieldPath"), + "isScope": OuterRef("isScope"), + } + ) + ) + ) + matching_fields_qs.delete() + + # Delete UniquenessRule rows for this discipline that are now empty + empty_rules_qs = ( + UniquenessRule.objects.filter(discipline_id=discipline_id) + .annotate(field_count=Count("uniquenessrulefield")) + .filter(field_count=0) # now empty after field deletions + .filter(Exists(global_rule_exists)) + ) + empty_rules_qs.delete() diff --git a/specifyweb/backend/patches/migration_utils.py b/specifyweb/backend/patches/migration_utils.py new file mode 100644 index 00000000000..a773cb10ad0 --- /dev/null +++ b/specifyweb/backend/patches/migration_utils.py @@ -0,0 +1,36 @@ +from django.db.models import F + +# REFACTOR: Use ALL_TRESS in specify/tree_views.py? +SPECIFY_TREES = ["Taxon", "Geography", "Storage", + "Geologictimeperiod", "Lithostrat"] + + +def apply_migrations(app_registry, schema_editor=None): + update_is_accepted(app_registry, schema_editor) + update_coordinates(app_registry, schema_editor) + +def update_is_accepted(app_registry, schema_editor=None): + for tree in SPECIFY_TREES: + tree_filters = { + "isaccepted": False, + "accepted" + tree.lower() + "__isnull": True + } + + tree_model = app_registry.get_model("specify", tree) + tree_model.objects.filter(**tree_filters).update(isaccepted=True) + + +def update_coordinates(app_registry, schema_editor=None): + Locality = app_registry.get_model("specify", "Locality") + + Locality.objects.filter(lat1text__isnull=True, latitude1__isnull=False) \ + .update(lat1text=F("latitude1")) + + Locality.objects.filter(long1text__isnull=True, longitude1__isnull=False) \ + .update(long1text=F("longitude1")) + + Locality.objects.filter(lat2text__isnull=True, latitude2__isnull=False) \ + .update(lat2text=F("latitude2")) + + Locality.objects.filter(long2text__isnull=True, longitude2__isnull=False) \ + .update(long2text=F("longitude2")) diff --git a/specifyweb/backend/patches/migrations/0001_restore_separators.py b/specifyweb/backend/patches/migrations/0001_restore_separators.py index 363ee2d5055..cdc34a59a06 100644 --- a/specifyweb/backend/patches/migrations/0001_restore_separators.py +++ b/specifyweb/backend/patches/migrations/0001_restore_separators.py @@ -7,6 +7,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ('specify', '0001_initial'), ] operations = [ diff --git a/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py b/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py index ec0ceae29f9..ce9e4213d4d 100644 --- a/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py +++ b/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py @@ -2,6 +2,8 @@ from django.db import migrations +from specifyweb.backend.patches.migration_utils import update_is_accepted + class Migration(migrations.Migration): @@ -13,10 +15,6 @@ class Migration(migrations.Migration): # Fixes an issue due to a bug in the WorkBench prior to 7.9.0 that did not # set accepted nodes to IsAccepted = 1 when they were not synonyms. # https://github.com/specify/specify7/issues/5131 - migrations.RunSQL( - 'UPDATE taxon t1 SET IsAccepted = TRUE WHERE t1.IsAccepted = 0 AND t1.AcceptedId IS NULL', - # This should not need to be reversed, but this allows for rollback without reversing the SQL - reverse_sql='' - ) + migrations.RunPython(update_is_accepted, + migrations.RunPython.noop, atomic=True), ] - diff --git a/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py b/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py index 2813b08ff16..eaccb47f93b 100644 --- a/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py +++ b/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py @@ -2,6 +2,8 @@ from django.db import migrations +from specifyweb.backend.patches.migration_utils import update_coordinates + class Migration(migrations.Migration): @@ -14,25 +16,6 @@ class Migration(migrations.Migration): # the accompanying text fields are not populated. In cases where the text fields are not populated, # it appears the records do not have coordinates. This script will populate the text fields with the # decimal values where the text fields are empty but the decimal values are not. - migrations.RunSQL( - """ - UPDATE locality - SET Lat1text = Latitude1 - WHERE Lat1text IS NULL AND Latitude1 IS NOT NULL; - - UPDATE locality - SET Long1text = Longitude1 - WHERE Long1text IS NULL AND Longitude1 IS NOT NULL; - - UPDATE locality - SET Lat2text = Latitude2 - WHERE Lat2text IS NULL AND Latitude2 IS NOT NULL; - - UPDATE locality - SET Long2text = Longitude2 - WHERE Long2text IS NULL AND Longitude2 IS NOT NULL; - """, - # This SQL statment should not be reversed, but this allows for rollback - reverse_sql='' - ) + migrations.RunPython(update_coordinates, + migrations.RunPython.noop, atomic=True), ] diff --git a/specifyweb/backend/permissions/initialize.py b/specifyweb/backend/permissions/initialize.py index c6ecd27d754..fb8d981891c 100644 --- a/specifyweb/backend/permissions/initialize.py +++ b/specifyweb/backend/permissions/initialize.py @@ -6,6 +6,11 @@ from .permissions import CollectionAccessPT +import sys +import logging + +logger = logging.getLogger(__name__) + def wipe_permissions(apps = apps) -> None: RolePolicy = apps.get_model('permissions', 'RolePolicy') UserRole = apps.get_model('permissions', 'UserRole') @@ -21,13 +26,22 @@ def wipe_permissions(apps = apps) -> None: LibraryRole.objects.all().delete() UserPolicy.objects.all().delete() +def is_sp6_user_permissions_migrated(user, apps=apps) -> bool: + UserPolicy = apps.get_model('permissions', 'UserPolicy') + UserRole = apps.get_model('permissions', 'UserRole') + return UserRole.objects.filter(specifyuser=user).exists() or \ + UserPolicy.objects.filter(specifyuser=user).exists() + def initialize(wipe: bool=False, apps=apps) -> None: with transaction.atomic(): if wipe: wipe_permissions(apps) create_admins(apps) create_roles(apps) - assign_users_to_roles(apps) + if 'test' in ''.join(sys.argv): + assign_users_to_roles_during_testing(apps) + else: + assign_users_to_roles(apps) def create_admins(apps=apps) -> None: UserPolicy = apps.get_model('permissions', 'UserPolicy') @@ -39,6 +53,8 @@ def create_admins(apps=apps) -> None: users = Specifyuser.objects.all() for user in users: + if is_sp6_user_permissions_migrated(user, apps): + continue if is_legacy_admin(user): UserPolicy.objects.get_or_create( collection=None, @@ -48,6 +64,84 @@ def create_admins(apps=apps) -> None: ) def assign_users_to_roles(apps=apps) -> None: + Role = apps.get_model('permissions', 'Role') + UserPolicy = apps.get_model('permissions', 'UserPolicy') + Agent = apps.get_model('specify', 'Agent') + UserPolicy = apps.get_model('permissions', 'UserPolicy') + UserRole = apps.get_model('permissions', 'UserRole') + + ROLE_DESCRIPTIONS = { + "Manager": "Grants full access to all abilities within a collection.", + "FullAccess": "This is a legacy role that provides read write access to most Specify resources and is assigned to users in the Full Access group from Specify 6. This is to maintain consistency with the permissions granted these users in previous versions of Specify 7.", + "LimitedAccess": "This is a legacy role that provides read only access and is assigned to user in the Limited Access and Guest groups from Specify 6. This is to maintain consistency with the permissions granted these users in previous versions of Specify 7.", + "Guest": "This is a legacy role that provides read only access and is assigned to user in the Limited Access and Guest groups from Specify 6. This is to maintain consistency with the permissions granted these users in previous versions of Specify 7.", + } + + ROLE_NAMES = { + "Manager": "Collection Admin", + "FullAccess": "Full Access - Legacy", + "LimitedAccess": "Read Only - Legacy", + "Guest": "Read Only - Legacy", + } + + cursor = connection.cursor() + cursor.execute(""" + SELECT + u.SpecifyUserID as user_id, + u.Name as user_name, + u.UserType as user_type, + p.usergroupscopeid as collection_id, + c.CollectionName as collection_name + FROM specifyuser u + JOIN specifyuser_spprincipal up ON up.SpecifyUserID = u.SpecifyUserID + JOIN spprincipal p ON p.SpPrincipalID = up.SpPrincipalID + JOIN collection c ON c.UserGroupScopeId = p.userGroupScopeID + WHERE p.groupType IS NULL + AND u.SpecifyUserID NOT IN ( + SELECT ur.specifyuser_id + FROM spuserrole ur + JOIN sprole r ON r.id = ur.role_id + WHERE r.collection_id = p.usergroupscopeid + ) + AND c.UserGroupScopeId NOT IN ( + SELECT DISTINCT r.collection_id + FROM spuserrole ur + JOIN sprole r ON r.id = ur.role_id + JOIN collection c ON c.UserGroupScopeId = r.collection_id + ); + """) + + results = cursor.fetchall() + + for user_id, user_name, user_type, collection_id, collection_name in results: + if user_type not in {'Manager', 'FullAccess', 'LimitedAccess', 'Guest'}: + continue + + role_name = ROLE_NAMES.get(user_type, f"{user_type} - {collection_name}") + role_description = ROLE_DESCRIPTIONS.get(user_type, "No description available.") + logger.info(f"Assigned user {user_name} to role {role_name} for collection {collection_name}.") + + role, is_new_role = Role.objects.get_or_create( + collection_id=collection_id, + name=role_name + ) + if is_new_role: + role.description = role_description + role.save() + UserRole.objects.get_or_create( + specifyuser_id=user_id, + role=role + ) + + if Agent.objects.filter(specifyuser_id=user_id, division__disciplines__collections__id=collection_id).exists(): + UserPolicy.objects.get_or_create( + collection_id=collection_id, + specifyuser_id=user_id, + resource=CollectionAccessPT.access.resource(), + action=CollectionAccessPT.access.action() + ) + +def assign_users_to_roles_during_testing(apps=apps) -> None: from specifyweb.backend.context.views import users_collections_for_sp6 Role = apps.get_model('permissions', 'Role') @@ -83,269 +177,284 @@ def create_roles(apps = apps) -> None: Collection = apps.get_model('specify', 'Collection') Specifyuser = apps.get_model('specify', 'Specifyuser') - role = LibraryRole.objects.create(name="Assign Roles", description="Gives ability to assign existing roles to existing users.") - role.policies.create(resource="/permissions/user/roles", action="read") - role.policies.create(resource="/permissions/user/roles", action="update") - role.policies.create(resource="/permissions/roles", action="read") - - role = LibraryRole.objects.create(name="Create Data Sets", description="Allows creating new Data Sets in the WorkBench, without ability to upload them.\n\nSuch user would create a Data Sets, map the columns, fix validation issues, and then transfer the Data Set to another user for review and upload.") - role.policies.create(resource="/workbench/dataset", action="create") - role.policies.create(resource="/workbench/dataset", action="update") - role.policies.create(resource="/workbench/dataset", action="delete") - role.policies.create(resource="/workbench/dataset", action="validate") - role.policies.create(resource="/workbench/dataset", action="transfer") - - role = LibraryRole.objects.create(name="Edit Forms and Global Preferences", description="Grants full access to resource editor. This allows editing form definitions and global Specify preferences.") - role.policies.create(resource="/table/spappresource", action="read") - role.policies.create(resource="/table/spappresource", action="create") - role.policies.create(resource="/table/spappresource", action="update") - role.policies.create(resource="/table/spappresource", action="delete") - role.policies.create(resource="/table/spappresourcedata", action="read") - role.policies.create(resource="/table/spappresourcedata", action="create") - role.policies.create(resource="/table/spappresourcedata", action="update") - role.policies.create(resource="/table/spappresourcedata", action="delete") - role.policies.create(resource="/table/spappresourcedir", action="read") - role.policies.create(resource="/table/spappresourcedir", action="create") - role.policies.create(resource="/table/spappresourcedir", action="update") - role.policies.create(resource="/table/spappresourcedir", action="delete") - role.policies.create(resource="/table/spviewsetobj", action="read") - role.policies.create(resource="/table/spviewsetobj", action="create") - role.policies.create(resource="/table/spviewsetobj", action="update") - role.policies.create(resource="/table/spviewsetobj", action="delete") - - role = LibraryRole.objects.create(name="Edit Pick lists", description="Gives full access to modifying pick lists.") - role.policies.create(resource="/table/picklist", action="read") - role.policies.create(resource="/table/picklist", action="create") - role.policies.create(resource="/table/picklist", action="update") - role.policies.create(resource="/table/picklist", action="delete") - role.policies.create(resource="/table/picklistitem", action="read") - role.policies.create(resource="/table/picklistitem", action="create") - role.policies.create(resource="/table/picklistitem", action="update") - role.policies.create(resource="/table/picklistitem", action="delete") - - role = LibraryRole.objects.create(name="Edit Taxon Tree", description="Gives full access to the Taxon Tree.\n\nWarning: Taxon Tree may be shared between collections. Edits in one collection may affect another.") - role.policies.create(resource="/tree/edit/taxon", action="merge") - role.policies.create(resource="/tree/edit/taxon", action="move") - role.policies.create(resource="/tree/edit/taxon", action="synonymize") - role.policies.create(resource="/tree/edit/taxon", action="desynonymize") - role.policies.create(resource="/tree/edit/taxon", action="repair") - role.policies.create(resource="/table/taxon", action="read") - role.policies.create(resource="/table/taxon", action="update") - role.policies.create(resource="/table/taxon", action="delete") - role.policies.create(resource="/table/taxon", action="create") - role.policies.create(resource="/table/taxonattribute", action="read") - role.policies.create(resource="/table/taxonattribute", action="delete") - role.policies.create(resource="/table/taxonattribute", action="update") - role.policies.create(resource="/table/taxonattribute", action="create") - role.policies.create(resource="/table/taxoncitation", action="read") - role.policies.create(resource="/table/taxoncitation", action="create") - role.policies.create(resource="/table/taxoncitation", action="update") - role.policies.create(resource="/table/taxoncitation", action="delete") - role.policies.create(resource="/table/taxontreedef", action="read") - role.policies.create(resource="/table/taxontreedef", action="update") - role.policies.create(resource="/table/taxontreedefitem", action="read") - role.policies.create(resource="/table/taxontreedefitem", action="update") - role.policies.create(resource="/table/taxonattachment", action="read") - role.policies.create(resource="/table/taxonattachment", action="create") - role.policies.create(resource="/table/taxonattachment", action="update") - role.policies.create(resource="/table/taxonattachment", action="delete") - - role = LibraryRole.objects.create(name="Export Data", description="Gives ability to export DwC Archive from any table.") - role.policies.create(resource="/export/dwca", action="execute") - role.policies.create(resource="/table/%", action="read") - - role = LibraryRole.objects.create(name="Full Data Access", description="Grants read and edit access to all tables") - role.policies.create(resource="/table/%", action="read") - role.policies.create(resource="/table/%", action="create") - role.policies.create(resource="/table/%", action="update") - role.policies.create(resource="/table/%", action="delete") - - role = LibraryRole.objects.create(name="Full WorkBench access", description="Gives full access to the WorkBench. Allows creating new records in any table.") - role.policies.create(resource="/workbench/dataset", action="create") - role.policies.create(resource="/workbench/dataset", action="update") - role.policies.create(resource="/workbench/dataset", action="delete") - role.policies.create(resource="/workbench/dataset", action="validate") - role.policies.create(resource="/workbench/dataset", action="upload") - role.policies.create(resource="/workbench/dataset", action="unupload") - role.policies.create(resource="/workbench/dataset", action="transfer") - role.policies.create(resource="/table/%", action="read") - role.policies.create(resource="/table/%", action="create") - - role = LibraryRole.objects.create(name="Inspect Audit Log", description="Allows to run a query builder query on the Audit Log table.") - role.policies.create(resource="/table/spauditlog", action="read") - role.policies.create(resource="/table/spauditlogfield", action="read") - role.policies.create(resource="/querybuilder/query", action="execute") - - role = LibraryRole.objects.create(name="Manage Interactions", description="Grants full access to interactions tables.") - role.policies.create(resource="/table/appraisal", action="read") - role.policies.create(resource="/table/appraisal", action="create") - role.policies.create(resource="/table/appraisal", action="update") - role.policies.create(resource="/table/appraisal", action="delete") - role.policies.create(resource="/table/borrow", action="read") - role.policies.create(resource="/table/borrow", action="create") - role.policies.create(resource="/table/borrow", action="delete") - role.policies.create(resource="/table/borrow", action="update") - role.policies.create(resource="/table/borrowagent", action="read") - role.policies.create(resource="/table/borrowagent", action="create") - role.policies.create(resource="/table/borrowagent", action="update") - role.policies.create(resource="/table/borrowagent", action="delete") - role.policies.create(resource="/table/borrowmaterial", action="read") - role.policies.create(resource="/table/borrowmaterial", action="create") - role.policies.create(resource="/table/borrowmaterial", action="update") - role.policies.create(resource="/table/borrowmaterial", action="delete") - role.policies.create(resource="/table/borrowreturnmaterial", action="read") - role.policies.create(resource="/table/borrowreturnmaterial", action="create") - role.policies.create(resource="/table/borrowreturnmaterial", action="update") - role.policies.create(resource="/table/borrowreturnmaterial", action="delete") - role.policies.create(resource="/table/deaccession", action="read") - role.policies.create(resource="/table/deaccession", action="create") - role.policies.create(resource="/table/deaccession", action="update") - role.policies.create(resource="/table/deaccession", action="delete") - role.policies.create(resource="/table/deaccessionagent", action="read") - role.policies.create(resource="/table/deaccessionagent", action="create") - role.policies.create(resource="/table/deaccessionagent", action="update") - role.policies.create(resource="/table/deaccessionagent", action="delete") - role.policies.create(resource="/table/disposal", action="read") - role.policies.create(resource="/table/disposal", action="create") - role.policies.create(resource="/table/disposal", action="update") - role.policies.create(resource="/table/disposal", action="delete") - role.policies.create(resource="/table/disposalagent", action="read") - role.policies.create(resource="/table/disposalagent", action="create") - role.policies.create(resource="/table/disposalagent", action="update") - role.policies.create(resource="/table/disposalagent", action="delete") - role.policies.create(resource="/table/disposalpreparation", action="read") - role.policies.create(resource="/table/disposalpreparation", action="create") - role.policies.create(resource="/table/disposalpreparation", action="update") - role.policies.create(resource="/table/disposalpreparation", action="delete") - role.policies.create(resource="/table/exchangein", action="read") - role.policies.create(resource="/table/exchangein", action="create") - role.policies.create(resource="/table/exchangein", action="update") - role.policies.create(resource="/table/exchangein", action="delete") - role.policies.create(resource="/table/exchangeinprep", action="read") - role.policies.create(resource="/table/exchangeinprep", action="create") - role.policies.create(resource="/table/exchangeinprep", action="delete") - role.policies.create(resource="/table/exchangeinprep", action="update") - role.policies.create(resource="/table/exchangeout", action="read") - role.policies.create(resource="/table/exchangeout", action="update") - role.policies.create(resource="/table/exchangeout", action="delete") - role.policies.create(resource="/table/exchangeout", action="create") - role.policies.create(resource="/table/exchangeoutprep", action="read") - role.policies.create(resource="/table/exchangeoutprep", action="create") - role.policies.create(resource="/table/exchangeoutprep", action="update") - role.policies.create(resource="/table/exchangeoutprep", action="delete") - role.policies.create(resource="/table/gift", action="read") - role.policies.create(resource="/table/gift", action="create") - role.policies.create(resource="/table/gift", action="update") - role.policies.create(resource="/table/gift", action="delete") - role.policies.create(resource="/table/giftagent", action="read") - role.policies.create(resource="/table/giftagent", action="create") - role.policies.create(resource="/table/giftagent", action="update") - role.policies.create(resource="/table/giftagent", action="delete") - role.policies.create(resource="/table/giftpreparation", action="read") - role.policies.create(resource="/table/giftpreparation", action="update") - role.policies.create(resource="/table/giftpreparation", action="delete") - role.policies.create(resource="/table/giftpreparation", action="create") - role.policies.create(resource="/table/inforequest", action="read") - role.policies.create(resource="/table/inforequest", action="create") - role.policies.create(resource="/table/inforequest", action="update") - role.policies.create(resource="/table/inforequest", action="delete") - role.policies.create(resource="/table/loan", action="read") - role.policies.create(resource="/table/loan", action="create") - role.policies.create(resource="/table/loan", action="update") - role.policies.create(resource="/table/loan", action="delete") - role.policies.create(resource="/table/loanagent", action="read") - role.policies.create(resource="/table/loanagent", action="create") - role.policies.create(resource="/table/loanagent", action="update") - role.policies.create(resource="/table/loanagent", action="delete") - role.policies.create(resource="/table/loanpreparation", action="read") - role.policies.create(resource="/table/loanpreparation", action="create") - role.policies.create(resource="/table/loanpreparation", action="update") - role.policies.create(resource="/table/loanpreparation", action="delete") - role.policies.create(resource="/table/loanreturnpreparation", action="read") - role.policies.create(resource="/table/loanreturnpreparation", action="create") - role.policies.create(resource="/table/loanreturnpreparation", action="update") - role.policies.create(resource="/table/loanreturnpreparation", action="delete") - role.policies.create(resource="/table/permit", action="read") - role.policies.create(resource="/table/permit", action="create") - role.policies.create(resource="/table/permit", action="update") - role.policies.create(resource="/table/permit", action="delete") - role.policies.create(resource="/table/shipment", action="read") - role.policies.create(resource="/table/shipment", action="create") - role.policies.create(resource="/table/shipment", action="update") - role.policies.create(resource="/table/shipment", action="delete") - role.policies.create(resource="/table/borrowattachment", action="read") - role.policies.create(resource="/table/borrowattachment", action="create") - role.policies.create(resource="/table/borrowattachment", action="update") - role.policies.create(resource="/table/borrowattachment", action="delete") - role.policies.create(resource="/table/deaccessionattachment", action="read") - role.policies.create(resource="/table/deaccessionattachment", action="create") - role.policies.create(resource="/table/deaccessionattachment", action="update") - role.policies.create(resource="/table/deaccessionattachment", action="delete") - role.policies.create(resource="/table/disposalattachment", action="read") - role.policies.create(resource="/table/disposalattachment", action="create") - role.policies.create(resource="/table/disposalattachment", action="update") - role.policies.create(resource="/table/disposalattachment", action="delete") - role.policies.create(resource="/table/giftattachment", action="read") - role.policies.create(resource="/table/giftattachment", action="create") - role.policies.create(resource="/table/giftattachment", action="update") - role.policies.create(resource="/table/giftattachment", action="delete") - role.policies.create(resource="/table/loanattachment", action="create") - role.policies.create(resource="/table/loanattachment", action="update") - role.policies.create(resource="/table/loanattachment", action="delete") - role.policies.create(resource="/table/loanattachment", action="read") - role.policies.create(resource="/table/permitattachment", action="read") - role.policies.create(resource="/table/permitattachment", action="create") - role.policies.create(resource="/table/permitattachment", action="update") - role.policies.create(resource="/table/permitattachment", action="delete") - - role = LibraryRole.objects.create(name="Print Reports", description="Gives ability to execute reports from any table.") - role.policies.create(resource="/report", action="execute") - role.policies.create(resource="/table/%", action="read") - - role = LibraryRole.objects.create(name="Read-Only Access", description="Grants read access to all tables") - role.policies.create(resource="/table/%", action="read") - - role = LibraryRole.objects.create(name="Run Queries", description="Gives access to execute queries on any table, export query results and create record sets.") - role.policies.create(resource="/querybuilder/query", action="execute") - role.policies.create(resource="/querybuilder/query", action="export_csv") - role.policies.create(resource="/querybuilder/query", action="export_kml") - role.policies.create(resource="/querybuilder/query", action="create_recordset") - role.policies.create(resource="/table/spquery", action="read") - role.policies.create(resource="/table/spquery", action="create") - role.policies.create(resource="/table/spquery", action="update") - role.policies.create(resource="/table/spquery", action="delete") - role.policies.create(resource="/table/spqueryfield", action="read") - role.policies.create(resource="/table/spqueryfield", action="create") - role.policies.create(resource="/table/spqueryfield", action="update") - role.policies.create(resource="/table/spqueryfield", action="delete") - role.policies.create(resource="/table/recordset", action="read") - role.policies.create(resource="/table/recordset", action="create") - role.policies.create(resource="/table/recordset", action="update") - role.policies.create(resource="/table/recordset", action="delete") - role.policies.create(resource="/table/recordsetitem", action="read") - role.policies.create(resource="/table/recordsetitem", action="create") - role.policies.create(resource="/table/recordsetitem", action="update") - role.policies.create(resource="/table/recordsetitem", action="delete") - role.policies.create(resource="/table/%", action="read") - - role = LibraryRole.objects.create(name="Security Admin", description="Grants full access to security settings within a collection.") - role.policies.create(resource="/permissions/%", action="read") - role.policies.create(resource="/permissions/%", action="update") - role.policies.create(resource="/permissions/%", action="create") - role.policies.create(resource="/permissions/%", action="delete") - role.policies.create(resource="/permissions/%", action="copy_from_library") - role.policies.create(resource="/table/specifyuser", action="read") - role.policies.create(resource="/table/specifyuser", action="create") - role.policies.create(resource="/table/specifyuser", action="update") - role.policies.create(resource="/table/specifyuser", action="delete") - - - collection_admin = LibraryRole.objects.create( + role, is_new = LibraryRole.objects.get_or_create(name="Assign Roles", description="Gives ability to assign existing roles to existing users.") + if is_new: + role.policies.get_or_create(resource="/permissions/user/roles", action="read") + role.policies.get_or_create(resource="/permissions/user/roles", action="update") + role.policies.get_or_create(resource="/permissions/roles", action="read") + + role, is_new = LibraryRole.objects.get_or_create(name="Create Data Sets", description="Allows creating new Data Sets in the WorkBench, without ability to upload them.\n\nSuch user would create a Data Sets, map the columns, fix validation issues, and then transfer the Data Set to another user for review and upload.") + if is_new: + role.policies.get_or_create(resource="/workbench/dataset", action="create") + role.policies.get_or_create(resource="/workbench/dataset", action="update") + role.policies.get_or_create(resource="/workbench/dataset", action="delete") + role.policies.get_or_create(resource="/workbench/dataset", action="validate") + role.policies.get_or_create(resource="/workbench/dataset", action="transfer") + + role, is_new = LibraryRole.objects.get_or_create(name="Edit Forms and Global Preferences", description="Grants full access to resource editor. This allows editing form definitions and global Specify preferences.") + if is_new: + role.policies.get_or_create(resource="/table/spappresource", action="read") + role.policies.get_or_create(resource="/table/spappresource", action="create") + role.policies.get_or_create(resource="/table/spappresource", action="update") + role.policies.get_or_create(resource="/table/spappresource", action="delete") + role.policies.get_or_create(resource="/table/spappresourcedata", action="read") + role.policies.get_or_create(resource="/table/spappresourcedata", action="create") + role.policies.get_or_create(resource="/table/spappresourcedata", action="update") + role.policies.get_or_create(resource="/table/spappresourcedata", action="delete") + role.policies.get_or_create(resource="/table/spappresourcedir", action="read") + role.policies.get_or_create(resource="/table/spappresourcedir", action="create") + role.policies.get_or_create(resource="/table/spappresourcedir", action="update") + role.policies.get_or_create(resource="/table/spappresourcedir", action="delete") + role.policies.get_or_create(resource="/table/spviewsetobj", action="read") + role.policies.get_or_create(resource="/table/spviewsetobj", action="create") + role.policies.get_or_create(resource="/table/spviewsetobj", action="update") + role.policies.get_or_create(resource="/table/spviewsetobj", action="delete") + + role, is_new = LibraryRole.objects.get_or_create(name="Edit Pick lists", description="Gives full access to modifying pick lists.") + if is_new: + role.policies.get_or_create(resource="/table/picklist", action="read") + role.policies.get_or_create(resource="/table/picklist", action="create") + role.policies.get_or_create(resource="/table/picklist", action="update") + role.policies.get_or_create(resource="/table/picklist", action="delete") + role.policies.get_or_create(resource="/table/picklistitem", action="read") + role.policies.get_or_create(resource="/table/picklistitem", action="create") + role.policies.get_or_create(resource="/table/picklistitem", action="update") + role.policies.get_or_create(resource="/table/picklistitem", action="delete") + + role, is_new = LibraryRole.objects.get_or_create(name="Edit Taxon Tree", description="Gives full access to the Taxon Tree.\n\nWarning: Taxon Tree may be shared between collections. Edits in one collection may affect another.") + if is_new: + role.policies.get_or_create(resource="/tree/edit/taxon", action="merge") + role.policies.get_or_create(resource="/tree/edit/taxon", action="move") + role.policies.get_or_create(resource="/tree/edit/taxon", action="synonymize") + role.policies.get_or_create(resource="/tree/edit/taxon", action="desynonymize") + role.policies.get_or_create(resource="/tree/edit/taxon", action="repair") + role.policies.get_or_create(resource="/table/taxon", action="read") + role.policies.get_or_create(resource="/table/taxon", action="update") + role.policies.get_or_create(resource="/table/taxon", action="delete") + role.policies.get_or_create(resource="/table/taxon", action="create") + role.policies.get_or_create(resource="/table/taxonattribute", action="read") + role.policies.get_or_create(resource="/table/taxonattribute", action="delete") + role.policies.get_or_create(resource="/table/taxonattribute", action="update") + role.policies.get_or_create(resource="/table/taxonattribute", action="create") + role.policies.get_or_create(resource="/table/taxoncitation", action="read") + role.policies.get_or_create(resource="/table/taxoncitation", action="create") + role.policies.get_or_create(resource="/table/taxoncitation", action="update") + role.policies.get_or_create(resource="/table/taxoncitation", action="delete") + role.policies.get_or_create(resource="/table/taxontreedef", action="read") + role.policies.get_or_create(resource="/table/taxontreedef", action="update") + role.policies.get_or_create(resource="/table/taxontreedefitem", action="read") + role.policies.get_or_create(resource="/table/taxontreedefitem", action="update") + role.policies.get_or_create(resource="/table/taxonattachment", action="read") + role.policies.get_or_create(resource="/table/taxonattachment", action="create") + role.policies.get_or_create(resource="/table/taxonattachment", action="update") + role.policies.get_or_create(resource="/table/taxonattachment", action="delete") + + role, is_new = LibraryRole.objects.get_or_create(name="Export Data", description="Gives ability to export DwC Archive from any table.") + if is_new: + role.policies.get_or_create(resource="/export/dwca", action="execute") + role.policies.get_or_create(resource="/table/%", action="read") + + role, is_new = LibraryRole.objects.get_or_create(name="Full Data Access", description="Grants read and edit access to all tables") + if is_new: + role.policies.get_or_create(resource="/table/%", action="read") + role.policies.get_or_create(resource="/table/%", action="create") + role.policies.get_or_create(resource="/table/%", action="update") + role.policies.get_or_create(resource="/table/%", action="delete") + + role, is_new = LibraryRole.objects.get_or_create(name="Full WorkBench access", description="Gives full access to the WorkBench. Allows creating new records in any table.") + if is_new: + role.policies.get_or_create(resource="/workbench/dataset", action="create") + role.policies.get_or_create(resource="/workbench/dataset", action="update") + role.policies.get_or_create(resource="/workbench/dataset", action="delete") + role.policies.get_or_create(resource="/workbench/dataset", action="validate") + role.policies.get_or_create(resource="/workbench/dataset", action="upload") + role.policies.get_or_create(resource="/workbench/dataset", action="unupload") + role.policies.get_or_create(resource="/workbench/dataset", action="transfer") + role.policies.get_or_create(resource="/table/%", action="read") + role.policies.get_or_create(resource="/table/%", action="create") + + role, is_new = LibraryRole.objects.get_or_create(name="Inspect Audit Log", description="Allows to run a query builder query on the Audit Log table.") + if is_new: + role.policies.get_or_create(resource="/table/spauditlog", action="read") + role.policies.get_or_create(resource="/table/spauditlogfield", action="read") + role.policies.get_or_create(resource="/querybuilder/query", action="execute") + + role, is_new = LibraryRole.objects.get_or_create(name="Manage Interactions", description="Grants full access to interactions tables.") + if is_new: + role.policies.get_or_create(resource="/table/appraisal", action="read") + role.policies.get_or_create(resource="/table/appraisal", action="create") + role.policies.get_or_create(resource="/table/appraisal", action="update") + role.policies.get_or_create(resource="/table/appraisal", action="delete") + role.policies.get_or_create(resource="/table/borrow", action="read") + role.policies.get_or_create(resource="/table/borrow", action="create") + role.policies.get_or_create(resource="/table/borrow", action="delete") + role.policies.get_or_create(resource="/table/borrow", action="update") + role.policies.get_or_create(resource="/table/borrowagent", action="read") + role.policies.get_or_create(resource="/table/borrowagent", action="create") + role.policies.get_or_create(resource="/table/borrowagent", action="update") + role.policies.get_or_create(resource="/table/borrowagent", action="delete") + role.policies.get_or_create(resource="/table/borrowmaterial", action="read") + role.policies.get_or_create(resource="/table/borrowmaterial", action="create") + role.policies.get_or_create(resource="/table/borrowmaterial", action="update") + role.policies.get_or_create(resource="/table/borrowmaterial", action="delete") + role.policies.get_or_create(resource="/table/borrowreturnmaterial", action="read") + role.policies.get_or_create(resource="/table/borrowreturnmaterial", action="create") + role.policies.get_or_create(resource="/table/borrowreturnmaterial", action="update") + role.policies.get_or_create(resource="/table/borrowreturnmaterial", action="delete") + role.policies.get_or_create(resource="/table/deaccession", action="read") + role.policies.get_or_create(resource="/table/deaccession", action="create") + role.policies.get_or_create(resource="/table/deaccession", action="update") + role.policies.get_or_create(resource="/table/deaccession", action="delete") + role.policies.get_or_create(resource="/table/deaccessionagent", action="read") + role.policies.get_or_create(resource="/table/deaccessionagent", action="create") + role.policies.get_or_create(resource="/table/deaccessionagent", action="update") + role.policies.get_or_create(resource="/table/deaccessionagent", action="delete") + role.policies.get_or_create(resource="/table/disposal", action="read") + role.policies.get_or_create(resource="/table/disposal", action="create") + role.policies.get_or_create(resource="/table/disposal", action="update") + role.policies.get_or_create(resource="/table/disposal", action="delete") + role.policies.get_or_create(resource="/table/disposalagent", action="read") + role.policies.get_or_create(resource="/table/disposalagent", action="create") + role.policies.get_or_create(resource="/table/disposalagent", action="update") + role.policies.get_or_create(resource="/table/disposalagent", action="delete") + role.policies.get_or_create(resource="/table/disposalpreparation", action="read") + role.policies.get_or_create(resource="/table/disposalpreparation", action="create") + role.policies.get_or_create(resource="/table/disposalpreparation", action="update") + role.policies.get_or_create(resource="/table/disposalpreparation", action="delete") + role.policies.get_or_create(resource="/table/exchangein", action="read") + role.policies.get_or_create(resource="/table/exchangein", action="create") + role.policies.get_or_create(resource="/table/exchangein", action="update") + role.policies.get_or_create(resource="/table/exchangein", action="delete") + role.policies.get_or_create(resource="/table/exchangeinprep", action="read") + role.policies.get_or_create(resource="/table/exchangeinprep", action="create") + role.policies.get_or_create(resource="/table/exchangeinprep", action="delete") + role.policies.get_or_create(resource="/table/exchangeinprep", action="update") + role.policies.get_or_create(resource="/table/exchangeout", action="read") + role.policies.get_or_create(resource="/table/exchangeout", action="update") + role.policies.get_or_create(resource="/table/exchangeout", action="delete") + role.policies.get_or_create(resource="/table/exchangeout", action="create") + role.policies.get_or_create(resource="/table/exchangeoutprep", action="read") + role.policies.get_or_create(resource="/table/exchangeoutprep", action="create") + role.policies.get_or_create(resource="/table/exchangeoutprep", action="update") + role.policies.get_or_create(resource="/table/exchangeoutprep", action="delete") + role.policies.get_or_create(resource="/table/gift", action="read") + role.policies.get_or_create(resource="/table/gift", action="create") + role.policies.get_or_create(resource="/table/gift", action="update") + role.policies.get_or_create(resource="/table/gift", action="delete") + role.policies.get_or_create(resource="/table/giftagent", action="read") + role.policies.get_or_create(resource="/table/giftagent", action="create") + role.policies.get_or_create(resource="/table/giftagent", action="update") + role.policies.get_or_create(resource="/table/giftagent", action="delete") + role.policies.get_or_create(resource="/table/giftpreparation", action="read") + role.policies.get_or_create(resource="/table/giftpreparation", action="update") + role.policies.get_or_create(resource="/table/giftpreparation", action="delete") + role.policies.get_or_create(resource="/table/giftpreparation", action="create") + role.policies.get_or_create(resource="/table/inforequest", action="read") + role.policies.get_or_create(resource="/table/inforequest", action="create") + role.policies.get_or_create(resource="/table/inforequest", action="update") + role.policies.get_or_create(resource="/table/inforequest", action="delete") + role.policies.get_or_create(resource="/table/loan", action="read") + role.policies.get_or_create(resource="/table/loan", action="create") + role.policies.get_or_create(resource="/table/loan", action="update") + role.policies.get_or_create(resource="/table/loan", action="delete") + role.policies.get_or_create(resource="/table/loanagent", action="read") + role.policies.get_or_create(resource="/table/loanagent", action="create") + role.policies.get_or_create(resource="/table/loanagent", action="update") + role.policies.get_or_create(resource="/table/loanagent", action="delete") + role.policies.get_or_create(resource="/table/loanpreparation", action="read") + role.policies.get_or_create(resource="/table/loanpreparation", action="create") + role.policies.get_or_create(resource="/table/loanpreparation", action="update") + role.policies.get_or_create(resource="/table/loanpreparation", action="delete") + role.policies.get_or_create(resource="/table/loanreturnpreparation", action="read") + role.policies.get_or_create(resource="/table/loanreturnpreparation", action="create") + role.policies.get_or_create(resource="/table/loanreturnpreparation", action="update") + role.policies.get_or_create(resource="/table/loanreturnpreparation", action="delete") + role.policies.get_or_create(resource="/table/permit", action="read") + role.policies.get_or_create(resource="/table/permit", action="create") + role.policies.get_or_create(resource="/table/permit", action="update") + role.policies.get_or_create(resource="/table/permit", action="delete") + role.policies.get_or_create(resource="/table/shipment", action="read") + role.policies.get_or_create(resource="/table/shipment", action="create") + role.policies.get_or_create(resource="/table/shipment", action="update") + role.policies.get_or_create(resource="/table/shipment", action="delete") + role.policies.get_or_create(resource="/table/borrowattachment", action="read") + role.policies.get_or_create(resource="/table/borrowattachment", action="create") + role.policies.get_or_create(resource="/table/borrowattachment", action="update") + role.policies.get_or_create(resource="/table/borrowattachment", action="delete") + role.policies.get_or_create(resource="/table/deaccessionattachment", action="read") + role.policies.get_or_create(resource="/table/deaccessionattachment", action="create") + role.policies.get_or_create(resource="/table/deaccessionattachment", action="update") + role.policies.get_or_create(resource="/table/deaccessionattachment", action="delete") + role.policies.get_or_create(resource="/table/disposalattachment", action="read") + role.policies.get_or_create(resource="/table/disposalattachment", action="create") + role.policies.get_or_create(resource="/table/disposalattachment", action="update") + role.policies.get_or_create(resource="/table/disposalattachment", action="delete") + role.policies.get_or_create(resource="/table/giftattachment", action="read") + role.policies.get_or_create(resource="/table/giftattachment", action="create") + role.policies.get_or_create(resource="/table/giftattachment", action="update") + role.policies.get_or_create(resource="/table/giftattachment", action="delete") + role.policies.get_or_create(resource="/table/loanattachment", action="create") + role.policies.get_or_create(resource="/table/loanattachment", action="update") + role.policies.get_or_create(resource="/table/loanattachment", action="delete") + role.policies.get_or_create(resource="/table/loanattachment", action="read") + role.policies.get_or_create(resource="/table/permitattachment", action="read") + role.policies.get_or_create(resource="/table/permitattachment", action="create") + role.policies.get_or_create(resource="/table/permitattachment", action="update") + role.policies.get_or_create(resource="/table/permitattachment", action="delete") + + role, is_new = LibraryRole.objects.get_or_create(name="Print Reports", description="Gives ability to execute reports from any table.") + if is_new: + role.policies.get_or_create(resource="/report", action="execute") + role.policies.get_or_create(resource="/table/%", action="read") + + role, is_new = LibraryRole.objects.get_or_create(name="Read-Only Access", description="Grants read access to all tables") + if is_new: + role.policies.get_or_create(resource="/table/%", action="read") + + role, is_new = LibraryRole.objects.get_or_create(name="Run Queries", description="Gives access to execute queries on any table, export query results and create record sets.") + if is_new: + role.policies.get_or_create(resource="/querybuilder/query", action="execute") + role.policies.get_or_create(resource="/querybuilder/query", action="export_csv") + role.policies.get_or_create(resource="/querybuilder/query", action="export_kml") + role.policies.get_or_create(resource="/querybuilder/query", action="create_recordset") + role.policies.get_or_create(resource="/table/spquery", action="read") + role.policies.get_or_create(resource="/table/spquery", action="create") + role.policies.get_or_create(resource="/table/spquery", action="update") + role.policies.get_or_create(resource="/table/spquery", action="delete") + role.policies.get_or_create(resource="/table/spqueryfield", action="read") + role.policies.get_or_create(resource="/table/spqueryfield", action="create") + role.policies.get_or_create(resource="/table/spqueryfield", action="update") + role.policies.get_or_create(resource="/table/spqueryfield", action="delete") + role.policies.get_or_create(resource="/table/recordset", action="read") + role.policies.get_or_create(resource="/table/recordset", action="create") + role.policies.get_or_create(resource="/table/recordset", action="update") + role.policies.get_or_create(resource="/table/recordset", action="delete") + role.policies.get_or_create(resource="/table/recordsetitem", action="read") + role.policies.get_or_create(resource="/table/recordsetitem", action="create") + role.policies.get_or_create(resource="/table/recordsetitem", action="update") + role.policies.get_or_create(resource="/table/recordsetitem", action="delete") + role.policies.get_or_create(resource="/table/%", action="read") + + role, is_new = LibraryRole.objects.get_or_create(name="Security Admin", description="Grants full access to security settings within a collection.") + if is_new: + role.policies.get_or_create(resource="/permissions/%", action="read") + role.policies.get_or_create(resource="/permissions/%", action="update") + role.policies.get_or_create(resource="/permissions/%", action="create") + role.policies.get_or_create(resource="/permissions/%", action="delete") + role.policies.get_or_create(resource="/permissions/%", action="copy_from_library") + role.policies.get_or_create(resource="/table/specifyuser", action="read") + role.policies.get_or_create(resource="/table/specifyuser", action="create") + role.policies.get_or_create(resource="/table/specifyuser", action="update") + role.policies.get_or_create(resource="/table/specifyuser", action="delete") + + + collection_admin, is_new = LibraryRole.objects.get_or_create( name="Collection Admin", description="Grants full access to all abilities within a collection.") - collection_admin.policies.create(resource="%", action="%") + if is_new: + collection_admin.policies.get_or_create(resource="%", action="%") - read_only = LibraryRole.objects.create( + read_only, is_new = LibraryRole.objects.get_or_create( name="Read Only - Legacy", description="This is a legacy role that provides " "read only access and is assigned to user in the " @@ -353,12 +462,13 @@ def create_roles(apps = apps) -> None: "This is to maintain consistency with the permissions " "granted these users in previous versions of Specify 7." ) - read_only.policies.create(resource="/field/%", action="%") - read_only.policies.create(resource="/table/%", action="read") + if is_new: + read_only.policies.get_or_create(resource="/field/%", action="%") + read_only.policies.get_or_create(resource="/table/%", action="read") - read_only.policies.create(resource="/querybuilder/%", action="%") + read_only.policies.get_or_create(resource="/querybuilder/%", action="%") - full_access = LibraryRole.objects.create( + full_access, is_new = LibraryRole.objects.get_or_create( name='Full Access - Legacy', description="This is a legacy role that provides " "read write access to most Specify resources and " @@ -366,26 +476,26 @@ def create_roles(apps = apps) -> None: "This is to maintain consistency with the permissions " "granted these users in previous versions of Specify 7." ) - - full_access.policies.create(resource="/field/%", action="%") - full_access.policies.create(resource="/table/%", action="read") + if is_new: + full_access.policies.get_or_create(resource="/field/%", action="%") + full_access.policies.get_or_create(resource="/table/%", action="read") for table in datamodel.tables: if not table.system or table.name.endswith('Attachment'): - full_access.policies.create(resource=f"/table/{table.name.lower()}", action="%") + full_access.policies.get_or_create(resource=f"/table/{table.name.lower()}", action="%") - full_access.policies.create(resource="/table/picklist", action="%") - full_access.policies.create(resource="/table/picklistitem", action="%") + full_access.policies.get_or_create(resource="/table/picklist", action="%") + full_access.policies.get_or_create(resource="/table/picklistitem", action="%") - full_access.policies.create(resource="/table/recordset", action="%") - full_access.policies.create(resource="/table/recordsetitem", action="%") + full_access.policies.get_or_create(resource="/table/recordset", action="%") + full_access.policies.get_or_create(resource="/table/recordsetitem", action="%") - full_access.policies.create(resource="/table/spquery", action="%") - full_access.policies.create(resource="/table/spqueryfield", action="%") + full_access.policies.get_or_create(resource="/table/spquery", action="%") + full_access.policies.get_or_create(resource="/table/spqueryfield", action="%") - full_access.policies.create(resource="/tree/%", action="%") - full_access.policies.create(resource="/report", action="%") - full_access.policies.create(resource="/querybuilder/%", action="%") + full_access.policies.get_or_create(resource="/tree/%", action="%") + full_access.policies.get_or_create(resource="/report", action="%") + full_access.policies.get_or_create(resource="/querybuilder/%", action="%") # copy the appropriate roles into the individual collections. users = Specifyuser.objects.all() @@ -393,31 +503,34 @@ def create_roles(apps = apps) -> None: if 'Guest' in user_types or 'LimitedAccess' in user_types: for collection in Collection.objects.all(): - r = Role.objects.create( + r, is_new = Role.objects.get_or_create( collection_id=collection.id, name=read_only.name, description=read_only.description, ) - for lp in read_only.policies.all(): - r.policies.create(resource=lp.resource, action=lp.action) + if is_new: + for lp in read_only.policies.all(): + r.policies.get_or_create(resource=lp.resource, action=lp.action) if 'FullAccess' in user_types: for collection in Collection.objects.all(): - r = Role.objects.create( + r, is_new = Role.objects.get_or_create( collection_id=collection.id, name=full_access.name, description=full_access.description, ) - for lp in full_access.policies.all(): - r.policies.create(resource=lp.resource, action=lp.action) + if is_new: + for lp in full_access.policies.all(): + r.policies.get_or_create(resource=lp.resource, action=lp.action) for collection_id in Collection.objects.values_list('id', flat=True): # Copy the collection admin role into the collection roles. - ca = Role.objects.create( + ca, is_new = Role.objects.get_or_create( collection_id=collection_id, name=collection_admin.name, description=collection_admin.description, ) - for lp in collection_admin.policies.all(): - ca.policies.create(resource=lp.resource, action=lp.action) + if is_new: + for lp in collection_admin.policies.all(): + ca.policies.get_or_create(resource=lp.resource, action=lp.action) diff --git a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py index 702defd3036..b9d092009af 100644 --- a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py +++ b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py @@ -2,32 +2,8 @@ from django.db import migrations -def add_permission(apps, schema_editor): - UserPolicy = apps.get_model('permissions', 'UserPolicy') - LibraryRolePolicy = apps.get_model('permissions', 'LibraryRolePolicy') - RolePolicy = apps.get_model('permissions', 'RolePolicy') +from specifyweb.permissions.migration_utils.edit_permissions import add_permission - for p in UserPolicy.objects.filter(resource='/workbench/dataset', action='upload'): - UserPolicy.objects.create( - collection=p.collection, - specifyuser=p.specifyuser, - resource=p.resource, - action='create_recordset', - ) - - for p in RolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - RolePolicy.objects.create( - role=p.role, - resource=p.resource, - action='create_recordset', - ) - - for p in LibraryRolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - LibraryRolePolicy.objects.create( - role=p.role, - resource=p.resource, - action='create_recordset', - ) class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py index 6d262b3fc64..fcb886f2dd2 100644 --- a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py +++ b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py @@ -1,19 +1,5 @@ from django.db import migrations -from specifyweb.specify.models import Collection # type: ignore -from ..models import Role - - -def add_stats_edit_permission(apps, schema_editor): - for collection_id in Collection.objects.values_list('id', flat=True): - try: - all_full_access_roles = Role.objects.filter(collection_id=collection_id, name="Full Access - Legacy") - for full_access_role in all_full_access_roles: - full_access_role.policies.create(resource="/preferences" - "/statistics", - action="edit") - except: - print("Failed to assign stats edit permission in collection: ", - collection_id) +from specifyweb.permissions.migration_utils.edit_permissions import add_stats_edit_permission class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py b/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py index 60ff925f311..a1cabb22429 100644 --- a/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py +++ b/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py @@ -1,4 +1,5 @@ from django.db import migrations +from specifyweb.backend.workbench.upload.auditlog import auditlog def add_attachment_import_role(apps, schema_editor): LibraryRole = apps.get_model('permissions', 'LibraryRole') @@ -6,187 +7,79 @@ def add_attachment_import_role(apps, schema_editor): name="Bulk Attachment Import", description="Gives full access to the Bulk Attachment Import. Allows creating new attachments for any attachment table" ) - # Attachment dataset permissions - role.policies.create(resource='/attachment_import/dataset', action='create') - role.policies.create(resource='/attachment_import/dataset', action='update') - role.policies.create(resource='/attachment_import/dataset', action='delete') - role.policies.create(resource='/attachment_import/dataset', action='upload') - role.policies.create(resource='/attachment_import/dataset', action='rollback') - - # Attachment permissions - role.policies.create(resource='/table/attachment', action='create') - role.policies.create(resource='/table/attachment', action='read') - role.policies.create(resource='/table/attachment', action='delete') - - # Table Specific permissions (Auto-generated) - role.policies.create(resource='/table/accession', action='read') - role.policies.create(resource='/table/accession', action='update') - role.policies.create(resource='/table/accessionattachment', action='read') - role.policies.create(resource='/table/accessionattachment', action='create') - role.policies.create(resource='/table/accessionattachment', action='delete') - - role.policies.create(resource='/table/agent', action='read') - role.policies.create(resource='/table/agent', action='update') - role.policies.create(resource='/table/agentattachment', action='read') - role.policies.create(resource='/table/agentattachment', action='create') - role.policies.create(resource='/table/agentattachment', action='delete') - - role.policies.create(resource='/table/borrow', action='read') - role.policies.create(resource='/table/borrow', action='update') - role.policies.create(resource='/table/borrowattachment', action='read') - role.policies.create(resource='/table/borrowattachment', action='create') - role.policies.create(resource='/table/borrowattachment', action='delete') - - role.policies.create(resource='/table/collectingevent', action='read') - role.policies.create(resource='/table/collectingevent', action='update') - role.policies.create(resource='/table/collectingeventattachment', action='read') - role.policies.create(resource='/table/collectingeventattachment', action='create') - role.policies.create(resource='/table/collectingeventattachment', action='delete') - - role.policies.create(resource='/table/collectingtrip', action='read') - role.policies.create(resource='/table/collectingtrip', action='update') - role.policies.create(resource='/table/collectingtripattachment', action='read') - role.policies.create(resource='/table/collectingtripattachment', action='create') - role.policies.create(resource='/table/collectingtripattachment', action='delete') - - role.policies.create(resource='/table/collectionobject', action='read') - role.policies.create(resource='/table/collectionobject', action='update') - role.policies.create(resource='/table/collectionobjectattachment', action='read') - role.policies.create(resource='/table/collectionobjectattachment', action='create') - role.policies.create(resource='/table/collectionobjectattachment', action='delete') - - role.policies.create(resource='/table/conservdescription', action='read') - role.policies.create(resource='/table/conservdescription', action='update') - role.policies.create(resource='/table/conservdescriptionattachment', action='read') - role.policies.create(resource='/table/conservdescriptionattachment', action='create') - role.policies.create(resource='/table/conservdescriptionattachment', action='delete') - - role.policies.create(resource='/table/conservevent', action='read') - role.policies.create(resource='/table/conservevent', action='update') - role.policies.create(resource='/table/conserveventattachment', action='read') - role.policies.create(resource='/table/conserveventattachment', action='create') - role.policies.create(resource='/table/conserveventattachment', action='delete') - - role.policies.create(resource='/table/dnasequence', action='read') - role.policies.create(resource='/table/dnasequence', action='update') - role.policies.create(resource='/table/dnasequenceattachment', action='read') - role.policies.create(resource='/table/dnasequenceattachment', action='create') - role.policies.create(resource='/table/dnasequenceattachment', action='delete') - - role.policies.create(resource='/table/dnasequencingrun', action='read') - role.policies.create(resource='/table/dnasequencingrun', action='update') - role.policies.create(resource='/table/dnasequencingrunattachment', action='read') - role.policies.create(resource='/table/dnasequencingrunattachment', action='create') - role.policies.create(resource='/table/dnasequencingrunattachment', action='delete') - - role.policies.create(resource='/table/deaccession', action='read') - role.policies.create(resource='/table/deaccession', action='update') - role.policies.create(resource='/table/deaccessionattachment', action='read') - role.policies.create(resource='/table/deaccessionattachment', action='create') - role.policies.create(resource='/table/deaccessionattachment', action='delete') - - role.policies.create(resource='/table/disposal', action='read') - role.policies.create(resource='/table/disposal', action='update') - role.policies.create(resource='/table/disposalattachment', action='read') - role.policies.create(resource='/table/disposalattachment', action='create') - role.policies.create(resource='/table/disposalattachment', action='delete') - - role.policies.create(resource='/table/exchangein', action='read') - role.policies.create(resource='/table/exchangein', action='update') - role.policies.create(resource='/table/exchangeinattachment', action='read') - role.policies.create(resource='/table/exchangeinattachment', action='create') - role.policies.create(resource='/table/exchangeinattachment', action='delete') - - role.policies.create(resource='/table/exchangeout', action='read') - role.policies.create(resource='/table/exchangeout', action='update') - role.policies.create(resource='/table/exchangeoutattachment', action='read') - role.policies.create(resource='/table/exchangeoutattachment', action='create') - role.policies.create(resource='/table/exchangeoutattachment', action='delete') - - role.policies.create(resource='/table/fieldnotebook', action='read') - role.policies.create(resource='/table/fieldnotebook', action='update') - role.policies.create(resource='/table/fieldnotebookattachment', action='read') - role.policies.create(resource='/table/fieldnotebookattachment', action='create') - role.policies.create(resource='/table/fieldnotebookattachment', action='delete') - - role.policies.create(resource='/table/fieldnotebookpage', action='read') - role.policies.create(resource='/table/fieldnotebookpage', action='update') - role.policies.create(resource='/table/fieldnotebookpageattachment', action='read') - role.policies.create(resource='/table/fieldnotebookpageattachment', action='create') - role.policies.create(resource='/table/fieldnotebookpageattachment', action='delete') - - role.policies.create(resource='/table/fieldnotebookpageset', action='read') - role.policies.create(resource='/table/fieldnotebookpageset', action='update') - role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='read') - role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='create') - role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='delete') - - role.policies.create(resource='/table/gift', action='read') - role.policies.create(resource='/table/gift', action='update') - role.policies.create(resource='/table/giftattachment', action='read') - role.policies.create(resource='/table/giftattachment', action='create') - role.policies.create(resource='/table/giftattachment', action='delete') - - role.policies.create(resource='/table/loan', action='read') - role.policies.create(resource='/table/loan', action='update') - role.policies.create(resource='/table/loanattachment', action='read') - role.policies.create(resource='/table/loanattachment', action='create') - role.policies.create(resource='/table/loanattachment', action='delete') - - role.policies.create(resource='/table/locality', action='read') - role.policies.create(resource='/table/locality', action='update') - role.policies.create(resource='/table/localityattachment', action='read') - role.policies.create(resource='/table/localityattachment', action='create') - role.policies.create(resource='/table/localityattachment', action='delete') - - role.policies.create(resource='/table/morphbankview', action='read') - role.policies.create(resource='/table/morphbankview', action='update') - role.policies.create(resource='/table/attachmentimageattribute', action='read') - role.policies.create(resource='/table/attachmentimageattribute', action='create') - role.policies.create(resource='/table/attachmentimageattribute', action='delete') - - role.policies.create(resource='/table/permit', action='read') - role.policies.create(resource='/table/permit', action='update') - role.policies.create(resource='/table/permitattachment', action='read') - role.policies.create(resource='/table/permitattachment', action='create') - role.policies.create(resource='/table/permitattachment', action='delete') - - role.policies.create(resource='/table/preparation', action='read') - role.policies.create(resource='/table/preparation', action='update') - role.policies.create(resource='/table/preparationattachment', action='read') - role.policies.create(resource='/table/preparationattachment', action='create') - role.policies.create(resource='/table/preparationattachment', action='delete') - - role.policies.create(resource='/table/referencework', action='read') - role.policies.create(resource='/table/referencework', action='update') - role.policies.create(resource='/table/referenceworkattachment', action='read') - role.policies.create(resource='/table/referenceworkattachment', action='create') - role.policies.create(resource='/table/referenceworkattachment', action='delete') - - role.policies.create(resource='/table/repositoryagreement', action='read') - role.policies.create(resource='/table/repositoryagreement', action='update') - role.policies.create(resource='/table/repositoryagreementattachment', action='read') - role.policies.create(resource='/table/repositoryagreementattachment', action='create') - role.policies.create(resource='/table/repositoryagreementattachment', action='delete') - - role.policies.create(resource='/table/storage', action='read') - role.policies.create(resource='/table/storage', action='update') - role.policies.create(resource='/table/storageattachment', action='read') - role.policies.create(resource='/table/storageattachment', action='create') - role.policies.create(resource='/table/storageattachment', action='delete') - - role.policies.create(resource='/table/taxon', action='read') - role.policies.create(resource='/table/taxon', action='update') - role.policies.create(resource='/table/taxonattachment', action='read') - role.policies.create(resource='/table/taxonattachment', action='create') - role.policies.create(resource='/table/taxonattachment', action='delete') - - role.policies.create(resource='/table/treatmentevent', action='read') - role.policies.create(resource='/table/treatmentevent', action='update') - role.policies.create(resource='/table/treatmenteventattachment', action='read') - role.policies.create(resource='/table/treatmenteventattachment', action='create') - role.policies.create(resource='/table/treatmenteventattachment', action='delete') + auditlog.insert(role) + + # Define policy sets grouped by resource + policy_definitions = [ + # Attachment dataset permissions + {'resource': '/attachment_import/dataset', 'actions': ['create', 'update', 'delete', 'upload', 'rollback']}, + # Attachment permissions + {'resource': '/table/attachment', 'actions': ['create', 'read', 'delete']}, + # Table Specific permissions + {'resource': '/table/accession', 'actions': ['read', 'update']}, + {'resource': '/table/accessionattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/agent', 'actions': ['read', 'update']}, + {'resource': '/table/agentattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/borrow', 'actions': ['read', 'update']}, + {'resource': '/table/borrowattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/collectingevent', 'actions': ['read', 'update']}, + {'resource': '/table/collectingeventattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/collectingtrip', 'actions': ['read', 'update']}, + {'resource': '/table/collectingtripattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/collectionobject', 'actions': ['read', 'update']}, + {'resource': '/table/collectionobjectattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/conservdescription', 'actions': ['read', 'update']}, + {'resource': '/table/conservdescriptionattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/conservevent', 'actions': ['read', 'update']}, + {'resource': '/table/conserveventattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/dnasequence', 'actions': ['read', 'update']}, + {'resource': '/table/dnasequenceattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/dnasequencingrun', 'actions': ['read', 'update']}, + {'resource': '/table/dnasequencingrunattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/deaccession', 'actions': ['read', 'update']}, + {'resource': '/table/deaccessionattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/disposal', 'actions': ['read', 'update']}, + {'resource': '/table/disposalattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/exchangein', 'actions': ['read', 'update']}, + {'resource': '/table/exchangeinattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/exchangeout', 'actions': ['read', 'update']}, + {'resource': '/table/exchangeoutattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/fieldnotebook', 'actions': ['read', 'update']}, + {'resource': '/table/fieldnotebookattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/fieldnotebookpage', 'actions': ['read', 'update']}, + {'resource': '/table/fieldnotebookpageattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/fieldnotebookpageset', 'actions': ['read', 'update']}, + {'resource': '/table/fieldnotebookpagesetattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/gift', 'actions': ['read', 'update']}, + {'resource': '/table/giftattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/loan', 'actions': ['read', 'update']}, + {'resource': '/table/loanattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/locality', 'actions': ['read', 'update']}, + {'resource': '/table/localityattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/morphbankview', 'actions': ['read', 'update']}, + {'resource': '/table/attachmentimageattribute', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/permit', 'actions': ['read', 'update']}, + {'resource': '/table/permitattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/preparation', 'actions': ['read', 'update']}, + {'resource': '/table/preparationattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/referencework', 'actions': ['read', 'update']}, + {'resource': '/table/referenceworkattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/repositoryagreement', 'actions': ['read', 'update']}, + {'resource': '/table/repositoryagreementattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/storage', 'actions': ['read', 'update']}, + {'resource': '/table/storageattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/taxon', 'actions': ['read', 'update']}, + {'resource': '/table/taxonattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/treatmentevent', 'actions': ['read', 'update']}, + {'resource': '/table/treatmenteventattachment', 'actions': ['read', 'create', 'delete']}, + ] + # Create each policy and log its creation + for policy in policy_definitions: + for action in policy['actions']: + obj, created = role.policies.get_or_create(resource=policy['resource'], action=action) + if created: + auditlog.insert(obj) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/backend/workbench/upload/auditlog.py b/specifyweb/backend/workbench/upload/auditlog.py index adec786d09c..c56252b5d32 100644 --- a/specifyweb/backend/workbench/upload/auditlog.py +++ b/specifyweb/backend/workbench/upload/auditlog.py @@ -7,6 +7,9 @@ logger = logging.getLogger(__name__) import re +from time import time +from typing import Iterable + from django.db import connection from django.conf import settings @@ -14,6 +17,8 @@ from specifyweb.backend.context.remote_prefs import get_remote_prefs, get_global_prefs from specifyweb.specify.models import datamodel +logger = logging.getLogger(__name__) + Collection = datamodel.get_table_strict('Collection') Discipline = datamodel.get_table_strict('Discipline') Division = datamodel.get_table_strict('Division') @@ -56,7 +61,7 @@ def isAuditing(self): self._lastCheck = time() return self._auditing; - def update(self, obj, agent, parent_record, dirty_flds): + def update(self, obj, agent, parent_record, dirty_flds: Iterable[FieldChangeInfo]): self.log_action(auditcodes.UPDATE, obj, agent, parent_record, dirty_flds) def log_action(self, action, obj, agent, parent_record, dirty_flds): @@ -66,7 +71,7 @@ def log_action(self, action, obj, agent, parent_record, dirty_flds): self._log_fld_update(vals, log_obj, agent) return log_obj - def insert(self, obj, agent, parent_record=None): + def insert(self, obj, agent=None, parent_record=None): return self._log(auditcodes.INSERT, obj, agent, parent_record) def remove(self, obj, agent, parent_record=None): @@ -119,7 +124,7 @@ def _log(self, action, obj, agent, parent_record): parenttablenum=parentTbl, recordid=obj.id, recordversion=obj.version if hasattr(obj, 'version') else 0, - tablenum=obj.specify_model.tableId, + tablenum=obj.specify_model.tableId if hasattr(obj, 'specify_model') else 0, # TODO: Checkout why LibraryRole has no specify_model during init migration createdbyagent_id=agent_id, modifiedbyagent_id=agent_id) diff --git a/specifyweb/frontend/js_src/lib/components/Attachments/__tests__/UploadAttachment.test.tsx b/specifyweb/frontend/js_src/lib/components/Attachments/__tests__/UploadAttachment.test.tsx index 88c7a933a0c..052fb73f2aa 100644 --- a/specifyweb/frontend/js_src/lib/components/Attachments/__tests__/UploadAttachment.test.tsx +++ b/specifyweb/frontend/js_src/lib/components/Attachments/__tests__/UploadAttachment.test.tsx @@ -1,24 +1,25 @@ +import { fireEvent, waitFor } from '@testing-library/react'; import React from 'react'; -import { mount } from '../../../tests/reactUtils'; -import { UploadAttachment } from '../Plugin'; + import { clearIdStore } from '../../../hooks/useId'; -import { LoadingContext } from '../../Core/Contexts'; -import { f } from '../../../utils/functools'; -import { fireEvent, waitFor } from '@testing-library/react'; -import { overrideAttachmentSettings } from '../attachments'; -import attachmentSettings from '../../../tests/ajax/static/context/attachment_settings.json'; import { overrideAjax } from '../../../tests/ajax'; -import * as Attachments from '../attachments'; +import attachmentSettings from '../../../tests/ajax/static/context/attachment_settings.json'; import { requireContext } from '../../../tests/helpers'; +import { mount } from '../../../tests/reactUtils'; +import { f } from '../../../utils/functools'; +import { LoadingContext } from '../../Core/Contexts'; +import type { SpecifyResource } from '../../DataModel/legacyTypes'; import { deserializeResource } from '../../DataModel/serializers'; +import type { Attachment } from '../../DataModel/types'; +import { overrideAttachmentSettings } from '../attachments'; +import * as Attachments from '../attachments'; +import { UploadAttachment } from '../Plugin'; import { testAttachment } from './utils'; -import { SpecifyResource } from '../../DataModel/legacyTypes'; -import { Attachment } from '../../DataModel/types'; requireContext(); -async function uploadFileMock() { - return deserializeResource(testAttachment) as SpecifyResource; +async function uploadFileMock(): Promise> { + return deserializeResource(testAttachment) ; } beforeEach(() => { @@ -57,7 +58,7 @@ describe('UploadAttachment', () => { fireEvent.change(input, { target: { files: [testFile] } }); await waitFor(() => { - expect(handleUploaded).toBeCalled(); + expect(handleUploaded).toHaveBeenCalled(); }); }); }); diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/WbAttachmentsPreview.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/WbAttachmentsPreview.tsx index ee1d1c7511b..a808e20a2b7 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/WbAttachmentsPreview.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/WbAttachmentsPreview.tsx @@ -173,7 +173,9 @@ function fetchRowAttachments( ) => void ): void { // Look for Attachments column - const attachmentColumnIndex = hot.toVisualColumn(getAttachmentsColumn(dataset)); + const attachmentColumnIndex = hot.toVisualColumn( + getAttachmentsColumn(dataset) + ); if (attachmentColumnIndex === -1) return; // Each row should have comma-separated IDs for SpDataSetAttachments diff --git a/specifyweb/permissions/migration_utils/__init__.py b/specifyweb/permissions/migration_utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/specifyweb/permissions/migration_utils/edit_permissions.py b/specifyweb/permissions/migration_utils/edit_permissions.py new file mode 100644 index 00000000000..6a2a8c7a408 --- /dev/null +++ b/specifyweb/permissions/migration_utils/edit_permissions.py @@ -0,0 +1,59 @@ +from specifyweb.backend.workbench.upload.auditlog import auditlog + + +def add_permission(apps, schema_editor=None): + UserPolicy = apps.get_model('permissions', 'UserPolicy') + LibraryRolePolicy = apps.get_model('permissions', 'LibraryRolePolicy') + RolePolicy = apps.get_model('permissions', 'RolePolicy') + + for p in UserPolicy.objects.filter(resource='/workbench/dataset', action='upload'): + if not UserPolicy.objects.filter( + collection=p.collection, + specifyuser=p.specifyuser, + resource=p.resource, + action='create_recordset', + ).exists(): + user_policy = UserPolicy.objects.create( + collection=p.collection, + specifyuser=p.specifyuser, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(user_policy) + + for p in RolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): + if not RolePolicy.objects.filter( + role=p.role, + resource=p.resource, + action='create_recordset', + ).exists(): + role_policy = RolePolicy.objects.create( + role=p.role, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(role_policy, None) + + for p in LibraryRolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): + if not LibraryRolePolicy.objects.filter( + role=p.role, + resource=p.resource, + action='create_recordset', + ).exists(): + library_role_policy = LibraryRolePolicy.objects.create( + role=p.role, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(library_role_policy, None) + + +def add_stats_edit_permission(apps, schema_editor=None): + Role = apps.get_model('permissions', 'Role') + + all_full_access_roles = Role.objects.filter(name="Full Access - Legacy") + + for full_access_role in all_full_access_roles: + new_policy, created = full_access_role.policies.get_or_create(resource="/preferences/statistics", action="edit") + if created: + auditlog.insert(new_policy, None) diff --git a/specifyweb/specify/api/utils.py b/specifyweb/specify/api/utils.py index 400658fd457..d9aeb22c5d3 100644 --- a/specifyweb/specify/api/utils.py +++ b/specifyweb/specify/api/utils.py @@ -18,6 +18,17 @@ def get_spmodel_class(model_name: str): return getattr(spmodels, attr_name) raise AttributeError(f"Model '{model_name}' not found in models module.") +def log_sqlalchemy_query(query): + # Call this function to debug the raw SQL query generated by SQLAlchemy + from sqlalchemy.dialects import mysql + compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) + raw_sql = str(compiled_query).replace('\n', ' ') + ';' + logger.debug('='.join(['' for _ in range(80)])) + logger.debug(raw_sql) + logger.debug('='.join(['' for _ in range(80)])) + # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 + # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) + def create_default_collection_types(apps): Collection = apps.get_model('specify', 'Collection') Collectionobject = apps.get_model('specify', 'Collectionobject') diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py new file mode 100644 index 00000000000..9aeeb3c8320 --- /dev/null +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -0,0 +1,176 @@ +import logging +from typing import Any +from collections.abc import Callable, Iterable +from django.core.management.base import BaseCommand +from django.apps import apps +from django.db import transaction +from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable +from specifyweb.backend.businessrules.uniqueness_rules import ( + apply_default_uniqueness_rules, + fix_global_default_rules +) +from specifyweb.permissions.migration_utils.edit_permissions import add_permission, add_stats_edit_permission +from specifyweb.specify.migration_utils.default_cots import ( + create_cogtype_type_picklist, + create_cotype_picklist, + create_default_collection_types, + create_default_discipline_for_tree_defs, + fix_taxon_treedef_discipline_links, + set_discipline_for_taxon_treedefs, + fix_tectonic_unit_treedef_discipline_links +) +from specifyweb.backend.permissions.initialize import initialize +from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.misc_migrations import make_selectseries_false +from specifyweb.specify.migration_utils.tectonic_ranks import create_default_tectonic_ranks, create_root_tectonic_node +from specifyweb.backend.patches.migration_utils import apply_migrations as apply_patches + +logger = logging.getLogger(__name__) + +MigrationFunction = Callable[[Any, Any | None], None] +WriteToStdOut = Callable[[str], None] + +def log_and_run(funcs: Iterable[MigrationFunction], stdout: WriteToStdOut | None = None) -> None: + for func in funcs: + if stdout is not None: + stdout(f"Running {func.__name__}...") + func(apps) + +def fix_cots(stdout: WriteToStdOut | None = None): + funcs = [ + create_default_collection_types, + create_default_discipline_for_tree_defs, + create_cogtype_type_picklist, + set_discipline_for_taxon_treedefs, + fix_taxon_treedef_discipline_links, + create_cotype_picklist + ] + log_and_run(funcs, stdout) + +def fix_schema_config(stdout: WriteToStdOut | None = None): + funcs = [ + usc.create_geo_table_schema_config_with_defaults, # specify 0002 + usc.create_cotype_splocalecontaineritem, # specify 0003 + usc.create_strat_table_schema_config_with_defaults, # specify 0004 - getting skip warnings + usc.create_agetype_picklist, # specify 0004 + usc.update_cog_type_fields, # specify 0007 + usc.create_cogtype_picklist, # specify 0007 + usc.update_cogtype_splocalecontaineritem, # specify 0007 + usc.update_systemcogtypes_picklist, # specify 0007 + usc.update_cogtype_type_splocalecontaineritem, # specify 0007 + usc.update_relative_age_fields, # specify 0008 + usc.add_cojo_to_schema_config, # specify 0012 + usc.update_cog_schema_config, # specify 0013 + usc.update_age_schema_config, # specify 0015 + usc.schemaconfig_fixes, # specify 0017 + usc.add_cot_catnum_to_schema, # specify 0018 + usc.add_tectonicunit_to_pc_in_schema_config, # specify 0020 + usc.fix_hidden_geo_prop, # specify 0021 + usc.update_schema_config_field_desc, # specify 0023 + usc.update_hidden_prop, # specify 0023 + usc.update_storage_unique_id_fields, # specify 0024 + usc.update_co_children_fields, # specify 0027 + usc.remove_collectionobject_parentco, # specify 0029 + usc.add_quantities_gift, # specify 0032 + usc.update_paleo_desc, # specify 0033 + usc.update_accession_date_fields # specify 0034 + ] + log_and_run(funcs, stdout) + +def apply_default_uniqueness_rules_to_disciplines(apps): + Discipline = apps.get_model('specify', 'Discipline') + UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + + for discipline in Discipline.objects.exclude( + id__in=set(UniquenessRule.objects.values_list('discipline_id', flat=True).distinct())): + apply_default_uniqueness_rules(discipline, registry=apps) + + +def fix_business_rules(stdout: WriteToStdOut | None = None): + funcs = [ + apply_default_uniqueness_rules_to_disciplines, + catnum_rule_editable, + fix_global_default_rules + ] + log_and_run(funcs, stdout) + +def initialize_permissions(apps): + initialize(False, apps) + +def fix_permissions(stdout: WriteToStdOut | None = None): + funcs = [ + initialize_permissions, + add_permission, + add_stats_edit_permission + ] + log_and_run(funcs, stdout) + +def fix_tectonic_ranks(stdout: WriteToStdOut | None = None): + funcs = [ + create_default_tectonic_ranks, + create_root_tectonic_node, + fix_tectonic_unit_treedef_discipline_links + ] + log_and_run(funcs, stdout) + +def fix_misc(stdout: WriteToStdOut | None = None): + funcs = [ + make_selectseries_false # specify 0031 + ] + log_and_run(funcs, stdout) + +class Command(BaseCommand): + help = "Runs this Django command to re-run important data migrations functions" + + funcs = { + "apply_patches": lambda _stdout: apply_patches(apps), + "fix_cots": fix_cots, + "fix_permissions": fix_permissions, + "fix_business_rules": fix_business_rules, + "fix_schema_config": fix_schema_config, + "fix_tectonic_ranks": fix_tectonic_ranks, + "fix_misc": fix_misc, + } + + def add_arguments(self, parser): + parser.add_argument( + "functions", + nargs="*", + type=str, + choices=tuple(self.funcs.keys()), + help=f"Optional: specify one or more functions to run", + ) + parser.add_argument( + "--verbose", + action='store_true', + dest="verbose", + default=False, + ) + + def handle(self, *args, **options): + functions = options.get("functions") + verbose = options.get("verbose", False) + + try: + with transaction.atomic(): + if len(functions) > 0: + for function in functions: + if function: + if function not in self.funcs: + self.stderr.write( + self.style.ERROR(f"Unknown function: {function}") + ) + return + self.stdout.write( + self.style.SUCCESS(f"Applying {function}...") + ) + self.funcs[function](self.stdout.write if verbose else None) + else: + self.stdout.write(self.style.SUCCESS("Running full pipeline...")) + for func_name, func in self.funcs.items(): + self.stdout.write(self.style.SUCCESS(f"Applying {func_name}...")) + func(self.stdout.write if verbose else None) + self.stdout.write(self.style.SUCCESS(f"Applied {func_name}")) + except Exception as e: + logger.error(f"An error occurred: {e}") + raise diff --git a/specifyweb/specify/migration_utils/default_cots.py b/specifyweb/specify/migration_utils/default_cots.py new file mode 100644 index 00000000000..e9387b5ca8e --- /dev/null +++ b/specifyweb/specify/migration_utils/default_cots.py @@ -0,0 +1,156 @@ +import logging +from django.db.models import F + +logger = logging.getLogger(__name__) + +DEFAULT_COG_TYPES = [ + 'Discrete', + 'Consolidated', + 'Drill Core', +] + +def create_default_collection_types(apps): + Collection = apps.get_model('specify', 'Collection') + Collectionobject = apps.get_model('specify', 'Collectionobject') + Collectionobjecttype = apps.get_model('specify', 'Collectionobjecttype') + + # Create default collection types for each collection, named after the discipline + for collection in Collection.objects.filter(collectionobjecttype__isnull=True): + discipline = collection.discipline + discipline_name = discipline.name + cot, created = Collectionobjecttype.objects.get_or_create( + name=discipline_name, + collection=collection, + taxontreedef_id=discipline.taxontreedef_id + ) + + # Update CollectionObjects' collectionobjecttype for the discipline + Collectionobject.objects.filter( + collection=collection).update(collectionobjecttype=cot) + collection.collectionobjecttype = cot + collection.save() + +def create_default_discipline_for_tree_defs(apps): + Discipline = apps.get_model('specify', 'Discipline') + Institution = apps.get_model('specify', 'Institution') + + for discipline in Discipline.objects.all(): + geography_tree_def = discipline.geographytreedef + if geography_tree_def and geography_tree_def.discipline_id is None: + geography_tree_def.discipline = discipline + geography_tree_def.save() + + geologic_time_period_tree_def = discipline.geologictimeperiodtreedef + if geologic_time_period_tree_def and geologic_time_period_tree_def.discipline_id is None: + geologic_time_period_tree_def.discipline = discipline + geologic_time_period_tree_def.save() + + lithostrat_tree_def = discipline.lithostrattreedef + if lithostrat_tree_def and lithostrat_tree_def.discipline_id is None: + lithostrat_tree_def.discipline = discipline + lithostrat_tree_def.save() + + taxon_tree_def = discipline.taxontreedef + if taxon_tree_def and taxon_tree_def.discipline_id is None: + taxon_tree_def.discipline = discipline + taxon_tree_def.save() + + for institution in Institution.objects.all(): + storage_tree_def = institution.storagetreedef + if storage_tree_def.institution_id is None: + storage_tree_def.institution = institution + storage_tree_def.save() + +def create_cogtype_type_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + Picklistitem = apps.get_model('specify', 'Picklistitem') + + for collection in Collection.objects.all(): + cog_type_picklist, _ = Picklist.objects.get_or_create( + name='SystemCOGTypes', # Default Collection Object Group Types + type=0, + collection=collection, + defaults={ + "issystem": False, + "readonly": False, + } + ) + for cog_type in DEFAULT_COG_TYPES: + Picklistitem.objects.get_or_create( + title=cog_type, + value=cog_type, + picklist=cog_type_picklist + ) + +COTYPE_PICKLIST_NAME = 'CollectionObjectType' +FIELD_NAME = 'collectionObjectType' +COTYPE_TEXT = 'Collection Object Type' + +def create_cotype_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + # Create a cotype picklist for each collection + for collection in Collection.objects.all(): + Picklist.objects.get_or_create( + name=COTYPE_PICKLIST_NAME, + type=1, + tablename='collectionobjecttype', + collection=collection, + defaults={ + "issystem": True, + "readonly": True, + "sizelimit": -1, + "sorttype": 1, + "formatter": COTYPE_PICKLIST_NAME, + } + ) + +def set_discipline_for_taxon_treedefs(apps): + Collectionobjecttype = apps.get_model('specify', 'Collectionobjecttype') + Taxontreedef = apps.get_model('specify', 'Taxontreedef') + + collection_object_types = Collectionobjecttype.objects.filter( + taxontreedef__discipline__isnull=True + ).annotate( + discipline=F('collection__discipline') + ) + + for cot in collection_object_types: + Taxontreedef.objects.filter(id=cot.taxontreedef_id).update(discipline=cot.discipline) + +def fix_taxon_treedef_discipline_links(apps): + Discipline = apps.get_model('specify', 'Discipline') + Taxontreedef = apps.get_model('specify', 'Taxontreedef') + + empty_taxon_treedefs = Taxontreedef.objects.filter(discipline__isnull=True) + disciplines = Discipline.objects.all() + for empty_taxon_treedef in empty_taxon_treedefs: + for discipline in disciplines: + if discipline.taxontreedef_id == empty_taxon_treedef.id: + empty_taxon_treedef.discipline = discipline + empty_taxon_treedef.save() + +def fix_tectonic_unit_treedef_discipline_links(apps): + Discipline = apps.get_model('specify', 'Discipline') + Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef') + + empty_tectonic_unit_treedefs = Tectonicunittreedef.objects.filter(discipline__isnull=True) + empty_disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) + for empty_discipline in empty_disciplines: + if not empty_tectonic_unit_treedefs.exists(): + new_tectonic_unit_treedef = Tectonicunittreedef.objects.create( + name=f'{empty_discipline.name} Tectonic Unit Tree', + discipline=empty_discipline + ) + else: + empty_discipline.tectonicunittreedef = empty_tectonic_unit_treedefs.first() + empty_discipline.save() + + for empty_tectonic_unit_treedef in empty_tectonic_unit_treedefs: + if empty_disciplines.exists(): + empty_tectonic_unit_treedef.discipline = empty_disciplines.first() + empty_tectonic_unit_treedef.save() + else: + empty_tectonic_unit_treedef.discipline = empty_disciplines.last() + empty_tectonic_unit_treedef.save() diff --git a/specifyweb/specify/migration_utils/misc_migrations.py b/specifyweb/specify/migration_utils/misc_migrations.py new file mode 100644 index 00000000000..f7c95092346 --- /dev/null +++ b/specifyweb/specify/migration_utils/misc_migrations.py @@ -0,0 +1,8 @@ + + +def make_selectseries_false(apps): + spquery = apps.get_model('specify', 'Spquery') + if 'selectseries' in [field.name for field in spquery._meta.get_fields()]: + spquery.objects.filter(selectseries=None).update(selectseries=False) + elif 'smushed' in [field.name for field in spquery._meta.get_fields()]: + spquery.objects.filter(smushed=None).update(smushed=False) diff --git a/specifyweb/specify/migration_utils/sp7_schemaconfig.py b/specifyweb/specify/migration_utils/sp7_schemaconfig.py index 07caa8dc3cd..d870be61399 100644 --- a/specifyweb/specify/migration_utils/sp7_schemaconfig.py +++ b/specifyweb/specify/migration_utils/sp7_schemaconfig.py @@ -47,11 +47,11 @@ # of the form {TableName: [...addedFields]} MIGRATION_0004_FIELDS = { - 'Collectionobject': ['relativeAges', 'absoluteAges', 'collectionObjectType'], + 'CollectionObject': ['relativeAges', 'absoluteAges', 'collectionObjectType'], 'Collection': ['collectionObjectType'], - 'Geographytreedef': ['discipline'], - 'Geologictimeperiodtreedef': ['discipline'], - 'Lithostrattreedef': ['discipline'], + 'GeographyTreeDef': ['discipline'], + 'GeologicTimePeriodTreeDef': ['discipline'], + 'LithoStratTreeDef': ['discipline'], } MIGRATION_0007_FIELDS = { @@ -91,7 +91,7 @@ ('yesno1', 'YesNo1', 'YesNo1'), ], - 'collectionobjectgroupjoin' : [ + 'CollectionObjectGroupJoin' : [ ('yesno2', 'YesNo2', 'YesNo2'), ('isSubstrate', 'Is Substrate?', 'The Collection Object that serves as the physical base for other items within the COG. This designation is useful for COGs with shared substrates.'), ('yesno1', 'YesNo1', 'YesNo1'), @@ -102,38 +102,38 @@ ('yesno3', 'YesNo3', 'YesNo3'), ], - 'collectionobjectgrouptype' : [ + 'CollectionObjectGroupType' : [ ('cogTypeId', 'Collection Object Group Type ID', 'Collection Object Group Type ID'), ('yesno3', 'YesNo3', 'YesNo3'), ], - 'collectionobjecttype': [ + 'CollectionObjectType': [ ('collectionObjectTypeId', 'Collection Object Type ID', 'Collection Object Type ID'), ('taxonTreeDef', 'Taxon Tree', 'The Taxon Tree associated with this Collection Object Type'), ], - 'absoluteage': [ + 'AbsoluteAge': [ ('yesno2', 'YesNo2', 'YesNo2'), ], - 'relativeage': [ + 'RelativeAge': [ ('yesno2', 'YesNo2', 'YesNo2'), ('yesno1', 'YesNo1', 'YesNo1'), ], - 'collectionobject': [ + 'CollectionObject': [ ('collectionObjectType', 'Type', 'The type of object, such as a fish, mammal, mineral, rock, or meteorite.'), ('cojo', 'Parent COG', 'Connects a Collection Object to its Collection Object Group'), ], - 'tectonicunit': [ + 'TectonicUnit': [ ('guid', 'GUID', 'GUID'), ('yesno1', 'YesNo1', 'YesNo1'), ('tectonicUnitId', 'Tectonic Unit ID', 'Tectonic Unit Id'), ('yesno2', 'YesNo2', 'YesNo2'), ], - 'tectonicunittreedefitem': [ + 'TectonicUnitTreeDefItem': [ ('createdbyagent', 'Created By Agent', 'Created By Agent'), ('rankId', 'Rank ID', 'Rank Id'), ] @@ -141,28 +141,17 @@ MIGRATION_0023_FIELDS_BIS = { 'CollectionObjectGroup': ['guid', ' text3', 'decimal2', 'igsn', 'text2', 'collection', 'description', 'text1', 'cojo', 'decimal1', 'yesno3', 'integer3', 'yesno2', 'collectionObjectGroupId', 'integer2', 'yesno1', 'integer1', 'decimal3', ], - - 'collectionobjectgroupjoin' : ['yesno2', 'text1', 'yesno1', 'integer3', 'integer2', 'integer1', 'text3', 'yesno3', 'precedence', 'text2'], - - 'collectionobjectgrouptype' : ['collection'], - - 'collectionobjecttype': ['text3', 'collectionObjectTypeId', 'text2', 'text1', 'collection'], - - 'absoluteage': ['collectionDate', 'absoluteAgeId', 'date1', 'date2', 'yesno1', 'yesno2', 'agent1', 'number1', 'number2', 'collectionObject', 'absoluteAgeCitations', 'text1', 'text2'], - - 'relativeage': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'], - - 'collectionobject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'], - - 'absoluteagecitation': ['collectionMember', 'absoluteAgeCitationId'], - - 'relativeagecitation': ['absoluteAgeCitationId', 'collectionMember'], - - 'tectonicunit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'], - - 'tectonicunittreedefitem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'], - - 'tectonicunittreedef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId'] + 'CollectionObjectGroupJoin' : ['yesno2', 'text1', 'yesno1', 'integer3', 'integer2', 'integer1', 'text3', 'yesno3', 'precedence', 'text2'], + 'CollectionObjectGroupType' : ['collection'], + 'CollectionObjectType': ['text3', 'collectionObjectTypeId', 'text2', 'text1', 'collection'], + 'AbsoluteAge': ['collectionDate', 'absoluteAgeId', 'date1', 'date2', 'yesno1', 'yesno2', 'agent1', 'number1', 'number2', 'collectionObject', 'absoluteAgeCitations', 'text1', 'text2'], + 'RelativeAge': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'], + 'CollectionObject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'], + 'AbsoluteAgeCitation': ['collectionMember', 'absoluteAgeCitationId'], + 'RelativeAgeCitation': ['absoluteAgeCitationId', 'collectionMember'], + 'TectonicUnit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'], + 'TectonicUnitTreeDefItem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'], + 'TectonicUnitTreeDef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId'] } MIGRATION_0024_FIELDS = { @@ -223,6 +212,201 @@ ] } +MIGRATION_0035_FIELDS = { + 'AbsoluteAge': ['version'], + 'AbsoluteAgeAttachment': ['version'], + 'AbsoluteAgeCitation': ['version'], + 'Accession': ['version'], + 'AccessionAgent': ['version'], + 'AccessionAttachment': ['version'], + 'AccessionAuthorization': ['version'], + 'AccessionCitation': ['version'], + 'Address': ['version'], + 'AddressOfRecord': ['version'], + 'Agent': ['version'], + 'AgentAttachment': ['version'], + 'AgentGeography': ['version'], + 'AgentIdentifier': ['version'], + 'AgentSpecialty': ['version'], + 'AgentVariant': ['version'], + 'Appraisal': ['version'], + 'Attachment': ['version'], + 'AttachmentImageAttribute': ['version'], + 'AttachmentMetadata': ['version'], + 'AttachmentTag': ['version'], + 'AttributeDef': ['version'], + 'Author': ['version'], + 'AutoNumberingScheme': ['version'], + 'Borrow': ['version'], + 'BorrowAgent': ['version'], + 'BorrowAttachment': ['version'], + 'BorrowMaterial': ['version'], + 'BorrowReturnMaterial': ['version'], + 'CollectingEvent': ['version'], + 'CollectingEventAttachment': ['version'], + 'CollectingEventAttr': ['version'], + 'CollectingEventAttribute': ['version'], + 'CollectingEventAuthorization': ['version'], + 'CollectingTrip': ['version'], + 'CollectingTripAttachment': ['version'], + 'CollectingTripAttribute': ['version'], + 'CollectingTripAuthorization': ['version'], + 'Collection': ['version'], + 'CollectionObject': ['version'], + 'CollectionObjectAttachment': ['version'], + 'CollectionObjectAttr': ['version'], + 'CollectionObjectAttribute': ['version'], + 'CollectionObjectCitation': ['version'], + 'CollectionObjectGroup': ['version'], + 'CollectionObjectGroupJoin': ['version'], + 'CollectionObjectGroupType': ['version'], + 'CollectionObjectProperty': ['version'], + 'CollectionObjectType': ['version'], + 'CollectionRelationship': ['version'], + 'CollectionRelType': ['version'], + 'Collector': ['version'], + 'CommonNameTx': ['version'], + 'CommonNameTxCitation': ['version'], + 'ConservDescription': ['version'], + 'ConservDescriptionAttachment': ['version'], + 'ConservEvent': ['version'], + 'ConservEventAttachment': ['version'], + 'Container': ['version'], + 'DataType': ['version'], + 'Deaccession': ['version'], + 'DeaccessionAgent': ['version'], + 'DeaccessionAttachment': ['version'], + 'Determination': ['version'], + 'DeterminationCitation': ['version'], + 'Determiner': ['version'], + 'Discipline': ['version'], + 'Disposal': ['version'], + 'DisposalAgent': ['version'], + 'DisposalAttachment': ['version'], + 'DisposalPreparation': ['version'], + 'Division': ['version'], + 'DnaPrimer': ['version'], + 'DnaSequence': ['version'], + 'DnaSequenceAttachment': ['version'], + 'DnaSequencingRun': ['version'], + 'DnaSequencingRunAttachment': ['version'], + 'DnaSequencingRunCitation': ['version'], + 'ExchangeIn': ['version'], + 'ExchangeInAttachment': ['version'], + 'ExchangeInPrep': ['version'], + 'ExchangeOut': ['version'], + 'ExchangeOutAttachment': ['version'], + 'ExchangeOutPrep': ['version'], + 'Exsiccata': ['version'], + 'ExsiccataItem': ['version'], + 'Extractor': ['version'], + 'FieldNotebook': ['version'], + 'FieldNotebookAttachment': ['version'], + 'FieldNotebookPage': ['version'], + 'FieldNotebookPageAttachment': ['version'], + 'FieldNotebookPageSet': ['version'], + 'FieldNotebookPageSetAttachment': ['version'], + 'FundingAgent': ['version'], + 'GeoCoordDetail': ['version'], + 'Geography': ['version'], + 'GeographyTreeDef': ['version'], + 'GeographyTreeDefItem': ['version'], + 'GeologicTimePeriod': ['version'], + 'GeologicTimePeriodTreeDef': ['version'], + 'GeologicTimePeriodTreeDefItem': ['version'], + 'Gift': ['version'], + 'GiftAgent': ['version'], + 'GiftAttachment': ['version'], + 'GiftPreparation': ['version'], + 'GroupPerson': ['version'], + 'InfoRequest': ['version'], + 'Institution': ['version'], + 'InstitutionNetwork': ['version'], + 'Journal': ['version'], + 'LatLonPolygon': ['version'], + 'LithoStrat': ['version'], + 'LithoStratTreeDef': ['version'], + 'LithoStratTreeDefItem': ['version'], + 'Loan': ['version'], + 'LoanAgent': ['version'], + 'LoanAttachment': ['version'], + 'LoanPreparation': ['version'], + 'LoanReturnPreparation': ['version'], + 'Locality': ['version'], + 'LocalityAttachment': ['version'], + 'LocalityCitation': ['version'], + 'LocalityDetail': ['version'], + 'LocalityNameAlias': ['version'], + 'MaterialSample': ['version'], + 'MorphbankView': ['version'], + 'OtherIdentifier': ['version'], + 'PaleoContext': ['version'], + 'PcrPerson': ['version'], + 'Permit': ['version'], + 'PermitAttachment': ['version'], + 'PickList': ['version'], + 'PickListItem': ['version'], + 'Preparation': ['version'], + 'PreparationAttachment': ['version'], + 'PreparationAttr': ['version'], + 'PreparationAttribute': ['version'], + 'PreparationProperty': ['version'], + 'PrepType': ['version'], + 'Project': ['version'], + 'Recordset': ['version'], + 'ReferenceWork': ['version'], + 'ReferenceWorkAttachment': ['version'], + 'RelativeAge': ['version'], + 'RelativeAgeAttachment': ['version'], + 'RelativeAgeCitation': ['version'], + 'RepositoryAgreement': ['version'], + 'RepositoryAgreementAttachment': ['version'], + 'Shipment': ['version'], + 'SpAppResource': ['version'], + 'SpAppResourceData': ['version'], + 'SpAppResourceDir': ['version'], + 'SpAuditLog': ['version'], + 'SpAuditLogField': ['version'], + 'SpecifyUser': ['version'], + 'SpExportSchema': ['version'], + 'SpExportSchemaItem': ['version'], + 'SpExportSchemaItemMapping': ['version'], + 'SpExportSchemaMapping': ['version'], + 'SpFieldValueDefault': ['version'], + 'SpLocaleContainer': ['version'], + 'SpLocaleContainerItem': ['version'], + 'SpLocaleItemStr': ['version'], + 'SpPrincipal': ['version'], + 'SpQuery': ['version'], + 'SpQueryField': ['version'], + 'SpReport': ['version'], + 'SpSymbiotaInstance': ['version'], + 'SpTaskSemaphore': ['version'], + 'SpVersion': ['version'], + 'SpViewSetObj': ['version'], + 'SpVisualQuery': ['version'], + 'Storage': ['version'], + 'StorageAttachment': ['version'], + 'StorageTreeDef': ['version'], + 'StorageTreeDefItem': ['version'], + 'Taxon': ['version'], + 'TaxonAttachment': ['version'], + 'TaxonAttribute': ['version'], + 'TaxonCitation': ['version'], + 'TaxonTreeDef': ['version'], + 'TaxonTreeDefItem': ['version'], + 'TectonicUnit': ['version'], + 'TectonicUnitTreeDef': ['version'], + 'TectonicUnitTreeDefItem': ['version'], + 'TreatmentEvent': ['version'], + 'TreatmentEventAttachment': ['version'], + 'VoucherRelationship': ['version'], + 'Workbench': ['version'], + 'WorkbenchRowExportedRelationship': ['version'], + 'WorkbenchTemplate': ['version'], + 'WorkbenchTemplateMappingItem': ['version'], +} + MIGRATION_0038_FIELDS = { 'Loan': ['agent1', 'agent2', 'agent3', 'agent4', 'agent5'], 'Gift': ['agent1', 'agent2', 'agent3', 'agent4', 'agent5'], @@ -243,4 +427,4 @@ ('agent4','Agent 4','Agent 4'), ('agent5','Agent 5','Agent 5'), ] -} \ No newline at end of file +} diff --git a/specifyweb/specify/migration_utils/tectonic_ranks.py b/specifyweb/specify/migration_utils/tectonic_ranks.py new file mode 100644 index 00000000000..c2afe88c6fa --- /dev/null +++ b/specifyweb/specify/migration_utils/tectonic_ranks.py @@ -0,0 +1,141 @@ +import logging +logger = logging.getLogger(__name__) + +def create_default_tectonic_ranks(apps): + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True).exclude( + id__in=TectonicTreeDef.objects.values_list('discipline_id', flat=True) + ) + + for discipline in disciplines: + tectonic_tree_def = TectonicTreeDef.objects.filter(discipline=discipline).first() + if not tectonic_tree_def: + tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) + + root, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Root", + title="Root", + rankid=0, + parent=None, + treedef=tectonic_tree_def, + isenforced=True + ) + superstructure, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Superstructure", + title="Superstructure", + rankid=10, + parent=root, + treedef=tectonic_tree_def, + ) + tectonic_domain, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Domain", + title="Tectonic Domain", + rankid=20, + parent=superstructure, + treedef=tectonic_tree_def, + ) + tectonic_subdomain, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Subdomain", + title="Tectonic Subdomain", + rankid=30, + parent=tectonic_domain, + treedef=tectonic_tree_def, + ) + tectonic_unit, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Unit", + title="Tectonic Unit", + rankid=40, + parent=tectonic_subdomain, + treedef=tectonic_tree_def, + ) + tectonic_subunit, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Subunit", + title="Tectonic Subunit", + rankid=50, + parent=tectonic_unit, + treedef=tectonic_tree_def, + ) + + discipline.tectonicunittreedef = tectonic_tree_def + discipline.save() + +def revert_default_tectonic_ranks(apps, schema_editor=None): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + tectonic_tree_defs = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline) + + for tectonic_tree_def in tectonic_tree_defs: + tectonic_unit_tree_def_items = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).order_by('-id') + + for item in tectonic_unit_tree_def_items: + TectonicUnit.objects.filter(definitionitem=item).delete() + + item.delete() + + discipline.tectonicunittreedef = None + discipline.save() + tectonic_tree_def.delete() + +def create_root_tectonic_node(apps): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicUnitTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + + tectonic_tree_def = TectonicUnitTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() + if not tectonic_tree_def: + tectonic_tree_def, is_created = TectonicUnitTreeDef.objects.get_or_create( + name="Tectonic Unit", + discipline=discipline + ) + + tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, name="Root").first() + if not tectonic_tree_def_item: + tectonic_tree_def_item, is_created = TectonicUnitTreeDefItem.objects.get_or_create( + name="Root", + title="Root", + treedef=tectonic_tree_def, + isenforced=True + ) + + root = TectonicUnit.objects.filter(name="Root", definition=tectonic_tree_def).first() + if not root: + root, is_created = TectonicUnit.objects.get_or_create( + name="Root", + fullname="Root", + isaccepted=1, + nodenumber=1, + rankid=0, + parent=None, + definition=tectonic_tree_def, + definitionitem=tectonic_tree_def_item + ) + + if is_created: + logger.info(f"Created root tectonic unit for discipline {discipline.name}") + + TectonicUnitTreeDefItem.objects.filter(rankid=0, isenforced__isnull=True).update(isenforced=True) + +def revert_create_root_tectonic_node(apps, schema_editor=None): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() + + if tectonic_tree_def: + TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() + TectonicUnit.objects.filter( + name="Root" + ).delete() \ No newline at end of file diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index 1e969a37b65..775299a8d07 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -3,13 +3,40 @@ from typing import NamedTuple, List import logging -from django.db.models import Q -from django.apps import apps +from django.db.models import Q, Count +from django.apps import apps as global_apps +from django.core.exceptions import MultipleObjectsReturned from specifyweb.specify.models_utils.load_datamodel import Table, FieldDoesNotExistError, TableDoesNotExistError +from specifyweb.specify.models_utils.model_extras import GEOLOGY_DISCIPLINES, PALEO_DISCIPLINES from specifyweb.specify.models import ( + Discipline, datamodel, ) +from specifyweb.specify.migration_utils.sp7_schemaconfig import ( + MIGRATION_0002_TABLES, + MIGRATION_0004_FIELDS, + MIGRATION_0004_TABLES, + MIGRATION_0007_FIELDS, + MIGRATION_0008_FIELDS, + MIGRATION_0012_FIELDS, + MIGRATION_0013_FIELDS, + MIGRATION_0020_FIELDS, + MIGRATION_0021_FIELDS, + MIGRATION_0023_FIELDS, + MIGRATION_0023_FIELDS_BIS, + MIGRATION_0024_FIELDS, + MIGRATION_0027_FIELDS, + MIGRATION_0027_UPDATE_FIELDS, + MIGRATION_0029_FIELDS, + MIGRATION_0029_UPDATE_FIELDS, + MIGRATION_0032_FIELDS, + MIGRATION_0032_UPDATE_FIELDS, + MIGRATION_0033_TABLES, + MIGRATION_0034_FIELDS, + MIGRATION_0034_UPDATE_FIELDS, + MIGRATION_0035_FIELDS, +) logger = logging.getLogger(__name__) @@ -61,15 +88,20 @@ def update_table_schema_config_with_defaults( table_name, discipline_id: int, description: str = None, - apps = apps + apps = global_apps ): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - try: - table: Table = datamodel.get_table_strict(table_name) - except TableDoesNotExistError: - logger.warning(f"Table does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name}") + table = datamodel.get_table(table_name) + + # BUG: The splocalecontainer related tables can still exist in the database, + # and this will result in skipping any operation if the table/field is + # removed, renamed, etc. + if table is None: + logger.warning( + f"Table does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name}" + ) return table_config = TableSchemaConfig( @@ -81,7 +113,7 @@ def update_table_schema_config_with_defaults( ) # Create Splocalecontainer for the table - sp_local_container, _ = Splocalecontainer.objects.get_or_create( + sp_local_container, is_new = Splocalecontainer.objects.get_or_create( name=table_config.name.lower(), discipline_id=discipline_id, schematype=table_config.schema_type, @@ -89,13 +121,18 @@ def update_table_schema_config_with_defaults( issystem=table.system, version=0, ) + if not is_new: + return # If the container already exists, we don't need to update it # Create a Splocaleitemstr for the table name and description - for k, text in {'containername': camel_to_spaced_title_case(uncapitilize(table.name)), 'containerdesc': table_config.description}.items(): + for k, text in { + "containername": camel_to_spaced_title_case(uncapitilize(table.name)), + "containerdesc": table_config.description, + }.items(): item_str = { - 'text': text, - 'language': 'en', - 'version': 0, + "text": text, + "language": "en", + "version": 0, } item_str[k] = sp_local_container Splocaleitemstr.objects.get_or_create(**item_str) @@ -104,7 +141,7 @@ def update_table_schema_config_with_defaults( update_table_field_schema_config_with_defaults(table_name, discipline_id, field.name, apps) -def revert_table_schema_config(table_name, apps = apps): +def revert_table_schema_config(table_name, apps=global_apps): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') @@ -124,14 +161,17 @@ def update_table_field_schema_config_with_defaults( table_name, discipline_id: int, field_name: str, - apps = apps + apps = global_apps ): - try: - table: Table = datamodel.get_table_strict(table_name) - except TableDoesNotExistError: + table = datamodel.get_table(table_name) + + # BUG: The splocalecontainer related tables can still exist in the database, + # and this will result in skipping any operation if the table/field is + # removed, renamed, etc. + if table is None: logger.warning(f"Table does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name}") return - + table_name = table.name table_config = TableSchemaConfig( name=table_name.lower(), @@ -144,16 +184,32 @@ def update_table_field_schema_config_with_defaults( Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - sp_local_container, _ = Splocalecontainer.objects.get_or_create( - name=table.name.lower(), - discipline_id=discipline_id, - schematype=table_config.schema_type, - ) + try: + sp_local_container, _ = Splocalecontainer.objects.get_or_create( + name=table.name.lower(), + discipline_id=discipline_id, + schematype=table_config.schema_type, + ) + except MultipleObjectsReturned: + sp_local_container = Splocalecontainer.objects.filter( + name=table.name.lower(), + discipline_id=discipline_id, + schematype=table_config.schema_type + ).first() try: field = table.get_field_strict(field_name) except FieldDoesNotExistError: - logger.warning(f"Field does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name} -> {field_name}") + if field_name in {'parentCog', 'parentCO', 'children', 'componentParent', 'components'}: + return + logger.warning( + f"Field does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name} -> {field_name}" + ) + return + except AttributeError: + logger.warning( + f"Field does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name} -> {field_name}" + ) return field_config = FieldSchemaConfig( @@ -186,7 +242,7 @@ def update_table_field_schema_config_with_defaults( itm_str[k] = sp_local_container_item Splocaleitemstr.objects.get_or_create(**itm_str) -def revert_table_field_schema_config(table_name, field_name, apps = apps): +def revert_table_field_schema_config(table_name, field_name, apps=global_apps): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') @@ -198,3 +254,1146 @@ def revert_table_field_schema_config(table_name, field_name, apps = apps): Q(itemdesc__in=items) ).delete() items.delete() + +def update_table_field_schema_config_params( + table_name, + discipline_id: int, + field_name: str, + update_params: dict, + apps = global_apps +): + table = datamodel.get_table(table_name) + + if table is None: + logger.warning(f"Table does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name}") + return + + table_name = table.name + table_config = TableSchemaConfig( + name=table_name.lower(), + discipline_id=discipline_id, + schema_type=0, + language="en" + ) + + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + try: + sp_local_container, _ = Splocalecontainer.objects.get_or_create( + name=table.name.lower(), + discipline_id=discipline_id, + schematype=table_config.schema_type, + ) + except MultipleObjectsReturned: + sp_local_container = Splocalecontainer.objects.filter( + name=table.name.lower(), + discipline_id=discipline_id, + schematype=table_config.schema_type + ).first() + + try: + field = table.get_field_strict(field_name) + except FieldDoesNotExistError: + logger.warning( + f"Field does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name} -> {field_name}" + ) + return + except AttributeError: + logger.warning( + f"Field does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name} -> {field_name}" + ) + return + + field_config = FieldSchemaConfig( + name=field_name, + column=field.column, + java_type=datamodel_type_to_schematype(field.type) if field.is_relationship else field.type, + description=camel_to_spaced_title_case(field.name), + language="en" + ) + + qs = Splocalecontaineritem.objects.filter( + name=field_config.name, + container=sp_local_container, + type=field_config.java_type, # maybe remove from filter + ) + count = qs.count() + + if count == 0: + # logger.warning(f"Splocalecontaineritem does not exist for: {table_name} -> {field_name}, skipping update") + return + + if count > 1: + updated = qs.update(**update_params) + logger.info(f"Updated {updated} duplicate Splocalecontaineritem rows for {table_name}.{field_name}") + return + + sp_local_container_item = qs.first() + for k, v in update_params.items(): + setattr(sp_local_container_item, k, v) + sp_local_container_item.save(update_fields=list(update_params.keys())) + +# ############################################################################## +# Migration schema config helper functions +# ############################################################################## + +# ########################################## +# Used in 0002_schema_config_update.py +# ########################################## + +DEFAULT_COG_TYPES = [ + 'Discrete', + 'Consolidated', + 'Drill Core', +] + +def create_geo_table_schema_config_with_defaults(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, desc in MIGRATION_0002_TABLES: + update_table_schema_config_with_defaults(table, discipline.id, desc, apps) + +# ########################################## +# Used in 0003_cotype_picklist.py +# ########################################## + +COT_PICKLIST_NAME = 'CollectionObjectType' +COT_FIELD_NAME = 'collectionObjectType' +COT_TEXT = 'Collection Object Type' + +def create_cotype_splocalecontaineritem(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + # Create a Splocalecontaineritem record for each CollectionObject Splocalecontainer + # NOTE: Each discipline has its own CollectionObject Splocalecontainer + for container in Splocalecontainer.objects.filter(name='collectionobject', schematype=0): + container_item, _ = Splocalecontaineritem.objects.get_or_create( + name=COT_FIELD_NAME, + picklistname=COT_PICKLIST_NAME, + type='ManyToOne', + container=container, + isrequired=True + ) + Splocaleitemstr.objects.get_or_create( + language='en', + text=COT_TEXT, + itemname=container_item + ) + Splocaleitemstr.objects.get_or_create( + language='en', + text=COT_TEXT, + itemdesc=container_item + ) + +# ########################################## +# Used in 0004_stratigraphy_age.py +# ########################################## + +AGETYPE_PICKLIST_NAME = 'AgeType' +DEFAULT_AGE_TYPES = [ + 'Sedimentation', + 'Metamorphism', + 'Erosion', + 'Diagenetic', +] + +def create_agetype_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + PicklistItem = apps.get_model('specify', 'Picklistitem') + + for collection in Collection.objects.all(): + age_type_picklist, created = Picklist.objects.get_or_create( + name=AGETYPE_PICKLIST_NAME, + type=0, + collection_id=collection.id, + defaults={ + "issystem": False, + "readonly": False, + "sizelimit": -1, + "sorttype": 1, + } + ) + if created: + for age_type in DEFAULT_AGE_TYPES: + PicklistItem.objects.get_or_create( + title=age_type, + value=age_type, + picklist=age_type_picklist + ) + +def create_strat_table_schema_config_with_defaults(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, desc in MIGRATION_0004_TABLES: # NOTE: lots of Nones, getting skips + update_table_schema_config_with_defaults(table, discipline.id, desc, apps) + + for table, fields in MIGRATION_0004_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + +def revert_strat_table_schema_config_with_defaults(apps): + for table, _ in MIGRATION_0004_TABLES: + revert_table_schema_config(table, apps) + for table, fields in MIGRATION_0004_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + +# ########################################## +# Used in 0007_schema_config_update.py +# ########################################## + +COG_PICKLIST_NAME = 'COGTypes' +COGTYPE_FIELD_NAME = 'cogType' +SYSTEM_COGTYPE_PICKLIST_NAME = "SystemCOGTypes" + +def update_cog_type_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + # Revert COG -> children before adding to avoid duplicates + revert_table_field_schema_config('CollectionObjectGroup', 'children', apps) + # Add StorageTreeDef -> institution and COG -> children + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0007_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + + # Remove COG -> cojo + revert_table_field_schema_config('CollectionObjectGroup', 'cojo', apps) + + # Remove duplicate CollectionObject -> collectionObjectType + container_items = Splocalecontaineritem.objects.filter( + name="collectionObjectType", + picklistname=None, + container__name="CollectionObject", + ) + for container_item in container_items: + Splocaleitemstr.objects.filter(itemname=container_item).delete() + Splocaleitemstr.objects.filter(itemdesc=container_item).delete() + container_items.delete() + +# NOTE: The reverse function will not re-add the duplicate CO -> coType or COG -> cojo as its unnecessary +def revert_cog_type_fields(apps): + # Remove StorageTreeDef -> institution and COG -> children + for table, fields in MIGRATION_0007_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + +def create_cogtype_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + + # Create a cogtype picklist for each collection + for collection in Collection.objects.all(): + Picklist.objects.get_or_create( + name=COG_PICKLIST_NAME, + type=1, + tablename='collectionobjectgrouptype', + collection=collection, + defaults={ + "issystem": True, + "readonly": True, + "sizelimit": -1, + "sorttype": 1, + "formatter": 'CollectionObjectGroupType', + } + ) + +def revert_cogtype_picklist(apps): + Picklist = apps.get_model('specify', 'Picklist') + + Picklist.objects.filter(name=COG_PICKLIST_NAME).delete() + + +# Updates COG -> cogtype to use the type 1 picklist created above +def update_cogtype_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model("specify", "Splocalecontaineritem") + + Splocalecontaineritem.objects.filter( + container__name="collectionobjectgroup", + container__schematype=0, + name=COGTYPE_FIELD_NAME, + ).update(picklistname=COG_PICKLIST_NAME, type="ManyToOne", isrequired=True) + + +def revert_cogtype_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model("specify", "Splocalecontaineritem") + + Splocalecontaineritem.objects.filter( + container__name="collectionobjectgroup", + container__schematype=0, + name=COGTYPE_FIELD_NAME, + ).update(picklistname=None, type=None, isrequired=None) + + +def update_systemcogtypes_picklist(apps): + Picklist = apps.get_model('specify', 'Picklist') + + Picklist.objects.filter(name='Default Collection Object Group Types').update( + name=SYSTEM_COGTYPE_PICKLIST_NAME, + type=0, + issystem=True, + readonly=True, + sizelimit=3, + tablename=None + ) + +def revert_systemcogtypes_picklist(apps): + Picklist = apps.get_model('specify', 'Picklist') + + # revert only changes the name and not the other attributes as those were incorrect + Picklist.objects.filter(name=SYSTEM_COGTYPE_PICKLIST_NAME).update( + name='Default Collection Object Group Types', + ) + + +# Updates cogtype -> type to use the Default COGType picklist (Drill Core, Discrete, Consolidated) +def update_cogtype_type_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model("specify", "Splocalecontaineritem") + + Splocalecontaineritem.objects.filter( + container__name="collectionobjectgrouptype", + container__schematype=0, + name="type", + ).update(picklistname=SYSTEM_COGTYPE_PICKLIST_NAME, isrequired=True) + + +def revert_cogtype_type_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model("specify", "Splocalecontaineritem") + + Splocalecontaineritem.objects.filter( + container__name="collectionobjectgrouptype", + container__schematype=0, + name="type", + ).update(picklistname=None, isrequired=None) + +# ########################################## +# Used in 0008_schema_config_update.py +# ########################################## + +def update_relative_age_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + # Add absoluteAgeCitation -> absoluteAge & Add relativeAgeCitation -> relativeAge + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0008_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + +def revert_relative_age_fields(apps): + # Remove absoluteAgeCitation -> absoluteAge and relativeAgeCitation -> relativeAge + for table, fields in MIGRATION_0008_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + +# ########################################## +# Used in 0012_add_cojo_to_schema_config.py +# ########################################## + +def add_cojo_to_schema_config(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0012_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults( + table, discipline.id, field, apps) + + +def remove_cojo_from_schema_config(apps): + for table, fields in MIGRATION_0012_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + + +# ########################################## +# Used in 0013_collectionobjectgroup_parentcog.py +# ########################################## + +def update_cog_schema_config(apps): + revert_table_field_schema_config( + 'CollectionObjectGroup', 'parentCojo', apps) + revert_table_field_schema_config( + 'CollectionObjectGroup', 'parentCog', apps) + + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0013_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults( + table, discipline.id, field, apps) + + +def revert_update_cog_schema_config(apps): + for table, fields in MIGRATION_0013_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + update_table_field_schema_config_with_defaults( + 'CollectionObjectGroup', discipline.id, 'parentCojo', apps) + +# ########################################## +# Used in 0015_add_version_to_ages.py +# ########################################## + +def update_age_schema_config(apps): + # Revert before adding to avoid duplicates + revert_update_age_schema_config(apps) + + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + update_table_field_schema_config_with_defaults('AbsoluteAge', discipline.id, 'version', apps) + update_table_field_schema_config_with_defaults('RelativeAge', discipline.id, 'version', apps) + +def revert_update_age_schema_config(apps): + revert_table_field_schema_config('AbsoluteAge', 'version', apps) + revert_table_field_schema_config('RelativeAge', 'version', apps) + +# ########################################## +# Used in 0017_schemaconfig_fixes.py +# ########################################## + +CONTAINER_MIGRATIONS = [MIGRATION_0002_TABLES, MIGRATION_0004_TABLES] + +CONTAINER_ITEM_MIGRATIONS = [ + MIGRATION_0004_FIELDS, + MIGRATION_0007_FIELDS, + MIGRATION_0008_FIELDS, + MIGRATION_0012_FIELDS, + MIGRATION_0013_FIELDS, +] + +def fix_table_captions(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for migration in CONTAINER_MIGRATIONS: + for table_name, table_desc in migration: + table = datamodel.get_table(table_name) + + # BUG: The splocalecontainer related tables can still exist in the + # database, and this will result in skipping any operation if the + # table/field is removed, renamed, etc. + if table is None: + logger.warning(f"Table does not exist in latest state of the datamodel, skipping Schema Config update for: {table_name}") + continue + containers = Splocalecontainer.objects.filter( + name=table_name.lower(), schematype=0) + + # If needed, correct the label of the table in the schema config + if table_desc is not None: + Splocaleitemstr.objects.filter( + containername__in=containers, text=table_desc + ).update(text=camel_to_spaced_title_case(uncapitilize(table.name))) + + # Update the types for the fields in the table + items = Splocalecontaineritem.objects.filter( + container__in=containers) + for item in items: + datamodel_field = table.get_field(item.name) + if not datamodel_field: + continue + + item.type = datamodel_type_to_schematype( + datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type + item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired + + item.save() + + +def fix_item_types(apps): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for migration in CONTAINER_ITEM_MIGRATIONS: + for table_name, fields in migration.items(): + table = datamodel.get_table(table_name) + # BUG: The splocalecontainer related tables can still exist in the + # database, and this will result in skipping any operation if the + # table/field is removed, renamed, etc. + if table is None: + logger.warning(f"Table does not exist in latest state of the datamodel, skipping Schema Config entry for: {table_name}") + continue + items = Splocalecontaineritem.objects.filter( + container__name=table_name.lower(), container__schematype=0, name__in=fields) + + for item in items: + datamodel_field = table.get_field(item.name) + if not datamodel_field: + continue + + item.type = datamodel_type_to_schematype( + datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type + item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired + + item.save() + + +def schemaconfig_fixes(apps, schema_editor=None): + fix_table_captions(apps) + fix_item_types(apps) + +# ########################################## +# Used in 0018_cot_catnum_schema.py +# ########################################## + +def add_cot_catnum_to_schema(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + CollectionObjectType_Table = datamodel.get_table_strict( + 'collectionobjecttype') + catalognumber_format_field = CollectionObjectType_Table.get_field_strict( + 'catalogNumberFormatName') + + for container in Splocalecontainer.objects.filter(name='collectionobjecttype', schematype=0): + schema_item, created = Splocalecontaineritem.objects.get_or_create( + name=catalognumber_format_field.name, type=catalognumber_format_field.type, container=container) + if created: + schema_item.version = 0 + + schema_item.isrequired = ( + catalognumber_format_field.required + if schema_item.isrequired is None + else schema_item.isrequired + ) + + schema_item.save() + + schema_name = camel_to_spaced_title_case( + catalognumber_format_field.name) + Splocaleitemstr.objects.get_or_create( + language='en', text=schema_name, itemname=schema_item) + Splocaleitemstr.objects.get_or_create( + language='en', text=schema_name, itemdesc=schema_item) + +def remove_cot_catnum_from_schema(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + CollectionObjectType_Table = datamodel.get_table_strict( + 'collectionobjecttype') + catalognumber_format_field = CollectionObjectType_Table.get_field_strict( + 'catalogNumberFormatName') + + containers = Splocalecontainer.objects.filter( + name='collectionobjecttype', schematype=0) + items = Splocalecontaineritem.objects.filter( + name='catalogNumberFormatName', container__in=containers) + + schema_name = camel_to_spaced_title_case(catalognumber_format_field.name) + filters = Q(language='en', text=schema_name) & ( + Q(itemname__in=items) | Q(itemdesc__in=items)) + locale_strings = Splocaleitemstr.objects.filter(filters) + + locale_strings.delete() + items.delete() + +# ########################################## +# Used in 0020_add_tectonicunit_to_pc_in_schema_config.py +# ########################################## + +def add_tectonicunit_to_pc_in_schema_config(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0020_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults( + table, discipline.id, field, apps) + +def remove_tectonicunit_from_pc_schema_config(apps): + for table, fields in MIGRATION_0020_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + +# ########################################## +# Used in 0021_update_hidden_geo_tables.py +# ########################################## + +def fix_hidden_geo_prop(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + + excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + + filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + + for discipline in filtered_disciplines: + for table, fields in MIGRATION_0021_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + discipline_id=discipline.id, + ) + for container in containers: + # BUG: What if the user wants the field unhidden? + Splocalecontaineritem.objects.filter( + container=container, + name__in=tuple(map(lambda field_name: field_name.lower(), fields)) + ).update(ishidden=True) + +def reverse_fix_hidden_geo_prop(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + + excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + + filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + + for discipline in filtered_disciplines: + for table, fields in MIGRATION_0021_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + discipline_id=discipline.id, + ) + for container in containers: + # BUG: What if the user wants the field hidden? + Splocalecontaineritem.objects.filter( + container=container, + name__in=tuple(map(lambda field_name: field_name.lower(), fields)) + ).update(ishidden=False) + +# ########################################## +# Used in 0023_update_schema_config_text.py +# ########################################## + +def update_schema_config_field_desc(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0023_FIELDS.items(): + #i.e: Collection Object + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + #i.e: COType + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = new_desc + localized_items_desc.save() + + localized_items_name.text = new_name + localized_items_name.save() + +def update_hidden_prop(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0023_FIELDS_BIS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + schematype=0 + ) + for container in containers: + items_updated = Splocalecontaineritem.objects.filter( + container=container, + ishidden=False, + name__in=[field_name.lower() for field_name in fields] + ).update(ishidden=True) + if items_updated > 0: + logger.info(f"Hid {items_updated} items for table {table} and container {container.id}") + + duplicates = ( + Splocalecontaineritem.objects.values("container", "name") + .annotate(count=Count("id")) + .filter(count__gt=1) + ) + for duplicate in duplicates: + container_id = duplicate['container'] + name = duplicate['name'] + duplicate_items = Splocalecontaineritem.objects.filter(container_id=container_id, name=name) + item_to_keep = duplicate_items.first() + items_to_delete = duplicate_items.exclude(id=item_to_keep.id) + + Splocaleitemstr.objects.filter(itemdesc_id__in=items_to_delete).update(itemdesc_id=item_to_keep.id) + Splocaleitemstr.objects.filter(itemname_id__in=items_to_delete).update(itemname_id=item_to_keep.id) + items_to_delete.delete() + +def reverse_update_hidden_prop(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in MIGRATION_0023_FIELDS_BIS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + for container in containers: + items = Splocalecontaineritem.objects.filter( + container=container, + name__in=[field_name.lower() for field_name in fields] + ) + logger.info(f"Reverting {items.count()} items for table {table} and container {container.id}") + items.update(ishidden=False) + +def reverse_update_schema_config_field_desc(apps, schema_editor=None): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0023_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = item.name + localized_items_desc.save() + + localized_items_name.text = item.name + localized_items_name.save() + +# ########################################## +# Used in 0024_add_uniqueIdentifier_storage.py +# ########################################## + +def update_storage_unique_id_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + # Add uniqueIdentifier -> storage + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0024_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + +def revert_storage_unique_id_fields(apps): + # Remove uniqueIdentifier -> storage + for table, fields in MIGRATION_0024_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + +# ########################################## +# Used in 0027_CO_children.py +# ########################################## + +def update_co_children_fields(apps): + def update_discipline_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0027_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + + def update_schema_config_field_desc(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0027_UPDATE_FIELDS.items(): + #i.e: Collection Object + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + #i.e: COType + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = new_desc + localized_items_desc.save() + + localized_items_name.text = new_name + localized_items_name.save() + + update_discipline_fields(apps) + update_schema_config_field_desc(apps) + +def revert_co_children_fields(apps): + def revert_update_fields(apps): + for table, fields in MIGRATION_0027_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + + def revert_update_schema_field(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in MIGRATION_0027_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name + ) + + for item in items: + item.ishidden = False + item.save() + + revert_update_fields(apps) + revert_update_schema_field(apps) + +# ########################################## +# Used in 0029_remove_collectionobject_parentco.py +# ########################################## + +def remove_collectionobject_parentco(apps): + def update_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0029_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + + def update_schema_config_field_desc(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0029_UPDATE_FIELDS.items(): + #i.e: Collection Object + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + #i.e: COType + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = new_desc + localized_items_desc.save() + + localized_items_name.text = new_name + localized_items_name.save() + + def hide_co_component(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + + disciplines = Discipline.objects.all() + + for discipline in disciplines: + for table, fields in MIGRATION_0029_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + discipline_id=discipline.id, + ) + for container in containers: + for field_name, _, _ in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + item.ishidden = True + item.save() + + update_fields(apps) + update_schema_config_field_desc(apps) + hide_co_component(apps) + +def revert_remove_collectionobject_parentco(apps): + def revert_update_fields(apps): + for table, fields in MIGRATION_0029_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + + def revert_update_schema_field(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in MIGRATION_0029_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name + ) + + for item in items: + item.ishidden = False + item.save() + + def reverse_hide_co_component(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + + disciplines = Discipline.objects.all() + + for discipline in disciplines: + for table, fields in MIGRATION_0029_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + discipline_id=discipline.id, + ) + for container in containers: + for field_name, _, _ in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + item.ishidden = False + item.save() + + revert_update_fields(apps) + revert_update_schema_field(apps) + reverse_hide_co_component(apps) + +# ########################################## +# Used in 0032_add_quantities_gift.py +# ########################################## + +def add_quantities_gift(apps): + def update_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0032_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + + def update_schema_config_field_desc(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0032_UPDATE_FIELDS.items(): + #i.e: Collection Object + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + #i.e: COType + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = new_desc + localized_items_desc.save() + + localized_items_name.text = new_name + localized_items_name.save() + + update_fields(apps) + update_schema_config_field_desc(apps) + +def revert_add_quantities_gift(apps): + def revert_update_fields(apps): + for table, fields in MIGRATION_0032_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + + def revert_update_schema_field(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in MIGRATION_0032_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name + ) + + for item in items: + item.ishidden = False + item.save() + + revert_update_fields(apps) + revert_update_schema_field(apps) + +# ########################################## +# Used in 0033_update_paleo_desc.py +# ########################################## + +def update_paleo_desc(apps): + def fix_table_description(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table_name, table_desc in MIGRATION_0033_TABLES: + containers = Splocalecontainer.objects.filter(name=table_name.lower(), schematype=0) + Splocaleitemstr.objects.filter(containerdesc__in=containers).update(text=table_desc) + + fix_table_description(apps) + +# ########################################## +# Used in 0034_accession_date_fields.py +# ########################################## + +def update_accession_date_fields(apps): + def update_0034_fields(apps): + """ + Update table-field schema entries for plain field names + (e.g., MIGRATION_0034_FIELDS). + """ + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0034_FIELDS.items(): + for field_name in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field_name, apps) + + def update_0034_schema_config_field_desc(apps): + """ + Update field descriptions and display names using MIGRATION_0034_UPDATE_FIELDS + (tuple: (fieldName, newLabel, newDesc)). + """ + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in MIGRATION_0034_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter(name=table.lower()) + for container in containers: + for (field_name, new_name, new_desc) in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + for item in items: + item.ishidden = True + item.save() + desc_str = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + name_str = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + if not desc_str or not name_str: + continue + desc_str.text = new_desc + desc_str.save() + name_str.text = new_name + name_str.save() + + update_0034_fields(apps) + update_0034_schema_config_field_desc(apps) + +def revert_update_accession_date_fields(apps): + def revert_0034_fields(apps): + """ + Revert table-field entries for plain field names. + """ + for table, fields in MIGRATION_0034_FIELDS.items(): + for field_name in fields: + revert_table_field_schema_config(table, field_name, apps) + + def revert_0034_schema_config_field_desc(apps): + """ + Revert the field name/description updates. + """ + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in MIGRATION_0034_UPDATE_FIELDS.items(): + containers = Splocalecontainer.objects.filter(name=table.lower()) + for container in containers: + for (field_name, _, _) in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + for item in items: + # If needed, reset ishidden or revert text + pass + + revert_0034_fields(apps) + revert_0034_schema_config_field_desc(apps) + +# ########################################## +# Used in 0035_version_required.py +# ########################################## + +def update_version_required(apps): + Discipline = apps.get_model('specify', 'Discipline') + updated_config_params = { + 'isrequired': False, + } + + # Update the schema config for each discipline with the version isHidden change + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0035_FIELDS.items(): + for field in fields: + update_table_field_schema_config_params(table, discipline.id, field, updated_config_params, apps) + +def revert_version_required(apps): + Discipline = apps.get_model('specify', 'Discipline') + updated_config_params = { + 'isrequired': True, + } + + # Revert the schema config for each discipline with the version isHidden change + for discipline in Discipline.objects.all(): + for table, fields in MIGRATION_0035_FIELDS.items(): + for field in fields: + update_table_field_schema_config_params(table, discipline.id, field, updated_config_params, apps) diff --git a/specifyweb/specify/migrations/0002_geo.py b/specifyweb/specify/migrations/0002_geo.py index 27940aec820..442fca149c4 100644 --- a/specifyweb/specify/migrations/0002_geo.py +++ b/specifyweb/specify/migrations/0002_geo.py @@ -3,16 +3,19 @@ import logging from django.db import migrations, models -from django.db.models import F +# from django.db.models import F import django.utils.timezone from specifyweb.specify.models import ( protect_with_blockers ) -from specifyweb.specify.migration_utils.update_schema_config import ( - update_table_schema_config_with_defaults, - revert_table_schema_config, -) from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0002_TABLES as SCHEMA_CONFIG_TABLES +from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.default_cots import ( + create_cogtype_type_picklist, + create_default_collection_types, + create_default_discipline_for_tree_defs, + set_discipline_for_taxon_treedefs, +) from specifyweb.specify.api.utils import create_default_collection_types logger = logging.getLogger(__name__) @@ -28,15 +31,6 @@ # 8. Add discipline relationship to TreeDef tables # 9. Add schema config for new sp7 tables -DEFAULT_COG_TYPES = [ - 'Discrete', - 'Consolidated', - 'Drill Core', -] - -def handle_default_collection_types(apps): - create_default_collection_types(apps) - def revert_default_collection_types(apps): # Reverse handeled by table deletion. pass @@ -45,67 +39,11 @@ def revert_default_cog_types(apps): # Reverse handeled by table deletion pass -def create_default_discipline_for_tree_defs(apps): - Discipline = apps.get_model('specify', 'Discipline') - Institution = apps.get_model('specify', 'Institution') - - for discipline in Discipline.objects.all(): - geography_tree_def = discipline.geographytreedef - geography_tree_def.discipline = discipline - geography_tree_def.save() - - geologic_time_period_tree_def = discipline.geologictimeperiodtreedef - geologic_time_period_tree_def.discipline = discipline - geologic_time_period_tree_def.save() - - lithostrat_tree_def = discipline.lithostrattreedef - lithostrat_tree_def.discipline = discipline - lithostrat_tree_def.save() - - taxon_tree_def = discipline.taxontreedef - taxon_tree_def.discipline = discipline - taxon_tree_def.save() - - for institution in Institution.objects.all(): - storage_tree_def = institution.storagetreedef - storage_tree_def.institution = institution - storage_tree_def.save() - def revert_default_discipline_for_tree_defs(apps): # Reverse handeled by table deletion pass -def create_table_schema_config_with_defaults(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, desc in SCHEMA_CONFIG_TABLES: - update_table_schema_config_with_defaults(table, discipline.id, desc, apps) - -def revert_table_schema_config_with_defaults(apps): - for table, _ in SCHEMA_CONFIG_TABLES: - revert_table_schema_config(table, apps) - -def create_default_collection_object_types(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - Picklistitem = apps.get_model('specify', 'Picklistitem') - - for collection in Collection.objects.all(): - cog_type_picklist = Picklist.objects.create( - name='Default Collection Object Group Types', - issystem=False, - type=0, - readonly=False, - collection=collection - ) - for cog_type in DEFAULT_COG_TYPES: - Picklistitem.objects.create( - title=cog_type, - value=cog_type, - picklist=cog_type_picklist - ) - -def revert_default_collection_object_types(apps): +def revert_cogtype_type_picklist(apps): Collection = apps.get_model('specify', 'Collection') Picklist = apps.get_model('specify', 'Picklist') Picklistitem = apps.get_model('specify', 'Picklistitem') @@ -113,6 +51,7 @@ def revert_default_collection_object_types(apps): for collection in Collection.objects.all(): cog_type_picklist_qs = Picklist.objects.filter( name='Default Collection Object Group Types', + type=0, collection=collection ) if cog_type_picklist_qs.exists(): @@ -120,18 +59,9 @@ def revert_default_collection_object_types(apps): Picklistitem.objects.filter(picklist=cog_type_picklist).delete() cog_type_picklist.delete() -def set_discipline_for_taxon_treedefs(apps): - Collectionobjecttype = apps.get_model('specify', 'Collectionobjecttype') - Taxontreedef = apps.get_model('specify', 'Taxontreedef') - - collection_object_types = Collectionobjecttype.objects.filter( - taxontreedef__discipline__isnull=True - ).annotate( - discipline=F('collection__discipline') - ) - - for cot in collection_object_types: - Taxontreedef.objects.filter(id=cot.taxontreedef_id).update(discipline=cot.discipline) +def revert_geo_table_schema_config_with_defaults(apps): + for table, _ in SCHEMA_CONFIG_TABLES: + usc.revert_table_schema_config(table, apps) class Migration(migrations.Migration): @@ -142,15 +72,15 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - handle_default_collection_types(apps) + create_default_collection_types(apps) create_default_discipline_for_tree_defs(apps) - create_table_schema_config_with_defaults(apps) - create_default_collection_object_types(apps) + usc.create_geo_table_schema_config_with_defaults(apps) + create_cogtype_type_picklist(apps) set_discipline_for_taxon_treedefs(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_default_collection_object_types(apps) - revert_table_schema_config_with_defaults(apps) + revert_cogtype_type_picklist(apps) + revert_geo_table_schema_config_with_defaults(apps) revert_default_discipline_for_tree_defs(apps) revert_default_collection_types(apps) diff --git a/specifyweb/specify/migrations/0003_cotype_picklist.py b/specifyweb/specify/migrations/0003_cotype_picklist.py index f3da52afc9e..9d921b074d0 100644 --- a/specifyweb/specify/migrations/0003_cotype_picklist.py +++ b/specifyweb/specify/migrations/0003_cotype_picklist.py @@ -1,65 +1,28 @@ from django.db import migrations - -PICKLIST_NAME = 'CollectionObjectType' -FIELD_NAME = 'collectionObjectType' -COTYPE_TEXT = 'Collection Object Type' - -def create_cotype_picklist(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - # Create a cotype picklist for each collection - for collection in Collection.objects.all(): - Picklist.objects.get_or_create( - name=PICKLIST_NAME, - issystem=True, - readonly=True, - sizelimit=-1, - sorttype=1, - type=1, - tablename='collectionobjecttype', - collection=collection, - formatter=PICKLIST_NAME - ) +from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.default_cots import create_cotype_picklist, COTYPE_PICKLIST_NAME def revert_cotype_picklist(apps): Picklist = apps.get_model('specify', 'Picklist') - Picklist.objects.filter(name=PICKLIST_NAME).delete() - - -def create_cotype_splocalecontaineritem(apps): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - # Create a Splocalecontaineritem record for each CollectionObject Splocalecontainer - # NOTE: Each discipline has its own CollectionObject Splocalecontainer - for container in Splocalecontainer.objects.filter(name='collectionobject', schematype=0): - container_item, _ = Splocalecontaineritem.objects.get_or_create( - name=FIELD_NAME, - picklistname=PICKLIST_NAME, - type='ManyToOne', - container=container, - isrequired=True - ) - Splocaleitemstr.objects.get_or_create( - language='en', - text=COTYPE_TEXT, - itemname=container_item - ) - Splocaleitemstr.objects.get_or_create( - language='en', - text=COTYPE_TEXT, - itemdesc=container_item - ) + Picklist.objects.filter(name=COTYPE_PICKLIST_NAME).delete() def revert_cotype_splocalecontaineritem(apps): Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - Splocaleitemstr.objects.filter(text=COTYPE_TEXT, itemdesc__container__name='collectionobject', itemdesc__container__schematype=0).delete() - Splocaleitemstr.objects.filter(text=COTYPE_TEXT, itemname__container__name='collectionobject', itemname__container__schematype=0).delete() - Splocalecontaineritem.objects.filter(name=FIELD_NAME, container__name='collectionobject', container__schematype=0).delete() - + Splocaleitemstr.objects.filter( + text=usc.COT_TEXT, + itemdesc__container__name="collectionobject", + itemdesc__container__schematype=0, + ).delete() + Splocaleitemstr.objects.filter( + text=usc.COT_TEXT, + itemname__container__name="collectionobject", + itemname__container__schematype=0, + ).delete() + Splocalecontaineritem.objects.filter( + name=usc.COT_FIELD_NAME, container__name="collectionobject", container__schematype=0 + ).delete() class Migration(migrations.Migration): dependencies = [ @@ -68,7 +31,7 @@ class Migration(migrations.Migration): def apply_migration(apps, schema_editor): create_cotype_picklist(apps) - create_cotype_splocalecontaineritem(apps) + usc.create_cotype_splocalecontaineritem(apps) def revert_migration(apps, schema_editor): revert_cotype_picklist(apps) diff --git a/specifyweb/specify/migrations/0004_stratigraphy_age.py b/specifyweb/specify/migrations/0004_stratigraphy_age.py index 18493d7da13..eb6636fff99 100644 --- a/specifyweb/specify/migrations/0004_stratigraphy_age.py +++ b/specifyweb/specify/migrations/0004_stratigraphy_age.py @@ -1,44 +1,10 @@ # Generated by Django 3.2.15 on 2024-08-02 06:04 -from django.apps import apps as specify_apps from django.db import migrations, models import django.db.models.deletion import django.utils.timezone from specifyweb.specify.models import protect_with_blockers -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, revert_table_schema_config, update_table_field_schema_config_with_defaults, update_table_schema_config_with_defaults - -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0004_TABLES as SCHEMA_CONFIG_TABLES, MIGRATION_0004_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -PICKLIST_NAME = 'AgeType' -DEFAULT_AGE_TYPES = [ - 'Sedimentation', - 'Metamorphism', - 'Erosion', - 'Diagenetic', -] - -def create_agetype_picklist(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - PicklistItem = apps.get_model('specify', 'Picklistitem') - # Create a AgeType picklist for each collection - for collection in Collection.objects.all(): - age_type_picklist, _ = Picklist.objects.get_or_create( - name=PICKLIST_NAME, - issystem=False, - readonly=False, - sizelimit=-1, - sorttype=1, - type=0, - collection=collection, - formatter=PICKLIST_NAME - ) - for age_type in DEFAULT_AGE_TYPES: - PicklistItem.objects.create( - title=age_type, - value=age_type, - picklist=age_type_picklist - ) +from specifyweb.specify.migration_utils import update_schema_config as usc def revert_agetype_picklist(apps): Collection = apps.get_model('specify', 'Collection') @@ -46,29 +12,12 @@ def revert_agetype_picklist(apps): PicklistItem = apps.get_model('specify', 'Picklistitem') for collection in Collection.objects.all(): - age_type_pick_lists = Picklist.objects.filter(name=PICKLIST_NAME, collection=collection) + age_type_pick_lists = Picklist.objects.filter(name=usc.AGETYPE_PICKLIST_NAME, collection=collection) for age_type_pick_list in age_type_pick_lists: PicklistItem.objects.filter(picklist=age_type_pick_list).delete() age_type_pick_list.delete() -def create_table_schema_config_with_defaults(apps): - Discipline = specify_apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, desc in SCHEMA_CONFIG_TABLES: - update_table_schema_config_with_defaults(table, discipline.id, desc, apps) - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def revert_table_schema_config_with_defaults(apps): - for table, _ in SCHEMA_CONFIG_TABLES: - revert_table_schema_config(table, apps) - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - class Migration(migrations.Migration): dependencies = [ @@ -76,11 +25,11 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - create_table_schema_config_with_defaults(apps) - create_agetype_picklist(apps) + usc.create_strat_table_schema_config_with_defaults(apps) + usc.create_agetype_picklist(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_table_schema_config_with_defaults(apps) + usc.revert_strat_table_schema_config_with_defaults(apps) revert_agetype_picklist(apps) operations = [ diff --git a/specifyweb/specify/migrations/0007_schema_config_update.py b/specifyweb/specify/migrations/0007_schema_config_update.py index 408b232679b..53b6a01f8ec 100644 --- a/specifyweb/specify/migrations/0007_schema_config_update.py +++ b/specifyweb/specify/migrations/0007_schema_config_update.py @@ -16,122 +16,7 @@ from django.db import migrations, models import django.db.models.deletion -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0007_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -PICKLIST_NAME = 'COGTypes' -COGTYPE_FIELD_NAME = 'cogType' -SYSTEM_COGTYPE_PICKLIST_NAME = "SystemCOGTypes" - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - # Revert COG -> children before adding to avoid duplicates - revert_table_field_schema_config('CollectionObjectGroup', 'children', apps) - # Add StorageTreeDef -> institution and COG -> children - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - - - # Remove COG -> cojo - revert_table_field_schema_config('CollectionObjectGroup', 'cojo', apps) - - # Remove duplicate CollectionObject -> collectionObjectType - container_items = Splocalecontaineritem.objects.filter(name='collectionObjectType', picklistname=None, container__name='CollectionObject') - for container_item in container_items: - Splocaleitemstr.objects.filter(itemname=container_item).delete() - Splocaleitemstr.objects.filter(itemdesc=container_item).delete() - container_items.delete() - -# NOTE: The reverse function will not re-add the duplicate CO -> coType or COG -> cojo as its unnecessary -def revert_update_fields(apps): - # Remove StorageTreeDef -> institution and COG -> children - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - -def create_cogtype_picklist(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - - # Create a cogtype picklist for each collection - for collection in Collection.objects.all(): - Picklist.objects.get_or_create( - name=PICKLIST_NAME, - issystem=True, - readonly=True, - sizelimit=-1, - sorttype=1, - type=1, - tablename='collectionobjectgrouptype', - collection=collection, - formatter='CollectionObjectGroupType' - ) - -def revert_cogtype_picklist(apps): - Picklist = apps.get_model('specify', 'Picklist') - - Picklist.objects.filter(name=PICKLIST_NAME).delete() - -# Updates COG -> cogtype to use the type 1 picklist created above -def update_cogtype_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgroup', container__schematype=0, name=COGTYPE_FIELD_NAME).update( - picklistname=PICKLIST_NAME, - type='ManyToOne', - isrequired=True - ) - -def revert_cogtype_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgroup', container__schematype=0, name=COGTYPE_FIELD_NAME).update( - picklistname=None, - type=None, - isrequired=None - ) - -def update_systemcogtypes_picklist(apps): - Picklist = apps.get_model('specify', 'Picklist') - - Picklist.objects.filter(name='Default Collection Object Group Types').update( - name=SYSTEM_COGTYPE_PICKLIST_NAME, - type=0, - issystem=True, - readonly=True, - sizelimit=3, - tablename=None - ) - -def revert_systemcogtypes_picklist(apps): - Picklist = apps.get_model('specify', 'Picklist') - - # revert only changes the name and not the other attributes as those were incorrect - Picklist.objects.filter(name=SYSTEM_COGTYPE_PICKLIST_NAME).update( - name='Default Collection Object Group Types', - ) - -# Updates cogtype -> type to use the Default COGType picklist (Drill Core, Discrete, Consolidated) -def update_cogtype_type_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgrouptype', container__schematype=0, name='type').update( - picklistname=SYSTEM_COGTYPE_PICKLIST_NAME, - isrequired=True - ) - -def revert_cogtype_type_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgrouptype', container__schematype=0, name='type').update( - picklistname=None, - isrequired=None - ) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): dependencies = [ @@ -139,24 +24,29 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_fields(apps) - create_cogtype_picklist(apps) - update_cogtype_splocalecontaineritem(apps) - update_systemcogtypes_picklist(apps) - update_cogtype_type_splocalecontaineritem(apps) + usc.update_cog_type_fields(apps) + usc.create_cogtype_picklist(apps) + usc.update_cogtype_splocalecontaineritem(apps) + usc.update_systemcogtypes_picklist(apps) + usc.update_cogtype_type_splocalecontaineritem(apps) def revert_migration(apps, schema_editor): - revert_update_fields(apps) - revert_cogtype_picklist(apps) - revert_cogtype_splocalecontaineritem(apps) - revert_systemcogtypes_picklist(apps) - revert_cogtype_type_splocalecontaineritem(apps) + usc.revert_cog_type_fields(apps) + usc.revert_cogtype_picklist(apps) + usc.revert_cogtype_splocalecontaineritem(apps) + usc.revert_systemcogtypes_picklist(apps) + usc.revert_cogtype_type_splocalecontaineritem(apps) operations = [ migrations.AlterField( - model_name='collectionobjectgroupjoin', - name='parentcog', - field=models.ForeignKey(db_column='ParentCOGID', on_delete=django.db.models.deletion.CASCADE, related_name='children', to='specify.collectionobjectgroup'), + model_name="collectionobjectgroupjoin", + name="parentcog", + field=models.ForeignKey( + db_column="ParentCOGID", + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="specify.collectionobjectgroup", + ), ), - migrations.RunPython(apply_migration, revert_migration, atomic=True) - ] \ No newline at end of file + migrations.RunPython(apply_migration, revert_migration, atomic=True), + ] diff --git a/specifyweb/specify/migrations/0008_ageCitations_fix.py b/specifyweb/specify/migrations/0008_ageCitations_fix.py index 63543ec6a4e..479d9dae005 100644 --- a/specifyweb/specify/migrations/0008_ageCitations_fix.py +++ b/specifyweb/specify/migrations/0008_ageCitations_fix.py @@ -3,23 +3,7 @@ from django.db import migrations, models import specifyweb.specify.models -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0008_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - # Add absoluteAgeCitation -> absoluteAge & Add relativeAgeCitation -> relativeAge - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def revert_update_fields(apps): - # Remove absoluteAgeCitation -> absoluteAge and relativeAgeCitation -> relativeAge - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -28,21 +12,33 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_fields(apps) + usc.update_relative_age_fields(apps) def revert_migration(apps, schema_editor): - revert_update_fields(apps) + usc.update_relative_age_fields(apps) operations = [ migrations.AddField( - model_name='absoluteage', - name='absoluteagecitation', - field=models.ForeignKey(db_column='AbsoluteAgeCitationID', null=True, on_delete=specifyweb.specify.models.protect_with_blockers, related_name='absoluteages', to='specify.absoluteagecitation'), + model_name="absoluteage", + name="absoluteagecitation", + field=models.ForeignKey( + db_column="AbsoluteAgeCitationID", + null=True, + on_delete=specifyweb.specify.models.protect_with_blockers, + related_name="absoluteages", + to="specify.absoluteagecitation", + ), ), migrations.AddField( - model_name='relativeage', - name='relativeagecitation', - field=models.ForeignKey(db_column='RelativeAgeCitationID', null=True, on_delete=specifyweb.specify.models.protect_with_blockers, related_name='relativeages', to='specify.relativeagecitation'), + model_name="relativeage", + name="relativeagecitation", + field=models.ForeignKey( + db_column="RelativeAgeCitationID", + null=True, + on_delete=specifyweb.specify.models.protect_with_blockers, + related_name="relativeages", + to="specify.relativeagecitation", + ), ), - migrations.RunPython(apply_migration, revert_migration, atomic=True) + migrations.RunPython(apply_migration, revert_migration, atomic=True), ] diff --git a/specifyweb/specify/migrations/0009_tectonic_ranks.py b/specifyweb/specify/migrations/0009_tectonic_ranks.py index 5f32ab332c7..54182060ebe 100644 --- a/specifyweb/specify/migrations/0009_tectonic_ranks.py +++ b/specifyweb/specify/migrations/0009_tectonic_ranks.py @@ -1,122 +1,11 @@ from django.db import migrations -from specifyweb.specify.models import protect_with_blockers - -def create_default_tectonic_ranks(apps): - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) - - root, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Root", - title="Root", - rankid=0, - parent=None, - treedef=tectonic_tree_def, - ) - superstructure, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Superstructure", - title="Superstructure", - rankid=10, - parent=root, - treedef=tectonic_tree_def, - ) - tectonic_domain, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Domain", - title="Tectonic Domain", - rankid=20, - parent=superstructure, - treedef=tectonic_tree_def, - ) - tectonic_subdomain, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Subdomain", - title="Tectonic Subdomain", - rankid=30, - parent=tectonic_domain, - treedef=tectonic_tree_def, - ) - tectonic_unit, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Unit", - title="Tectonic Unit", - rankid=40, - parent=tectonic_subdomain, - treedef=tectonic_tree_def, - ) - tectonic_subunit, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Subunit", - title="Tectonic Subunit", - rankid=50, - parent=tectonic_unit, - treedef=tectonic_tree_def, - ) - - Discipline.objects.filter(id=discipline.id).update(tectonicunittreedef=tectonic_tree_def) - -def revert_default_tectonic_ranks(apps, schema_editor): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - tectonic_tree_defs = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline) - - for tectonic_tree_def in tectonic_tree_defs: - tectonic_unit_tree_def_items = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).order_by('-id') - - for item in tectonic_unit_tree_def_items: - TectonicUnit.objects.filter(definitionitem=item).delete() - - item.delete() - - Discipline.objects.filter(id=discipline.id).update(tectonicunittreedef=None) - tectonic_tree_def.delete() - -def create_root_tectonic_node(apps): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicUnitTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - - tectonic_tree_def, created = TectonicUnitTreeDef.objects.get_or_create( - name="Tectonic Unit", - discipline=discipline - ) - - tectonic_tree_def_item, create = TectonicUnitTreeDefItem.objects.get_or_create( - name="Root", - treedef=tectonic_tree_def - ) - - root, _ = TectonicUnit.objects.get_or_create( - name="Root", - fullname="Root", - isaccepted=1, - nodenumber=1, - rankid=0, - parent=None, - definition=tectonic_tree_def, - definitionitem=tectonic_tree_def_item - ) - -def revert_create_root_tectonic_node(apps, schema_editor): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() - - if tectonic_tree_def: - TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() - TectonicUnit.objects.filter( - name="Root" - ).delete() +from specifyweb.specify.migration_utils.tectonic_ranks import ( + create_default_tectonic_ranks, + create_root_tectonic_node, + revert_create_root_tectonic_node, + revert_default_tectonic_ranks, +) class Migration(migrations.Migration): @@ -131,7 +20,11 @@ def consolidated_python_django_migration_operations(apps, schema_editor): def revert_cosolidated_python_django_migration_operations(apps, schema_editor): revert_default_tectonic_ranks(apps, schema_editor) revert_create_root_tectonic_node(apps, schema_editor) - + operations = [ - migrations.RunPython(consolidated_python_django_migration_operations, revert_cosolidated_python_django_migration_operations, atomic=True), + migrations.RunPython( + consolidated_python_django_migration_operations, + revert_cosolidated_python_django_migration_operations, + atomic=True, + ) ] diff --git a/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py b/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py index 2ca78d86333..a289553b43f 100644 --- a/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py +++ b/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py @@ -2,24 +2,7 @@ This migration adds COG -> cojo and CO -> cojo to Schema Config. """ from django.db import migrations -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0012_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - - -def add_cojo_to_schema_config(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults( - table, discipline.id, field, apps) - - -def remove_cojo_from_schema_config(apps): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -28,10 +11,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - add_cojo_to_schema_config(apps) + usc.add_cojo_to_schema_config(apps) def revert_migration(apps, schema_editor): - remove_cojo_from_schema_config(apps) + usc.remove_cojo_from_schema_config(apps) operations = [ migrations.RunPython(apply_migration, revert_migration, atomic=True), diff --git a/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py b/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py index 490ef8b90a7..5c0f5f25d4e 100644 --- a/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py +++ b/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py @@ -2,34 +2,7 @@ from django.db import migrations, models import specifyweb.specify.models -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0013_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - - -def update_schema_config(apps): - revert_table_field_schema_config( - 'CollectionObjectGroup', 'parentCojo', apps) - revert_table_field_schema_config( - 'CollectionObjectGroup', 'parentCog', apps) - - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults( - table, discipline.id, field, apps) - - -def revert_update(apps): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - update_table_field_schema_config_with_defaults( - 'CollectionObjectGroup', discipline.id, 'parentCojo', apps) - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -38,10 +11,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_schema_config(apps) + usc.update_cog_schema_config(apps) def revert_migration(apps, schema_editor): - revert_update(apps) + usc.revert_update_cog_schema_config(apps) operations = [ migrations.RemoveField( diff --git a/specifyweb/specify/migrations/0015_add_version_to_ages.py b/specifyweb/specify/migrations/0015_add_version_to_ages.py index 07b9e9eff80..2dd5d0651b1 100644 --- a/specifyweb/specify/migrations/0015_add_version_to_ages.py +++ b/specifyweb/specify/migrations/0015_add_version_to_ages.py @@ -1,20 +1,7 @@ # Generated by Django 3.2.15 on 2024-12-03 18:59 from django.db import migrations, models -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults - -def update_schema_config(apps): - # Revert before adding to avoid duplicates - revert_update(apps) - - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - update_table_field_schema_config_with_defaults('AbsoluteAge', discipline.id, 'version', apps) - update_table_field_schema_config_with_defaults('RelativeAge', discipline.id, 'version', apps) - -def revert_update(apps): - revert_table_field_schema_config('AbsoluteAge', 'version', apps) - revert_table_field_schema_config('RelativeAge', 'version', apps) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -23,10 +10,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_schema_config(apps) + usc.update_age_schema_config(apps) def revert_migration(apps, schema_editor): - revert_update(apps) + usc.revert_update_age_schema_config(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0017_schemaconfig_fixes.py b/specifyweb/specify/migrations/0017_schemaconfig_fixes.py index 16b9bf605af..f543efeed97 100644 --- a/specifyweb/specify/migrations/0017_schemaconfig_fixes.py +++ b/specifyweb/specify/migrations/0017_schemaconfig_fixes.py @@ -1,23 +1,7 @@ # Generated by Django 3.2.15 on 2024-11-21 20:08 from django.db import migrations -from django.db.models import F - -from specifyweb.specify.migration_utils.update_schema_config import datamodel_type_to_schematype, uncapitilize, camel_to_spaced_title_case -from specifyweb.specify.datamodel import datamodel - -from specifyweb.specify.migration_utils.sp7_schemaconfig import ( - # SpLocaleContainer migration changes - MIGRATION_0002_TABLES, MIGRATION_0004_TABLES, - - # SpLocaleContainerItem migration changes - MIGRATION_0004_FIELDS, MIGRATION_0007_FIELDS, MIGRATION_0008_FIELDS, MIGRATION_0012_FIELDS, MIGRATION_0013_FIELDS -) - -CONTAINER_MIGRATIONS = [MIGRATION_0002_TABLES, MIGRATION_0004_TABLES] - -CONTAINER_ITEM_MIGRATIONS = [ - MIGRATION_0004_FIELDS, MIGRATION_0007_FIELDS, MIGRATION_0008_FIELDS, MIGRATION_0012_FIELDS, MIGRATION_0013_FIELDS] +from specifyweb.specify.migration_utils import update_schema_config as usc """ This migration fixes two bugs introduced in other migrations by the functions @@ -30,70 +14,12 @@ the proper lables """ - -def fix_table_captions(apps): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for migration in CONTAINER_MIGRATIONS: - for table_name, table_desc in migration: - table = datamodel.get_table_strict(table_name) - containers = Splocalecontainer.objects.filter( - name=table_name.lower(), schematype=0) - - # If needed, correct the label of the table in the schema config - if table_desc is not None: - Splocaleitemstr.objects.filter( - containername__in=containers, text=table_desc).update(text=camel_to_spaced_title_case(uncapitilize(table.name))) - - # Update the types for the fields in the table - items = Splocalecontaineritem.objects.filter( - container__in=containers) - for item in items: - datamodel_field = table.get_field(item.name) - if not datamodel_field: - continue - - item.type = datamodel_type_to_schematype( - datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type - item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired - - item.save() - - -def fix_item_types(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for migration in CONTAINER_ITEM_MIGRATIONS: - for table_name, fields in migration.items(): - table = datamodel.get_table_strict(table_name) - items = Splocalecontaineritem.objects.filter( - container__name=table_name.lower(), container__schematype=0, name__in=fields) - - for item in items: - datamodel_field = table.get_field(item.name) - if not datamodel_field: - continue - - item.type = datamodel_type_to_schematype( - datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type - item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired - - item.save() - - -def schemaconfig_fixes(apps, schema_editor): - fix_table_captions(apps) - fix_item_types(apps) - - class Migration(migrations.Migration): dependencies = [ ('specify', '0016_collectionobjecttype_catalognumformatname'), ] operations = [ - migrations.RunPython(schemaconfig_fixes, + migrations.RunPython(usc.schemaconfig_fixes, migrations.RunPython.noop, atomic=True) ] diff --git a/specifyweb/specify/migrations/0018_cot_catnum_schema.py b/specifyweb/specify/migrations/0018_cot_catnum_schema.py index 6919fbe89cf..7e5c378be6d 100644 --- a/specifyweb/specify/migrations/0018_cot_catnum_schema.py +++ b/specifyweb/specify/migrations/0018_cot_catnum_schema.py @@ -1,63 +1,7 @@ # Generated by Django 3.2.15 on 2024-11-21 20:08 from django.db import migrations -from django.db.models import Q - -from specifyweb.specify.datamodel import datamodel -from specifyweb.specify.migration_utils.update_schema_config import camel_to_spaced_title_case - - -def add_cot_catnum_to_schema(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - CollectionObjectType_Table = datamodel.get_table_strict( - 'collectionobjecttype') - catalognumber_format_field = CollectionObjectType_Table.get_field_strict( - 'catalogNumberFormatName') - - for container in Splocalecontainer.objects.filter(name='collectionobjecttype', schematype=0): - schema_item, created = Splocalecontaineritem.objects.get_or_create( - name=catalognumber_format_field.name, type=catalognumber_format_field.type, container=container) - if created: - schema_item.version = 0 - - schema_item.isrequired = catalognumber_format_field.required if schema_item.isrequired is None else schema_item.isrequired - - schema_item.save() - - schema_name = camel_to_spaced_title_case( - catalognumber_format_field.name) - Splocaleitemstr.objects.get_or_create( - language='en', text=schema_name, itemname=schema_item) - Splocaleitemstr.objects.get_or_create( - language='en', text=schema_name, itemdesc=schema_item) - - -def remove_cot_catnum_from_schema(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - CollectionObjectType_Table = datamodel.get_table_strict( - 'collectionobjecttype') - catalognumber_format_field = CollectionObjectType_Table.get_field_strict( - 'catalogNumberFormatName') - - containers = Splocalecontainer.objects.filter( - name='collectionobjecttype', schematype=0) - items = Splocalecontaineritem.objects.filter( - name='catalogNumberFormatName', container__in=containers) - - schema_name = camel_to_spaced_title_case(catalognumber_format_field.name) - filters = Q(language='en', text=schema_name) & ( - Q(itemname__in=items) | Q(itemdesc__in=items)) - locale_strings = Splocaleitemstr.objects.filter(filters) - - locale_strings.delete() - items.delete() - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -66,6 +10,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(add_cot_catnum_to_schema, - remove_cot_catnum_from_schema, atomic=True) + migrations.RunPython(usc.add_cot_catnum_to_schema, + usc.remove_cot_catnum_from_schema, atomic=True) ] diff --git a/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py b/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py index 2b84618a827..455dd7ee047 100644 --- a/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py +++ b/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py @@ -2,24 +2,7 @@ This migration adds Tectonic Unit -> Paleo Context in the Schema Config. """ from django.db import migrations -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0020_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - - -def add_tectonicunit_to_pc_in_schema_config(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults( - table, discipline.id, field, apps) - - -def remove_tectonicunit_from_pc_schema_config(apps): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -28,10 +11,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - add_tectonicunit_to_pc_in_schema_config(apps) + usc.add_tectonicunit_to_pc_in_schema_config(apps) def revert_migration(apps, schema_editor): - remove_tectonicunit_from_pc_schema_config(apps) + usc.remove_tectonicunit_from_pc_schema_config(apps) operations = [ migrations.RunPython(apply_migration, revert_migration, atomic=True), diff --git a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py index bc5738d69d4..26dd616c1de 100644 --- a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py +++ b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py @@ -3,54 +3,55 @@ """ from django.db import migrations +from specifyweb.specify.migration_utils import update_schema_config as usc from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0021_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS from specifyweb.specify.models_utils.model_extras import GEOLOGY_DISCIPLINES, PALEO_DISCIPLINES -def fix_hidden_geo_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Discipline = apps.get_model('specify', 'Discipline') - - excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES - - filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) - - for discipline in filtered_disciplines: - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - discipline_id=discipline.id, - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - items.update(ishidden=True) - -def reverse_fix_hidden_geo_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Discipline = apps.get_model('specify', 'Discipline') - - excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES - - filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) - - for discipline in filtered_disciplines: - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - discipline_id=discipline.id, - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - items.update(ishidden=True) +# def fix_hidden_geo_prop(apps, schema_editor): +# Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') +# Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') +# Discipline = apps.get_model('specify', 'Discipline') + +# excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + +# filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + +# for discipline in filtered_disciplines: +# for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): +# containers = Splocalecontainer.objects.filter( +# name=table.lower(), +# discipline_id=discipline.id, +# ) +# for container in containers: +# for field_name in fields: +# items = Splocalecontaineritem.objects.filter( +# container=container, +# name=field_name.lower() +# ) +# items.update(ishidden=True) + +# def reverse_fix_hidden_geo_prop(apps, schema_editor): +# Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') +# Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') +# Discipline = apps.get_model('specify', 'Discipline') + +# excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + +# filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + +# for discipline in filtered_disciplines: +# for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): +# containers = Splocalecontainer.objects.filter( +# name=table.lower(), +# discipline_id=discipline.id, +# ) +# for container in containers: +# for field_name in fields: +# items = Splocalecontaineritem.objects.filter( +# container=container, +# name=field_name.lower() +# ) +# items.update(ishidden=True) class Migration(migrations.Migration): dependencies = [ @@ -58,5 +59,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(fix_hidden_geo_prop, reverse_fix_hidden_geo_prop, atomic=True) + migrations.RunPython(usc.fix_hidden_geo_prop, usc.reverse_fix_hidden_geo_prop, atomic=True) ] +# migrations.RunPython(fix_hidden_geo_prop, reverse_fix_hidden_geo_prop, atomic=True) diff --git a/specifyweb/specify/migrations/0022_ensure_default_cots.py b/specifyweb/specify/migrations/0022_ensure_default_cots.py index d687d1a9677..9f2219e6645 100644 --- a/specifyweb/specify/migrations/0022_ensure_default_cots.py +++ b/specifyweb/specify/migrations/0022_ensure_default_cots.py @@ -1,5 +1,5 @@ from django.db import migrations -from specifyweb.specify.api.utils import create_default_collection_types +from specifyweb.specify.migration_utils.default_cots import create_default_collection_types class Migration(migrations.Migration): diff --git a/specifyweb/specify/migrations/0023_update_schema_config_text.py b/specifyweb/specify/migrations/0023_update_schema_config_text.py index 533b9e119ff..4ddecb4d688 100644 --- a/specifyweb/specify/migrations/0023_update_schema_config_text.py +++ b/specifyweb/specify/migrations/0023_update_schema_config_text.py @@ -3,109 +3,7 @@ """ from django.db import migrations -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0023_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0023_FIELDS_BIS as SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS - -def update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - #i.e: Collection Object - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - #i.e: COType - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = new_desc - localized_items_desc.save() - - localized_items_name.text = new_name - localized_items_name.save() - -def update_hidden_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - item.ishidden = True - item.save() - -def reverse_update_hidden_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - item.ishidden = False - item.save() - -def reverse_update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = item.name - localized_items_desc.save() - - localized_items_name.text = item.name - localized_items_name.save() +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): dependencies = [ @@ -117,9 +15,9 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython( - update_schema_config_field_desc, reverse_update_schema_config_field_desc, atomic=True + usc.update_schema_config_field_desc, usc.reverse_update_schema_config_field_desc, atomic=True ), migrations.RunPython( - update_hidden_prop, reverse_update_hidden_prop, atomic=True + usc.update_hidden_prop, usc.reverse_update_hidden_prop, atomic=True ), ] \ No newline at end of file diff --git a/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py b/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py index 35cc56b98f9..e7b26de5b9e 100644 --- a/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py +++ b/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py @@ -1,22 +1,6 @@ from django.db import migrations, models - -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0024_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - # Add uniqueIdentifier -> storage - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def revert_update_fields(apps): - # Remove uniqueIdentifier -> storage - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) +from specifyweb.backend.businessrules.uniqueness_rules import DEFAULT_UNIQUENESS_RULES +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -25,10 +9,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_fields(apps) + usc.update_storage_unique_id_fields(apps) def revert_migration(apps, schema_editor): - revert_update_fields(apps) + usc.revert_storage_unique_id_fields(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0027_CO_children.py b/specifyweb/specify/migrations/0027_CO_children.py index 9d7ac906294..0dc7ccadad1 100644 --- a/specifyweb/specify/migrations/0027_CO_children.py +++ b/specifyweb/specify/migrations/0027_CO_children.py @@ -2,75 +2,7 @@ from django.apps import apps as specify_apps from django.db import migrations, models import django.db.models.deletion -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -import specifyweb.specify.models - -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0027_FIELDS as SCHEMA_CONFIG_TABLE_FIELDS, MIGRATION_0027_UPDATE_FIELDS as SCHEMA_CONFIG_CO_TABLE_FIELDS - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_CO_TABLE_FIELDS.items(): - #i.e: Collection Object - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - #i.e: COType - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = new_desc - localized_items_desc.save() - - localized_items_name.text = new_name - localized_items_name.save() - -def revert_update_fields(apps): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - -def revert_update_schema_field(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_CO_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name - ) - - for item in items: - item.ishidden = False - item.save() - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -78,13 +10,11 @@ class Migration(migrations.Migration): ('specify', '0026_taxontreedef_alter_delete'), ] - def consolidated_python_django_migration_operations(apps, schema_editor): - update_fields(apps) - update_schema_config_field_desc(apps, schema_editor) + def apply_migration(apps, schema_editor): + usc.update_co_children_fields(apps) - def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_update_fields(apps) - revert_update_schema_field(apps, schema_editor) + def revert_migration(apps, schema_editor): + usc.revert_co_children_fields(apps) operations = [ migrations.AddField( @@ -92,5 +22,5 @@ def revert_cosolidated_python_django_migration_operations(apps, schema_editor): name='parentco', field=models.ForeignKey(db_column='ParentCOID', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='specify.collectionobject'), ), - migrations.RunPython(consolidated_python_django_migration_operations, revert_cosolidated_python_django_migration_operations, atomic=True), + migrations.RunPython(apply_migration, revert_migration, atomic=True), ] diff --git a/specifyweb/specify/migrations/0029_remove_collectionobject_parentco.py b/specifyweb/specify/migrations/0029_remove_collectionobject_parentco.py index 6bdce282f8e..2eeece84774 100644 --- a/specifyweb/specify/migrations/0029_remove_collectionobject_parentco.py +++ b/specifyweb/specify/migrations/0029_remove_collectionobject_parentco.py @@ -3,136 +3,18 @@ from django.db import migrations, models import django.db.models.deletion -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0029_FIELDS as SCHEMA_CONFIG_TABLE_FIELDS, MIGRATION_0029_UPDATE_FIELDS as SCHEMA_CONFIG_CO_TABLE_FIELDS -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_CO_TABLE_FIELDS.items(): - #i.e: Collection Object - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - #i.e: COType - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = new_desc - localized_items_desc.save() - - localized_items_name.text = new_name - localized_items_name.save() - -def hide_co_component(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Discipline = apps.get_model('specify', 'Discipline') - - disciplines = Discipline.objects.all() - - for discipline in disciplines: - for table, fields in SCHEMA_CONFIG_CO_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - discipline_id=discipline.id, - ) - for container in containers: - for field_name, _, _ in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - item.ishidden = True - item.save() - -def revert_update_fields(apps): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - -def revert_update_schema_field(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_CO_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name - ) - - for item in items: - item.ishidden = False - item.save() - -def reverse_hide_co_component(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Discipline = apps.get_model('specify', 'Discipline') - - disciplines = Discipline.objects.all() - - for discipline in disciplines: - for table, fields in SCHEMA_CONFIG_CO_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - discipline_id=discipline.id, - ) - for container in containers: - for field_name, _, _ in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - item.ishidden = False - item.save() - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): dependencies = [ ('specify', '0028_selectseries'), ] - def consolidated_python_django_migration_operations(apps, schema_editor): - update_fields(apps) - update_schema_config_field_desc(apps, schema_editor) - hide_co_component(apps, schema_editor) + def apply_migration(apps, schema_editor): + usc.remove_collectionobject_parentco(apps) - def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_update_fields(apps) - revert_update_schema_field(apps, schema_editor) - reverse_hide_co_component(apps, schema_editor) + def revert_migration(apps, schema_editor): + usc.revert_remove_collectionobject_parentco(apps) operations = [ migrations.RemoveField( @@ -144,5 +26,5 @@ def revert_cosolidated_python_django_migration_operations(apps, schema_editor): name='componentParent', field=models.ForeignKey(db_column='ComponentParentID', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='components', to='specify.collectionobject'), ), - migrations.RunPython(consolidated_python_django_migration_operations, revert_cosolidated_python_django_migration_operations, atomic=True), + migrations.RunPython(apply_migration, revert_migration, atomic=True), ] diff --git a/specifyweb/specify/migrations/0031_add_default_for_selectseries.py b/specifyweb/specify/migrations/0031_add_default_for_selectseries.py index 47d76e3ad11..41482df6491 100644 --- a/specifyweb/specify/migrations/0031_add_default_for_selectseries.py +++ b/specifyweb/specify/migrations/0031_add_default_for_selectseries.py @@ -7,13 +7,12 @@ from django.db import migrations, models +from specifyweb.specify.migration_utils.misc_migrations import make_selectseries_false -def make_selectseries_false(apps): - spquery = apps.get_model('specify', 'Spquery') - spquery.objects.filter(selectseries=None).update(selectseries=False) -def revert_selectseries(apps, schema_editor): - pass +# def make_selectseries_false(apps): +# spquery = apps.get_model('specify', 'Spquery') +# spquery.objects.filter(selectseries=None).update(selectseries=False) class Migration(migrations.Migration): @@ -31,5 +30,5 @@ def apply_migration(apps, schema_editor): field=models.BooleanField(blank=True, db_column='SelectSeries', default=False, null=True), ), # No revert function added since this patches faulty data - migrations.RunPython(apply_migration, revert_selectseries, atomic=True) + migrations.RunPython(apply_migration, migrations.RunPython.noop, atomic=True) ] diff --git a/specifyweb/specify/migrations/0032_add_quantities_gift.py b/specifyweb/specify/migrations/0032_add_quantities_gift.py index 2676e5d38c8..c3acc18d753 100644 --- a/specifyweb/specify/migrations/0032_add_quantities_gift.py +++ b/specifyweb/specify/migrations/0032_add_quantities_gift.py @@ -3,76 +3,7 @@ from django.db import migrations, models from django.apps import apps as specify_apps -import django.db.models.deletion -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -import specifyweb.specify.models - -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0032_FIELDS as SCHEMA_CONFIG_TABLE_FIELDS, MIGRATION_0032_UPDATE_FIELDS as SCHEMA_CONFIG_GIFTPREPARATION_TABLE_FIELDS - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_GIFTPREPARATION_TABLE_FIELDS.items(): - #i.e: Collection Object - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - #i.e: COType - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = new_desc - localized_items_desc.save() - - localized_items_name.text = new_name - localized_items_name.save() - -def revert_update_fields(apps): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - -def revert_update_schema_field(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_GIFTPREPARATION_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name - ) - - for item in items: - item.ishidden = False - item.save() - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -80,13 +11,11 @@ class Migration(migrations.Migration): ('specify', '0031_add_default_for_selectseries'), ] - def consolidated_python_django_migration_operations(apps, schema_editor): - update_fields(apps) - update_schema_config_field_desc(apps, schema_editor) + def apply_migration(apps, schema_editor): + usc.add_quantities_gift(apps) - def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_update_fields(apps) - revert_update_schema_field(apps, schema_editor) + def revert_migration(apps, schema_editor): + usc.revert_add_quantities_gift(apps) operations = [ migrations.AddField( @@ -99,5 +28,5 @@ def revert_cosolidated_python_django_migration_operations(apps, schema_editor): name='quantityreturned', field=models.IntegerField(blank=True, db_column='QuantityReturned', null=True), ), - migrations.RunPython(consolidated_python_django_migration_operations, revert_cosolidated_python_django_migration_operations, atomic=True), + migrations.RunPython(apply_migration, revert_migration, atomic=True), ] diff --git a/specifyweb/specify/migrations/0033_update_paleo_desc.py b/specifyweb/specify/migrations/0033_update_paleo_desc.py index 882e4ce3c6e..fc9d403e077 100644 --- a/specifyweb/specify/migrations/0033_update_paleo_desc.py +++ b/specifyweb/specify/migrations/0033_update_paleo_desc.py @@ -1,26 +1,8 @@ from django.db import migrations -from django.db.models import F - -from specifyweb.specify.migration_utils.sp7_schemaconfig import ( -MIGRATION_0033_TABLES -) - -CONTAINER_MIGRATIONS = MIGRATION_0033_TABLES - -def fix_table_description(apps): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table_name, table_desc in CONTAINER_MIGRATIONS: - containers = Splocalecontainer.objects.filter( - name=table_name.lower(), schematype=0) - - Splocaleitemstr.objects.filter( - containerdesc__in=containers - ).update(text=table_desc) +from specifyweb.specify.migration_utils import update_schema_config as usc def schemaconfig_fixes(apps, schema_editor): - fix_table_description(apps) + usc.update_paleo_desc(apps) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/specify/migrations/0034_accession_date_fields.py b/specifyweb/specify/migrations/0034_accession_date_fields.py index c443efec719..376cbb80dc5 100644 --- a/specifyweb/specify/migrations/0034_accession_date_fields.py +++ b/specifyweb/specify/migrations/0034_accession_date_fields.py @@ -1,91 +1,13 @@ # Generated by Django 4.2.18 on 2025-06-12 23:10 from django.db import migrations, models +from specifyweb.specify.migration_utils import update_schema_config as usc -# Import your field update utilities -from specifyweb.specify.migration_utils.update_schema_config import ( - revert_table_field_schema_config, - update_table_field_schema_config_with_defaults, -) -from specifyweb.specify.migration_utils.sp7_schemaconfig import ( - MIGRATION_0034_FIELDS as SCHEMA_CONFIG_RAW_FIELDS, - MIGRATION_0034_UPDATE_FIELDS as SCHEMA_CONFIG_FIELD_DESC, -) +def apply_migration(apps, schema_editor): + usc.update_accession_date_fields(apps) -def update_0034_fields(apps): - """ - Update table-field schema entries for plain field names - (e.g., MIGRATION_0034_FIELDS). - """ - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_RAW_FIELDS.items(): - for field_name in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field_name, apps) - -def revert_0034_fields(apps): - """ - Revert table-field entries for plain field names. - """ - for table, fields in SCHEMA_CONFIG_RAW_FIELDS.items(): - for field_name in fields: - revert_table_field_schema_config(table, field_name, apps) - -def update_0034_schema_config_field_desc(apps, schema_editor): - """ - Update field descriptions and display names using MIGRATION_0034_UPDATE_FIELDS - (tuple: (fieldName, newLabel, newDesc)). - """ - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_FIELD_DESC.items(): - containers = Splocalecontainer.objects.filter(name=table.lower()) - for container in containers: - for (field_name, new_name, new_desc) in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - for item in items: - item.ishidden = True - item.save() - desc_str = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - name_str = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - if not desc_str or not name_str: - continue - desc_str.text = new_desc - desc_str.save() - name_str.text = new_name - name_str.save() - -def revert_0034_schema_config_field_desc(apps, schema_editor): - """ - Revert the field name/description updates. - """ - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_FIELD_DESC.items(): - containers = Splocalecontainer.objects.filter(name=table.lower()) - for container in containers: - for (field_name, _, _) in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - for item in items: - # If needed, reset ishidden or revert text - pass - -def consolidated_0034_forward(apps, schema_editor): - update_0034_fields(apps) - update_0034_schema_config_field_desc(apps, schema_editor) - -def consolidated_0034_backward(apps, schema_editor): - revert_0034_schema_config_field_desc(apps, schema_editor) - revert_0034_fields(apps) +def revert_migration(apps, schema_editor): + usc.revert_update_accession_date_fields(apps) class Migration(migrations.Migration): @@ -130,8 +52,8 @@ class Migration(migrations.Migration): field=models.SmallIntegerField(default=1, blank=True, null=True, db_column='DateReceivedPrecision'), ), migrations.RunPython( - consolidated_0034_forward, - consolidated_0034_backward, + apply_migration, + revert_migration, atomic=True, ), ] \ No newline at end of file diff --git a/specifyweb/specify/migrations/0035_version_required.py b/specifyweb/specify/migrations/0035_version_required.py index 3523b126ac8..d02b9b543fc 100644 --- a/specifyweb/specify/migrations/0035_version_required.py +++ b/specifyweb/specify/migrations/0035_version_required.py @@ -1,7 +1,13 @@ # Generated by Django 3.2.15 on 2025-04-02 15:28 from django.db import migrations, models +from specifyweb.specify.migration_utils import update_schema_config as usc +def apply_migration(apps, schema_editor): + usc.update_version_required(apps) + +def revert_migration(apps, schema_editor): + usc.revert_version_required(apps) class Migration(migrations.Migration): @@ -970,4 +976,9 @@ class Migration(migrations.Migration): name='version', field=models.IntegerField(blank=True, db_column='Version', default=0), ), + migrations.RunPython( + apply_migration, + revert_migration, + atomic=True, + ), ] diff --git a/specifyweb/specify/tests/test_utils/test_create_default_collection_types.py b/specifyweb/specify/tests/test_utils/test_create_default_collection_types.py index 3a876c67afa..ec0e7bba66d 100644 --- a/specifyweb/specify/tests/test_utils/test_create_default_collection_types.py +++ b/specifyweb/specify/tests/test_utils/test_create_default_collection_types.py @@ -1,7 +1,7 @@ +from specifyweb.specify.migration_utils.default_cots import create_default_collection_types from specifyweb.specify.tests.test_api import ApiTests from django.apps import apps from specifyweb.specify.models import Collection, Collectionobject, Collectionobjecttype -from specifyweb.specify.api.utils import create_default_collection_types class TestCreateDefaultCollectionTypes(ApiTests): @@ -50,9 +50,9 @@ def test_not_unique_collection_code(self): self.assertEqual(Collection.objects.all().count(), 2) - Collection.objects.all().update( - code="test_code" - ) + for idx, col_id in enumerate(Collection.objects.order_by('id').values_list('id', flat=True)): + code_val = "test_code" if idx - 1 == 0 else f"test_code{idx - 1}" + Collection.objects.filter(id=col_id).update(code=code_val) create_default_collection_types(apps) diff --git a/specifyweb/specify/utils/field_change_info.py b/specifyweb/specify/utils/field_change_info.py index a3a233272f5..8c11afd6c3b 100644 --- a/specifyweb/specify/utils/field_change_info.py +++ b/specifyweb/specify/utils/field_change_info.py @@ -1,7 +1,9 @@ from typing import Any, TypedDict # All field change infos are of this type. Placing it here to avoid circular dependencies with, almost every data modification file. + + class FieldChangeInfo(TypedDict): field_name: str old_value: Any - new_value: Any \ No newline at end of file + new_value: Any From f2a35168af06431dffa2e961eddd2cb738e7ddd4 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 19 May 2026 10:29:58 -0500 Subject: [PATCH 02/17] Fix PR review issues for key migration pipeline --- .../backend/businessrules/migration_utils.py | 8 +- .../migrations/0004_catnum_uniquerule.py | 29 ---- .../backend/businessrules/uniqueness_rules.py | 91 +++++----- specifyweb/backend/patches/migration_utils.py | 12 +- specifyweb/backend/permissions/initialize.py | 24 ++- ...add_dataset_create_recordset_permission.py | 49 +++++- .../0007_add_stats_edit_permission.py | 16 +- .../backend/stored_queries/execution.py | 8 +- .../backend/stored_queries/geology_time.py | 4 +- specifyweb/backend/stored_queries/utils.py | 10 +- .../commands/run_key_migration_functions.py | 13 +- .../specify/migration_utils/default_cots.py | 41 +++-- .../migration_utils/sp7_schemaconfig.py | 2 +- .../specify/migration_utils/tectonic_ranks.py | 37 ++-- .../migration_utils/update_schema_config.py | 56 +++--- specifyweb/specify/migrations/0002_geo.py | 1 - .../migrations/0008_ageCitations_fix.py | 2 +- .../specify/migrations/0009_tectonic_ranks.py | 2 +- .../0021_update_hidden_geo_tables.py | 49 ------ .../specify/migrations/0027_CO_children.py | 162 +++++++++++++++++- 20 files changed, 392 insertions(+), 224 deletions(-) diff --git a/specifyweb/backend/businessrules/migration_utils.py b/specifyweb/backend/businessrules/migration_utils.py index 38265a6b20c..f5d689b9853 100644 --- a/specifyweb/backend/businessrules/migration_utils.py +++ b/specifyweb/backend/businessrules/migration_utils.py @@ -1,4 +1,4 @@ -from typing import Tuple, List +from typing import List from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule @@ -47,6 +47,7 @@ def catnum_rule_uneditable(apps, schema_editor=None): model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", discipline_id=discipline.id, isDatabaseConstraint=False) has_catalognumber_rule = False + matching_rule_ids: List[int] = [] for rule in model_rules: rule_fields = rule.uniquenessrulefield_set.all() @@ -59,8 +60,11 @@ def catnum_rule_uneditable(apps, schema_editor=None): # exception if more than one result is returned if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): has_catalognumber_rule = True + matching_rule_ids.append(rule.id) - if not has_catalognumber_rule: + if has_catalognumber_rule: + UniquenessRule.objects.filter(id__in=matching_rule_ids).update(isDatabaseConstraint=True) + else: create_uniqueness_rule( "Collectionobject", discipline=discipline, diff --git a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py index 33f2798cb81..314a8ad9725 100644 --- a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py +++ b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py @@ -1,35 +1,6 @@ from django.db import migrations from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable, catnum_rule_uneditable -from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule - - -def catnum_rule_editable(apps, schema_editor): - UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') - - candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__isDatabaseConstraint=True, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) - - candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) - - candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) - candidate_rules.update(isDatabaseConstraint=False) - -def catnum_rule_uneditable(apps, schema_editor): - Discipline = apps.get_model('specify', 'Discipline') - UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') - - for discipline in Discipline.objects.all(): - candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__discipline=discipline.id, uniquenessrule__isDatabaseConstraint=False, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) - - candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) - - candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) - if len(candidate_rules) == 0: - create_uniqueness_rule('Collectionobject', discipline=discipline, is_database_constraint=True, fields=['catalogNumber'], scopes=['collection'], registry=apps) - else: - candidate_rules.update(isDatabaseConstraint=True) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/backend/businessrules/uniqueness_rules.py b/specifyweb/backend/businessrules/uniqueness_rules.py index fb3f055dbce..dec36a13a7e 100644 --- a/specifyweb/backend/businessrules/uniqueness_rules.py +++ b/specifyweb/backend/businessrules/uniqueness_rules.py @@ -7,7 +7,7 @@ from django.apps import apps from django.db import connections, transaction -from django.db.models import Q, Count, Exists, OuterRef +from django.db.models import Q, Count from django.db.migrations.recorder import MigrationRecorder from django.core.exceptions import ObjectDoesNotExist from specifyweb.specify.api.crud import get_model @@ -266,13 +266,19 @@ def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f candidate_rules = UniquenessRule.objects.filter(modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) + expected_fields = set(fields) + expected_scopes = set(scopes) for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() - matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) - matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) + existing_fields = set( + all_fields.filter(isScope=False).values_list("fieldPath", flat=True) + ) + existing_scopes = set( + all_fields.filter(isScope=True).values_list("fieldPath", flat=True) + ) # If the rule already exists, skip creating the rule - if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): + if existing_fields == expected_fields and existing_scopes == expected_scopes: return logger.info(f"Creating uniqueness rule on {model_name} with fields {fields} and scopes {scopes} for the discipline {discipline.name if discipline else 'Global'}") @@ -295,13 +301,19 @@ def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f candidate_rules = UniquenessRule.objects.filter( modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) + expected_fields = set(fields) + expected_scopes = set(scopes) rule_ids = [] for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() - matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) - matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) + existing_fields = set( + all_fields.filter(isScope=False).values_list("fieldPath", flat=True) + ) + existing_scopes = set( + all_fields.filter(isScope=True).values_list("fieldPath", flat=True) + ) # If the rule exists, add it to the list of rules to be deleted - if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): + if existing_fields == expected_fields and existing_scopes == expected_scopes: rule_ids.append(rule.id) UniquenessRuleField.objects.filter( @@ -323,24 +335,19 @@ def fix_global_default_rules(registry=None): UniquenessRule = registry.get_model('businessrules', 'UniquenessRule') \ if registry \ else models.UniquenessRule - UniquenessRuleField = registry.get_model('businessrules', 'UniquenessRuleField') \ - if registry \ - else models.UniquenessRuleField - - global_rule_fields = UniquenessRuleField.objects.filter( - uniquenessrule__discipline__isnull=True - ).values( - "uniquenessrule__modelName", - "uniquenessrule__isDatabaseConstraint", - "fieldPath", - "isScope", - ) - global_rule_exists = UniquenessRule.objects.filter( - discipline__isnull=True, - modelName=OuterRef("modelName"), - isDatabaseConstraint=OuterRef("isDatabaseConstraint"), - ) + global_rule_signatures = { + ( + rule.modelName, + rule.isDatabaseConstraint, + frozenset( + rule.uniquenessrulefield_set.values_list("fieldPath", "isScope") + ), + ) + for rule in UniquenessRule.objects.filter( + discipline__isnull=True + ).prefetch_related("uniquenessrulefield_set") + } discipline_ids = ( UniquenessRule.objects.exclude(discipline__isnull=True) @@ -350,28 +357,16 @@ def fix_global_default_rules(registry=None): for discipline_id in discipline_ids: with transaction.atomic(): - # Delete matching fields for this discipline - matching_fields_qs = UniquenessRuleField.objects.filter( - uniquenessrule__discipline_id=discipline_id - ).filter( - Exists( - global_rule_fields.filter( - **{ - "uniquenessrule__modelName": OuterRef("uniquenessrule__modelName"), - "uniquenessrule__isDatabaseConstraint": OuterRef("uniquenessrule__isDatabaseConstraint"), - "fieldPath": OuterRef("fieldPath"), - "isScope": OuterRef("isScope"), - } - ) + for rule in UniquenessRule.objects.filter( + discipline_id=discipline_id + ).prefetch_related("uniquenessrulefield_set"): + signature = ( + rule.modelName, + rule.isDatabaseConstraint, + frozenset( + rule.uniquenessrulefield_set.values_list("fieldPath", "isScope") + ), ) - ) - matching_fields_qs.delete() - - # Delete UniquenessRule rows for this discipline that are now empty - empty_rules_qs = ( - UniquenessRule.objects.filter(discipline_id=discipline_id) - .annotate(field_count=Count("uniquenessrulefield")) - .filter(field_count=0) # now empty after field deletions - .filter(Exists(global_rule_exists)) - ) - empty_rules_qs.delete() + if signature in global_rule_signatures: + rule.uniquenessrulefield_set.all().delete() + rule.delete() diff --git a/specifyweb/backend/patches/migration_utils.py b/specifyweb/backend/patches/migration_utils.py index a773cb10ad0..4dfdc463387 100644 --- a/specifyweb/backend/patches/migration_utils.py +++ b/specifyweb/backend/patches/migration_utils.py @@ -10,6 +10,7 @@ def apply_migrations(app_registry, schema_editor=None): update_coordinates(app_registry, schema_editor) def update_is_accepted(app_registry, schema_editor=None): + db_alias = schema_editor.connection.alias if schema_editor is not None else "default" for tree in SPECIFY_TREES: tree_filters = { "isaccepted": False, @@ -17,20 +18,21 @@ def update_is_accepted(app_registry, schema_editor=None): } tree_model = app_registry.get_model("specify", tree) - tree_model.objects.filter(**tree_filters).update(isaccepted=True) + tree_model._base_manager.using(db_alias).filter(**tree_filters).update(isaccepted=True) def update_coordinates(app_registry, schema_editor=None): + db_alias = schema_editor.connection.alias if schema_editor is not None else "default" Locality = app_registry.get_model("specify", "Locality") - Locality.objects.filter(lat1text__isnull=True, latitude1__isnull=False) \ + Locality._base_manager.using(db_alias).filter(lat1text__isnull=True, latitude1__isnull=False) \ .update(lat1text=F("latitude1")) - Locality.objects.filter(long1text__isnull=True, longitude1__isnull=False) \ + Locality._base_manager.using(db_alias).filter(long1text__isnull=True, longitude1__isnull=False) \ .update(long1text=F("longitude1")) - Locality.objects.filter(lat2text__isnull=True, latitude2__isnull=False) \ + Locality._base_manager.using(db_alias).filter(lat2text__isnull=True, latitude2__isnull=False) \ .update(lat2text=F("latitude2")) - Locality.objects.filter(long2text__isnull=True, longitude2__isnull=False) \ + Locality._base_manager.using(db_alias).filter(long2text__isnull=True, longitude2__isnull=False) \ .update(long2text=F("longitude2")) diff --git a/specifyweb/backend/permissions/initialize.py b/specifyweb/backend/permissions/initialize.py index fb8d981891c..fe04beac578 100644 --- a/specifyweb/backend/permissions/initialize.py +++ b/specifyweb/backend/permissions/initialize.py @@ -47,13 +47,14 @@ def create_admins(apps=apps) -> None: UserPolicy = apps.get_model('permissions', 'UserPolicy') Specifyuser = apps.get_model('specify', 'Specifyuser') - if UserPolicy.objects.filter(collection__isnull=True, resource='%', action='%').exists(): - # don't do anything if there is already any admin. - return - users = Specifyuser.objects.all() for user in users: - if is_sp6_user_permissions_migrated(user, apps): + if UserPolicy.objects.filter( + collection__isnull=True, + specifyuser_id=user.id, + resource="%", + action="%", + ).exists(): continue if is_legacy_admin(user): UserPolicy.objects.get_or_create( @@ -97,17 +98,12 @@ def assign_users_to_roles(apps=apps) -> None: JOIN spprincipal p ON p.SpPrincipalID = up.SpPrincipalID JOIN collection c ON c.UserGroupScopeId = p.userGroupScopeID WHERE p.groupType IS NULL - AND u.SpecifyUserID NOT IN ( - SELECT ur.specifyuser_id + AND NOT EXISTS ( + SELECT 1 FROM spuserrole ur JOIN sprole r ON r.id = ur.role_id - WHERE r.collection_id = p.usergroupscopeid - ) - AND c.UserGroupScopeId NOT IN ( - SELECT DISTINCT r.collection_id - FROM spuserrole ur - JOIN sprole r ON r.id = ur.role_id - JOIN collection c ON c.UserGroupScopeId = r.collection_id + WHERE r.collection_id = c.UserGroupScopeId + AND ur.specifyuser_id = u.SpecifyUserID ); """) diff --git a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py index b9d092009af..936fc821511 100644 --- a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py +++ b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py @@ -2,7 +2,54 @@ from django.db import migrations -from specifyweb.permissions.migration_utils.edit_permissions import add_permission +from specifyweb.backend.workbench.upload.auditlog import auditlog + + +def add_permission(apps, schema_editor=None): + UserPolicy = apps.get_model('permissions', 'UserPolicy') + LibraryRolePolicy = apps.get_model('permissions', 'LibraryRolePolicy') + RolePolicy = apps.get_model('permissions', 'RolePolicy') + + for p in UserPolicy.objects.filter(resource='/workbench/dataset', action='upload'): + if not UserPolicy.objects.filter( + collection=p.collection, + specifyuser=p.specifyuser, + resource=p.resource, + action='create_recordset', + ).exists(): + user_policy = UserPolicy.objects.create( + collection=p.collection, + specifyuser=p.specifyuser, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(user_policy) + + for p in RolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): + if not RolePolicy.objects.filter( + role=p.role, + resource=p.resource, + action='create_recordset', + ).exists(): + role_policy = RolePolicy.objects.create( + role=p.role, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(role_policy, None) + + for p in LibraryRolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): + if not LibraryRolePolicy.objects.filter( + role=p.role, + resource=p.resource, + action='create_recordset', + ).exists(): + library_role_policy = LibraryRolePolicy.objects.create( + role=p.role, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(library_role_policy, None) class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py index fcb886f2dd2..a850816b560 100644 --- a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py +++ b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py @@ -1,5 +1,19 @@ from django.db import migrations -from specifyweb.permissions.migration_utils.edit_permissions import add_stats_edit_permission +from specifyweb.backend.workbench.upload.auditlog import auditlog + + +def add_stats_edit_permission(apps, schema_editor=None): + Role = apps.get_model('permissions', 'Role') + + all_full_access_roles = Role.objects.filter(name="Full Access - Legacy") + + for full_access_role in all_full_access_roles: + new_policy, created = full_access_role.policies.get_or_create( + resource="/preferences/statistics", + action="edit", + ) + if created: + auditlog.insert(new_policy, None) class Migration(migrations.Migration): diff --git a/specifyweb/backend/stored_queries/execution.py b/specifyweb/backend/stored_queries/execution.py index 14121c0d676..9e611590c49 100644 --- a/specifyweb/backend/stored_queries/execution.py +++ b/specifyweb/backend/stored_queries/execution.py @@ -852,7 +852,8 @@ def execute( - log_sqlalchemy_query(query) # Debugging + if settings.DEBUG: + log_sqlalchemy_query(query) # Debugging return {"results": apply_special_post_query_processing(query, tableid, field_specs, collection, user)} def build_query( @@ -1059,7 +1060,8 @@ def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" - log_sqlalchemy_query(query) # Debugging + if settings.DEBUG: + log_sqlalchemy_query(query) # Debugging def parse_catalog_for_comparing(s): def check_for_decimal(s): @@ -1184,4 +1186,4 @@ def apply_special_post_query_processing(query, tableid, field_specs, collection, if should_list_query: return list(query) - return query \ No newline at end of file + return query diff --git a/specifyweb/backend/stored_queries/geology_time.py b/specifyweb/backend/stored_queries/geology_time.py index 0a1a4ffe9d8..8946241b0f2 100644 --- a/specifyweb/backend/stored_queries/geology_time.py +++ b/specifyweb/backend/stored_queries/geology_time.py @@ -1,5 +1,6 @@ import logging import os +from django.conf import settings from django.db.models import Case, FloatField, F, Q, Value, When from django.db.models.functions import Coalesce, Greatest, Least, Cast from specifyweb.backend.stored_queries.utils import log_sqlalchemy_query @@ -977,5 +978,6 @@ def modify_query_add_meta_age_range(query, start_time, end_time, require_full_ov ).label("age") new_query = new_query.add_columns(age_expr) - log_sqlalchemy_query(new_query) + if settings.DEBUG: + log_sqlalchemy_query(new_query) return new_query diff --git a/specifyweb/backend/stored_queries/utils.py b/specifyweb/backend/stored_queries/utils.py index 218423c48f0..1ab1dbcc3cb 100644 --- a/specifyweb/backend/stored_queries/utils.py +++ b/specifyweb/backend/stored_queries/utils.py @@ -5,11 +5,15 @@ def log_sqlalchemy_query(query): # Call this function to debug the raw SQL query generated by SQLAlchemy # TODO: verify theis import + from django.conf import settings from sqlalchemy.dialects import mysql - compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) + if not settings.DEBUG: + return + compiled_query = query.statement.compile(dialect=mysql.dialect()) raw_sql = str(compiled_query).replace('\n', ' ') + ';' logger.debug('='.join(['' for _ in range(80)])) - logger.debug(raw_sql) + logger.debug("SQL: %s", raw_sql) + logger.debug("Params: %s", compiled_query.params) logger.debug('='.join(['' for _ in range(80)])) # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 - # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) \ No newline at end of file + # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index 9aeeb3c8320..ae767208d27 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -4,6 +4,7 @@ from django.core.management.base import BaseCommand from django.apps import apps from django.db import transaction +from django.db.models import Subquery from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable from specifyweb.backend.businessrules.uniqueness_rules import ( apply_default_uniqueness_rules, @@ -81,8 +82,12 @@ def apply_default_uniqueness_rules_to_disciplines(apps): Discipline = apps.get_model('specify', 'Discipline') UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + disciplines_with_rules = UniquenessRule.objects.exclude( + discipline_id__isnull=True + ).values("discipline_id").distinct() for discipline in Discipline.objects.exclude( - id__in=set(UniquenessRule.objects.values_list('discipline_id', flat=True).distinct())): + id__in=Subquery(disciplines_with_rules) + ): apply_default_uniqueness_rules(discipline, registry=apps) @@ -138,7 +143,7 @@ def add_arguments(self, parser): nargs="*", type=str, choices=tuple(self.funcs.keys()), - help=f"Optional: specify one or more functions to run", + help="Optional: specify one or more functions to run", ) parser.add_argument( "--verbose", @@ -171,6 +176,6 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS(f"Applying {func_name}...")) func(self.stdout.write if verbose else None) self.stdout.write(self.style.SUCCESS(f"Applied {func_name}")) - except Exception as e: - logger.error(f"An error occurred: {e}") + except Exception: + logger.exception("An error occurred while running key migrations") raise diff --git a/specifyweb/specify/migration_utils/default_cots.py b/specifyweb/specify/migration_utils/default_cots.py index e9387b5ca8e..1b2c916131c 100644 --- a/specifyweb/specify/migration_utils/default_cots.py +++ b/specifyweb/specify/migration_utils/default_cots.py @@ -135,22 +135,25 @@ def fix_tectonic_unit_treedef_discipline_links(apps): Discipline = apps.get_model('specify', 'Discipline') Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef') - empty_tectonic_unit_treedefs = Tectonicunittreedef.objects.filter(discipline__isnull=True) - empty_disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) - for empty_discipline in empty_disciplines: - if not empty_tectonic_unit_treedefs.exists(): - new_tectonic_unit_treedef = Tectonicunittreedef.objects.create( - name=f'{empty_discipline.name} Tectonic Unit Tree', - discipline=empty_discipline - ) - else: - empty_discipline.tectonicunittreedef = empty_tectonic_unit_treedefs.first() - empty_discipline.save() - - for empty_tectonic_unit_treedef in empty_tectonic_unit_treedefs: - if empty_disciplines.exists(): - empty_tectonic_unit_treedef.discipline = empty_disciplines.first() - empty_tectonic_unit_treedef.save() - else: - empty_tectonic_unit_treedef.discipline = empty_disciplines.last() - empty_tectonic_unit_treedef.save() + empty_tectonic_unit_treedefs = list( + Tectonicunittreedef.objects.filter(discipline__isnull=True) + ) + empty_disciplines = list( + Discipline.objects.filter(tectonicunittreedef__isnull=True) + ) + + for discipline, tectonic_unit_treedef in zip( + empty_disciplines, empty_tectonic_unit_treedefs + ): + tectonic_unit_treedef.discipline = discipline + tectonic_unit_treedef.save() + discipline.tectonicunittreedef = tectonic_unit_treedef + discipline.save() + + for discipline in empty_disciplines[len(empty_tectonic_unit_treedefs):]: + tectonic_unit_treedef = Tectonicunittreedef.objects.create( + name=f'{discipline.name} Tectonic Unit Tree', + discipline=discipline + ) + discipline.tectonicunittreedef = tectonic_unit_treedef + discipline.save() diff --git a/specifyweb/specify/migration_utils/sp7_schemaconfig.py b/specifyweb/specify/migration_utils/sp7_schemaconfig.py index d870be61399..81d28bca732 100644 --- a/specifyweb/specify/migration_utils/sp7_schemaconfig.py +++ b/specifyweb/specify/migration_utils/sp7_schemaconfig.py @@ -148,7 +148,7 @@ 'RelativeAge': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'], 'CollectionObject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'], 'AbsoluteAgeCitation': ['collectionMember', 'absoluteAgeCitationId'], - 'RelativeAgeCitation': ['absoluteAgeCitationId', 'collectionMember'], + 'RelativeAgeCitation': ['relativeAgeCitationId', 'collectionMember'], 'TectonicUnit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'], 'TectonicUnitTreeDefItem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'], 'TectonicUnitTreeDef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId'] diff --git a/specifyweb/specify/migration_utils/tectonic_ranks.py b/specifyweb/specify/migration_utils/tectonic_ranks.py index c2afe88c6fa..1288236778a 100644 --- a/specifyweb/specify/migration_utils/tectonic_ranks.py +++ b/specifyweb/specify/migration_utils/tectonic_ranks.py @@ -6,14 +6,13 @@ def create_default_tectonic_ranks(apps): TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') Discipline = apps.get_model('specify', 'Discipline') - disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True).exclude( - id__in=TectonicTreeDef.objects.values_list('discipline_id', flat=True) - ) + disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) for discipline in disciplines: - tectonic_tree_def = TectonicTreeDef.objects.filter(discipline=discipline).first() - if not tectonic_tree_def: - tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) + tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create( + name="Tectonic Unit", + discipline=discipline, + ) root, _ = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", @@ -93,16 +92,26 @@ def create_root_tectonic_node(apps): tectonic_tree_def = TectonicUnitTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() if not tectonic_tree_def: - tectonic_tree_def, is_created = TectonicUnitTreeDef.objects.get_or_create( + tectonic_tree_def, _ = TectonicUnitTreeDef.objects.get_or_create( name="Tectonic Unit", discipline=discipline ) - tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, name="Root").first() - if not tectonic_tree_def_item: - tectonic_tree_def_item, is_created = TectonicUnitTreeDefItem.objects.get_or_create( + tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter( + treedef=tectonic_tree_def, + name="Root", + ).first() + if tectonic_tree_def_item: + tectonic_tree_def_item.rankid = 0 + tectonic_tree_def_item.parent = None + tectonic_tree_def_item.isenforced = True + tectonic_tree_def_item.save() + else: + tectonic_tree_def_item, _ = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", title="Root", + rankid=0, + parent=None, treedef=tectonic_tree_def, isenforced=True ) @@ -135,7 +144,9 @@ def revert_create_root_tectonic_node(apps, schema_editor=None): tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() if tectonic_tree_def: - TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() TectonicUnit.objects.filter( - name="Root" - ).delete() \ No newline at end of file + name="Root", + definition=tectonic_tree_def, + parent__isnull=True, + ).delete() + TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index 775299a8d07..bffe878184e 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -121,21 +121,20 @@ def update_table_schema_config_with_defaults( issystem=table.system, version=0, ) - if not is_new: - return # If the container already exists, we don't need to update it - # Create a Splocaleitemstr for the table name and description - for k, text in { - "containername": camel_to_spaced_title_case(uncapitilize(table.name)), - "containerdesc": table_config.description, - }.items(): - item_str = { - "text": text, - "language": "en", - "version": 0, - } - item_str[k] = sp_local_container - Splocaleitemstr.objects.get_or_create(**item_str) + if is_new: + # Create a Splocaleitemstr for the table name and description + for k, text in { + "containername": camel_to_spaced_title_case(uncapitilize(table.name)), + "containerdesc": table_config.description, + }.items(): + item_str = { + "text": text, + "language": "en", + "version": 0, + } + item_str[k] = sp_local_container + Splocaleitemstr.objects.get_or_create(**item_str) for field in table.all_fields: update_table_field_schema_config_with_defaults(table_name, discipline_id, field.name, apps) @@ -470,7 +469,7 @@ def update_cog_type_fields(apps): container_items = Splocalecontaineritem.objects.filter( name="collectionObjectType", picklistname=None, - container__name="CollectionObject", + container__name="collectionobject", ) for container_item in container_items: Splocaleitemstr.objects.filter(itemname=container_item).delete() @@ -882,7 +881,7 @@ def update_schema_config_field_desc(apps, schema_editor=None): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -963,7 +962,7 @@ def reverse_update_schema_config_field_desc(apps, schema_editor=None): for field_name, new_name, new_desc in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1027,7 +1026,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1103,7 +1102,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1322,7 +1321,7 @@ def update_0034_schema_config_field_desc(apps): for (field_name, new_name, new_desc) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: item.ishidden = True @@ -1354,18 +1353,27 @@ def revert_0034_schema_config_field_desc(apps): """ Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') for table, fields in MIGRATION_0034_UPDATE_FIELDS.items(): containers = Splocalecontainer.objects.filter(name=table.lower()) for container in containers: - for (field_name, _, _) in fields: + for (field_name, original_label, original_description) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: - # If needed, reset ishidden or revert text - pass + item.ishidden = False + item.save() + desc_str = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + name_str = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + if desc_str is not None: + desc_str.text = original_description + desc_str.save() + if name_str is not None: + name_str.text = original_label + name_str.save() revert_0034_fields(apps) revert_0034_schema_config_field_desc(apps) diff --git a/specifyweb/specify/migrations/0002_geo.py b/specifyweb/specify/migrations/0002_geo.py index 442fca149c4..e84e59d5c2e 100644 --- a/specifyweb/specify/migrations/0002_geo.py +++ b/specifyweb/specify/migrations/0002_geo.py @@ -16,7 +16,6 @@ create_default_discipline_for_tree_defs, set_discipline_for_taxon_treedefs, ) -from specifyweb.specify.api.utils import create_default_collection_types logger = logging.getLogger(__name__) diff --git a/specifyweb/specify/migrations/0008_ageCitations_fix.py b/specifyweb/specify/migrations/0008_ageCitations_fix.py index 479d9dae005..9f058e2dbf3 100644 --- a/specifyweb/specify/migrations/0008_ageCitations_fix.py +++ b/specifyweb/specify/migrations/0008_ageCitations_fix.py @@ -15,7 +15,7 @@ def apply_migration(apps, schema_editor): usc.update_relative_age_fields(apps) def revert_migration(apps, schema_editor): - usc.update_relative_age_fields(apps) + usc.revert_relative_age_fields(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0009_tectonic_ranks.py b/specifyweb/specify/migrations/0009_tectonic_ranks.py index 54182060ebe..f97c7560a19 100644 --- a/specifyweb/specify/migrations/0009_tectonic_ranks.py +++ b/specifyweb/specify/migrations/0009_tectonic_ranks.py @@ -18,8 +18,8 @@ def consolidated_python_django_migration_operations(apps, schema_editor): create_root_tectonic_node(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_default_tectonic_ranks(apps, schema_editor) revert_create_root_tectonic_node(apps, schema_editor) + revert_default_tectonic_ranks(apps, schema_editor) operations = [ migrations.RunPython( diff --git a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py index 26dd616c1de..26a1a2c4c25 100644 --- a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py +++ b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py @@ -4,54 +4,6 @@ from django.db import migrations from specifyweb.specify.migration_utils import update_schema_config as usc -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0021_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS -from specifyweb.specify.models_utils.model_extras import GEOLOGY_DISCIPLINES, PALEO_DISCIPLINES - -# def fix_hidden_geo_prop(apps, schema_editor): -# Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') -# Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') -# Discipline = apps.get_model('specify', 'Discipline') - -# excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES - -# filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) - -# for discipline in filtered_disciplines: -# for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): -# containers = Splocalecontainer.objects.filter( -# name=table.lower(), -# discipline_id=discipline.id, -# ) -# for container in containers: -# for field_name in fields: -# items = Splocalecontaineritem.objects.filter( -# container=container, -# name=field_name.lower() -# ) -# items.update(ishidden=True) - -# def reverse_fix_hidden_geo_prop(apps, schema_editor): -# Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') -# Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') -# Discipline = apps.get_model('specify', 'Discipline') - -# excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES - -# filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) - -# for discipline in filtered_disciplines: -# for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): -# containers = Splocalecontainer.objects.filter( -# name=table.lower(), -# discipline_id=discipline.id, -# ) -# for container in containers: -# for field_name in fields: -# items = Splocalecontaineritem.objects.filter( -# container=container, -# name=field_name.lower() -# ) -# items.update(ishidden=True) class Migration(migrations.Migration): dependencies = [ @@ -61,4 +13,3 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(usc.fix_hidden_geo_prop, usc.reverse_fix_hidden_geo_prop, atomic=True) ] -# migrations.RunPython(fix_hidden_geo_prop, reverse_fix_hidden_geo_prop, atomic=True) diff --git a/specifyweb/specify/migrations/0027_CO_children.py b/specifyweb/specify/migrations/0027_CO_children.py index 0dc7ccadad1..f7f5f8c18a7 100644 --- a/specifyweb/specify/migrations/0027_CO_children.py +++ b/specifyweb/specify/migrations/0027_CO_children.py @@ -1,8 +1,162 @@ # Generated by Django 3.2.15 on 2025-04-11 15:35 -from django.apps import apps as specify_apps +import re + +from django.core.exceptions import MultipleObjectsReturned from django.db import migrations, models +from django.db.models import Q import django.db.models.deletion -from specifyweb.specify.migration_utils import update_schema_config as usc + +from specifyweb.specify.models import datamodel +from specifyweb.specify.models_utils.load_datamodel import FieldDoesNotExistError + + +MIGRATION_0027_FIELDS = { + 'CollectionObject': ['parentCO', 'children'], +} + +MIGRATION_0027_UPDATE_FIELDS = { + 'CollectionObject': [ + ('parentCO', 'Parent Collection Object', 'Parent CollectionObject'), + ('children', 'Children', 'Children'), + ] +} + +HIDDEN_FIELDS = [ + "timestampcreated", "timestampmodified", "version", "createdbyagent", "modifiedbyagent" +] + + +def datamodel_type_to_schematype(datamodel_type: str) -> str: + return "".join(map(lambda type_part: type_part.lower().capitalize(), datamodel_type.split('-'))) + + +def camel_to_spaced_title_case(camel_case: str) -> str: + return re.sub(r"(? Date: Wed, 20 May 2026 11:55:54 -0500 Subject: [PATCH 03/17] Make migration edits while preserving PR review fixes --- .../migrations/0004_catnum_uniquerule.py | 32 ++- .../businessrules/migrations/0005_cojo.py | 2 +- .../0008_fix_global_default_rules.py | 18 -- .../migrations/0001_restore_separators.py | 1 - .../migrations/0002_fix_accepted_taxon.py | 10 +- .../migrations/0003_coordinate_fields_fix.py | 25 +- ...add_dataset_create_recordset_permission.py | 37 +-- .../0007_add_stats_edit_permission.py | 26 +- .../migrations/0008_attachment_import_role.py | 253 +++++++++++++----- .../commands/run_key_migration_functions.py | 3 +- specifyweb/specify/migrations/0002_geo.py | 107 ++++++-- .../migrations/0003_cotype_picklist.py | 71 +++-- .../migrations/0004_stratigraphy_age.py | 61 ++++- .../migrations/0007_schema_config_update.py | 152 +++++++++-- .../migrations/0008_ageCitations_fix.py | 48 ++-- .../specify/migrations/0009_tectonic_ranks.py | 133 ++++++++- .../0012_add_cojo_to_schema_config.py | 23 +- .../0013_collectionobjectgroup_parentcog.py | 33 ++- .../migrations/0015_add_version_to_ages.py | 19 +- .../migrations/0017_schemaconfig_fixes.py | 78 +++++- .../migrations/0018_cot_catnum_schema.py | 62 ++++- ...add_tectonicunit_to_pc_in_schema_config.py | 23 +- .../0021_update_hidden_geo_tables.py | 51 +++- .../migrations/0022_ensure_default_cots.py | 2 +- .../0023_update_schema_config_text.py | 108 +++++++- .../0024_add_uniqueIdentifier_storage.py | 24 +- .../specify/migrations/0027_CO_children.py | 178 ++++-------- .../0029_remove_collectionobject_parentco.py | 130 ++++++++- .../0031_add_default_for_selectseries.py | 11 +- .../migrations/0032_add_quantities_gift.py | 83 +++++- .../migrations/0033_update_paleo_desc.py | 22 +- .../migrations/0034_accession_date_fields.py | 92 ++++++- .../migrations/0035_version_required.py | 11 - 33 files changed, 1492 insertions(+), 437 deletions(-) delete mode 100644 specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py diff --git a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py index 314a8ad9725..fd1e3fe66ec 100644 --- a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py +++ b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py @@ -1,6 +1,36 @@ +from typing import Tuple + from django.db import migrations -from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable, catnum_rule_uneditable +from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule + + +def catnum_rule_editable(apps, schema_editor): + UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') + + candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__isDatabaseConstraint=True, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) + + candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) + + candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) + candidate_rules.update(isDatabaseConstraint=False) + +def catnum_rule_uneditable(apps, schema_editor): + Discipline = apps.get_model('specify', 'Discipline') + UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') + + for discipline in Discipline.objects.all(): + candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__discipline=discipline.id, uniquenessrule__isDatabaseConstraint=False, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) + + candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) + + candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) + if len(candidate_rules) == 0: + create_uniqueness_rule('Collectionobject', discipline=discipline, is_database_constraint=True, fields=['catalogNumber'], scopes=['collection'], registry=apps) + else: + candidate_rules.update(isDatabaseConstraint=True) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/backend/businessrules/migrations/0005_cojo.py b/specifyweb/backend/businessrules/migrations/0005_cojo.py index c006d850af6..e9440465fd8 100644 --- a/specifyweb/backend/businessrules/migrations/0005_cojo.py +++ b/specifyweb/backend/businessrules/migrations/0005_cojo.py @@ -6,7 +6,7 @@ Applies the COJO uniqueness rule to the database. """ def apply_migration(apps, schema_editor): - cojo_rules = DEFAULT_UNIQUENESS_RULES["Collectionobjectgroupjoin"] + cojo_rules = DEFAULT_UNIQUENESS_RULES["CollectionObjectGroupJoin"] Discipline = apps.get_model('specify', 'Discipline') for rule in cojo_rules: diff --git a/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py b/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py deleted file mode 100644 index 49208174a35..00000000000 --- a/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging -from django.db import migrations -from specifyweb.backend.businessrules.uniqueness_rules import fix_global_default_rules - -logger = logging.getLogger(__name__) - -def apply_migration(apps, schema_editor): - fix_global_default_rules(apps) - -class Migration(migrations.Migration): - - dependencies = [ - ('businessrules', '0007_more_uniqueness_rules'), - ] - - operations = [ - migrations.RunPython(apply_migration, migrations.RunPython.noop, atomic=True) - ] diff --git a/specifyweb/backend/patches/migrations/0001_restore_separators.py b/specifyweb/backend/patches/migrations/0001_restore_separators.py index cdc34a59a06..363ee2d5055 100644 --- a/specifyweb/backend/patches/migrations/0001_restore_separators.py +++ b/specifyweb/backend/patches/migrations/0001_restore_separators.py @@ -7,7 +7,6 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('specify', '0001_initial'), ] operations = [ diff --git a/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py b/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py index ce9e4213d4d..ec0ceae29f9 100644 --- a/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py +++ b/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py @@ -2,8 +2,6 @@ from django.db import migrations -from specifyweb.backend.patches.migration_utils import update_is_accepted - class Migration(migrations.Migration): @@ -15,6 +13,10 @@ class Migration(migrations.Migration): # Fixes an issue due to a bug in the WorkBench prior to 7.9.0 that did not # set accepted nodes to IsAccepted = 1 when they were not synonyms. # https://github.com/specify/specify7/issues/5131 - migrations.RunPython(update_is_accepted, - migrations.RunPython.noop, atomic=True), + migrations.RunSQL( + 'UPDATE taxon t1 SET IsAccepted = TRUE WHERE t1.IsAccepted = 0 AND t1.AcceptedId IS NULL', + # This should not need to be reversed, but this allows for rollback without reversing the SQL + reverse_sql='' + ) ] + diff --git a/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py b/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py index eaccb47f93b..2813b08ff16 100644 --- a/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py +++ b/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py @@ -2,8 +2,6 @@ from django.db import migrations -from specifyweb.backend.patches.migration_utils import update_coordinates - class Migration(migrations.Migration): @@ -16,6 +14,25 @@ class Migration(migrations.Migration): # the accompanying text fields are not populated. In cases where the text fields are not populated, # it appears the records do not have coordinates. This script will populate the text fields with the # decimal values where the text fields are empty but the decimal values are not. - migrations.RunPython(update_coordinates, - migrations.RunPython.noop, atomic=True), + migrations.RunSQL( + """ + UPDATE locality + SET Lat1text = Latitude1 + WHERE Lat1text IS NULL AND Latitude1 IS NOT NULL; + + UPDATE locality + SET Long1text = Longitude1 + WHERE Long1text IS NULL AND Longitude1 IS NOT NULL; + + UPDATE locality + SET Lat2text = Latitude2 + WHERE Lat2text IS NULL AND Latitude2 IS NOT NULL; + + UPDATE locality + SET Long2text = Longitude2 + WHERE Long2text IS NULL AND Longitude2 IS NOT NULL; + """, + # This SQL statment should not be reversed, but this allows for rollback + reverse_sql='' + ) ] diff --git a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py index 936fc821511..702defd3036 100644 --- a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py +++ b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py @@ -2,55 +2,32 @@ from django.db import migrations -from specifyweb.backend.workbench.upload.auditlog import auditlog - - -def add_permission(apps, schema_editor=None): +def add_permission(apps, schema_editor): UserPolicy = apps.get_model('permissions', 'UserPolicy') LibraryRolePolicy = apps.get_model('permissions', 'LibraryRolePolicy') RolePolicy = apps.get_model('permissions', 'RolePolicy') for p in UserPolicy.objects.filter(resource='/workbench/dataset', action='upload'): - if not UserPolicy.objects.filter( + UserPolicy.objects.create( collection=p.collection, specifyuser=p.specifyuser, resource=p.resource, action='create_recordset', - ).exists(): - user_policy = UserPolicy.objects.create( - collection=p.collection, - specifyuser=p.specifyuser, - resource=p.resource, - action='create_recordset', - ) - auditlog.insert(user_policy) + ) for p in RolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - if not RolePolicy.objects.filter( + RolePolicy.objects.create( role=p.role, resource=p.resource, action='create_recordset', - ).exists(): - role_policy = RolePolicy.objects.create( - role=p.role, - resource=p.resource, - action='create_recordset', - ) - auditlog.insert(role_policy, None) + ) for p in LibraryRolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - if not LibraryRolePolicy.objects.filter( + LibraryRolePolicy.objects.create( role=p.role, resource=p.resource, action='create_recordset', - ).exists(): - library_role_policy = LibraryRolePolicy.objects.create( - role=p.role, - resource=p.resource, - action='create_recordset', - ) - auditlog.insert(library_role_policy, None) - + ) class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py index a850816b560..6d262b3fc64 100644 --- a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py +++ b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py @@ -1,19 +1,19 @@ from django.db import migrations -from specifyweb.backend.workbench.upload.auditlog import auditlog +from specifyweb.specify.models import Collection # type: ignore +from ..models import Role -def add_stats_edit_permission(apps, schema_editor=None): - Role = apps.get_model('permissions', 'Role') - - all_full_access_roles = Role.objects.filter(name="Full Access - Legacy") - - for full_access_role in all_full_access_roles: - new_policy, created = full_access_role.policies.get_or_create( - resource="/preferences/statistics", - action="edit", - ) - if created: - auditlog.insert(new_policy, None) +def add_stats_edit_permission(apps, schema_editor): + for collection_id in Collection.objects.values_list('id', flat=True): + try: + all_full_access_roles = Role.objects.filter(collection_id=collection_id, name="Full Access - Legacy") + for full_access_role in all_full_access_roles: + full_access_role.policies.create(resource="/preferences" + "/statistics", + action="edit") + except: + print("Failed to assign stats edit permission in collection: ", + collection_id) class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py b/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py index a1cabb22429..60ff925f311 100644 --- a/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py +++ b/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py @@ -1,5 +1,4 @@ from django.db import migrations -from specifyweb.backend.workbench.upload.auditlog import auditlog def add_attachment_import_role(apps, schema_editor): LibraryRole = apps.get_model('permissions', 'LibraryRole') @@ -7,79 +6,187 @@ def add_attachment_import_role(apps, schema_editor): name="Bulk Attachment Import", description="Gives full access to the Bulk Attachment Import. Allows creating new attachments for any attachment table" ) - auditlog.insert(role) - - # Define policy sets grouped by resource - policy_definitions = [ - # Attachment dataset permissions - {'resource': '/attachment_import/dataset', 'actions': ['create', 'update', 'delete', 'upload', 'rollback']}, - # Attachment permissions - {'resource': '/table/attachment', 'actions': ['create', 'read', 'delete']}, - # Table Specific permissions - {'resource': '/table/accession', 'actions': ['read', 'update']}, - {'resource': '/table/accessionattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/agent', 'actions': ['read', 'update']}, - {'resource': '/table/agentattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/borrow', 'actions': ['read', 'update']}, - {'resource': '/table/borrowattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/collectingevent', 'actions': ['read', 'update']}, - {'resource': '/table/collectingeventattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/collectingtrip', 'actions': ['read', 'update']}, - {'resource': '/table/collectingtripattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/collectionobject', 'actions': ['read', 'update']}, - {'resource': '/table/collectionobjectattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/conservdescription', 'actions': ['read', 'update']}, - {'resource': '/table/conservdescriptionattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/conservevent', 'actions': ['read', 'update']}, - {'resource': '/table/conserveventattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/dnasequence', 'actions': ['read', 'update']}, - {'resource': '/table/dnasequenceattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/dnasequencingrun', 'actions': ['read', 'update']}, - {'resource': '/table/dnasequencingrunattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/deaccession', 'actions': ['read', 'update']}, - {'resource': '/table/deaccessionattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/disposal', 'actions': ['read', 'update']}, - {'resource': '/table/disposalattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/exchangein', 'actions': ['read', 'update']}, - {'resource': '/table/exchangeinattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/exchangeout', 'actions': ['read', 'update']}, - {'resource': '/table/exchangeoutattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/fieldnotebook', 'actions': ['read', 'update']}, - {'resource': '/table/fieldnotebookattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/fieldnotebookpage', 'actions': ['read', 'update']}, - {'resource': '/table/fieldnotebookpageattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/fieldnotebookpageset', 'actions': ['read', 'update']}, - {'resource': '/table/fieldnotebookpagesetattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/gift', 'actions': ['read', 'update']}, - {'resource': '/table/giftattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/loan', 'actions': ['read', 'update']}, - {'resource': '/table/loanattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/locality', 'actions': ['read', 'update']}, - {'resource': '/table/localityattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/morphbankview', 'actions': ['read', 'update']}, - {'resource': '/table/attachmentimageattribute', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/permit', 'actions': ['read', 'update']}, - {'resource': '/table/permitattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/preparation', 'actions': ['read', 'update']}, - {'resource': '/table/preparationattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/referencework', 'actions': ['read', 'update']}, - {'resource': '/table/referenceworkattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/repositoryagreement', 'actions': ['read', 'update']}, - {'resource': '/table/repositoryagreementattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/storage', 'actions': ['read', 'update']}, - {'resource': '/table/storageattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/taxon', 'actions': ['read', 'update']}, - {'resource': '/table/taxonattachment', 'actions': ['read', 'create', 'delete']}, - {'resource': '/table/treatmentevent', 'actions': ['read', 'update']}, - {'resource': '/table/treatmenteventattachment', 'actions': ['read', 'create', 'delete']}, - ] + # Attachment dataset permissions + role.policies.create(resource='/attachment_import/dataset', action='create') + role.policies.create(resource='/attachment_import/dataset', action='update') + role.policies.create(resource='/attachment_import/dataset', action='delete') + role.policies.create(resource='/attachment_import/dataset', action='upload') + role.policies.create(resource='/attachment_import/dataset', action='rollback') + + # Attachment permissions + role.policies.create(resource='/table/attachment', action='create') + role.policies.create(resource='/table/attachment', action='read') + role.policies.create(resource='/table/attachment', action='delete') + + # Table Specific permissions (Auto-generated) + role.policies.create(resource='/table/accession', action='read') + role.policies.create(resource='/table/accession', action='update') + role.policies.create(resource='/table/accessionattachment', action='read') + role.policies.create(resource='/table/accessionattachment', action='create') + role.policies.create(resource='/table/accessionattachment', action='delete') + + role.policies.create(resource='/table/agent', action='read') + role.policies.create(resource='/table/agent', action='update') + role.policies.create(resource='/table/agentattachment', action='read') + role.policies.create(resource='/table/agentattachment', action='create') + role.policies.create(resource='/table/agentattachment', action='delete') + + role.policies.create(resource='/table/borrow', action='read') + role.policies.create(resource='/table/borrow', action='update') + role.policies.create(resource='/table/borrowattachment', action='read') + role.policies.create(resource='/table/borrowattachment', action='create') + role.policies.create(resource='/table/borrowattachment', action='delete') + + role.policies.create(resource='/table/collectingevent', action='read') + role.policies.create(resource='/table/collectingevent', action='update') + role.policies.create(resource='/table/collectingeventattachment', action='read') + role.policies.create(resource='/table/collectingeventattachment', action='create') + role.policies.create(resource='/table/collectingeventattachment', action='delete') + + role.policies.create(resource='/table/collectingtrip', action='read') + role.policies.create(resource='/table/collectingtrip', action='update') + role.policies.create(resource='/table/collectingtripattachment', action='read') + role.policies.create(resource='/table/collectingtripattachment', action='create') + role.policies.create(resource='/table/collectingtripattachment', action='delete') + + role.policies.create(resource='/table/collectionobject', action='read') + role.policies.create(resource='/table/collectionobject', action='update') + role.policies.create(resource='/table/collectionobjectattachment', action='read') + role.policies.create(resource='/table/collectionobjectattachment', action='create') + role.policies.create(resource='/table/collectionobjectattachment', action='delete') + + role.policies.create(resource='/table/conservdescription', action='read') + role.policies.create(resource='/table/conservdescription', action='update') + role.policies.create(resource='/table/conservdescriptionattachment', action='read') + role.policies.create(resource='/table/conservdescriptionattachment', action='create') + role.policies.create(resource='/table/conservdescriptionattachment', action='delete') + + role.policies.create(resource='/table/conservevent', action='read') + role.policies.create(resource='/table/conservevent', action='update') + role.policies.create(resource='/table/conserveventattachment', action='read') + role.policies.create(resource='/table/conserveventattachment', action='create') + role.policies.create(resource='/table/conserveventattachment', action='delete') + + role.policies.create(resource='/table/dnasequence', action='read') + role.policies.create(resource='/table/dnasequence', action='update') + role.policies.create(resource='/table/dnasequenceattachment', action='read') + role.policies.create(resource='/table/dnasequenceattachment', action='create') + role.policies.create(resource='/table/dnasequenceattachment', action='delete') + + role.policies.create(resource='/table/dnasequencingrun', action='read') + role.policies.create(resource='/table/dnasequencingrun', action='update') + role.policies.create(resource='/table/dnasequencingrunattachment', action='read') + role.policies.create(resource='/table/dnasequencingrunattachment', action='create') + role.policies.create(resource='/table/dnasequencingrunattachment', action='delete') + + role.policies.create(resource='/table/deaccession', action='read') + role.policies.create(resource='/table/deaccession', action='update') + role.policies.create(resource='/table/deaccessionattachment', action='read') + role.policies.create(resource='/table/deaccessionattachment', action='create') + role.policies.create(resource='/table/deaccessionattachment', action='delete') + + role.policies.create(resource='/table/disposal', action='read') + role.policies.create(resource='/table/disposal', action='update') + role.policies.create(resource='/table/disposalattachment', action='read') + role.policies.create(resource='/table/disposalattachment', action='create') + role.policies.create(resource='/table/disposalattachment', action='delete') + + role.policies.create(resource='/table/exchangein', action='read') + role.policies.create(resource='/table/exchangein', action='update') + role.policies.create(resource='/table/exchangeinattachment', action='read') + role.policies.create(resource='/table/exchangeinattachment', action='create') + role.policies.create(resource='/table/exchangeinattachment', action='delete') + + role.policies.create(resource='/table/exchangeout', action='read') + role.policies.create(resource='/table/exchangeout', action='update') + role.policies.create(resource='/table/exchangeoutattachment', action='read') + role.policies.create(resource='/table/exchangeoutattachment', action='create') + role.policies.create(resource='/table/exchangeoutattachment', action='delete') + + role.policies.create(resource='/table/fieldnotebook', action='read') + role.policies.create(resource='/table/fieldnotebook', action='update') + role.policies.create(resource='/table/fieldnotebookattachment', action='read') + role.policies.create(resource='/table/fieldnotebookattachment', action='create') + role.policies.create(resource='/table/fieldnotebookattachment', action='delete') + + role.policies.create(resource='/table/fieldnotebookpage', action='read') + role.policies.create(resource='/table/fieldnotebookpage', action='update') + role.policies.create(resource='/table/fieldnotebookpageattachment', action='read') + role.policies.create(resource='/table/fieldnotebookpageattachment', action='create') + role.policies.create(resource='/table/fieldnotebookpageattachment', action='delete') + + role.policies.create(resource='/table/fieldnotebookpageset', action='read') + role.policies.create(resource='/table/fieldnotebookpageset', action='update') + role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='read') + role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='create') + role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='delete') + + role.policies.create(resource='/table/gift', action='read') + role.policies.create(resource='/table/gift', action='update') + role.policies.create(resource='/table/giftattachment', action='read') + role.policies.create(resource='/table/giftattachment', action='create') + role.policies.create(resource='/table/giftattachment', action='delete') + + role.policies.create(resource='/table/loan', action='read') + role.policies.create(resource='/table/loan', action='update') + role.policies.create(resource='/table/loanattachment', action='read') + role.policies.create(resource='/table/loanattachment', action='create') + role.policies.create(resource='/table/loanattachment', action='delete') + + role.policies.create(resource='/table/locality', action='read') + role.policies.create(resource='/table/locality', action='update') + role.policies.create(resource='/table/localityattachment', action='read') + role.policies.create(resource='/table/localityattachment', action='create') + role.policies.create(resource='/table/localityattachment', action='delete') + + role.policies.create(resource='/table/morphbankview', action='read') + role.policies.create(resource='/table/morphbankview', action='update') + role.policies.create(resource='/table/attachmentimageattribute', action='read') + role.policies.create(resource='/table/attachmentimageattribute', action='create') + role.policies.create(resource='/table/attachmentimageattribute', action='delete') + + role.policies.create(resource='/table/permit', action='read') + role.policies.create(resource='/table/permit', action='update') + role.policies.create(resource='/table/permitattachment', action='read') + role.policies.create(resource='/table/permitattachment', action='create') + role.policies.create(resource='/table/permitattachment', action='delete') + + role.policies.create(resource='/table/preparation', action='read') + role.policies.create(resource='/table/preparation', action='update') + role.policies.create(resource='/table/preparationattachment', action='read') + role.policies.create(resource='/table/preparationattachment', action='create') + role.policies.create(resource='/table/preparationattachment', action='delete') + + role.policies.create(resource='/table/referencework', action='read') + role.policies.create(resource='/table/referencework', action='update') + role.policies.create(resource='/table/referenceworkattachment', action='read') + role.policies.create(resource='/table/referenceworkattachment', action='create') + role.policies.create(resource='/table/referenceworkattachment', action='delete') + + role.policies.create(resource='/table/repositoryagreement', action='read') + role.policies.create(resource='/table/repositoryagreement', action='update') + role.policies.create(resource='/table/repositoryagreementattachment', action='read') + role.policies.create(resource='/table/repositoryagreementattachment', action='create') + role.policies.create(resource='/table/repositoryagreementattachment', action='delete') + + role.policies.create(resource='/table/storage', action='read') + role.policies.create(resource='/table/storage', action='update') + role.policies.create(resource='/table/storageattachment', action='read') + role.policies.create(resource='/table/storageattachment', action='create') + role.policies.create(resource='/table/storageattachment', action='delete') + + role.policies.create(resource='/table/taxon', action='read') + role.policies.create(resource='/table/taxon', action='update') + role.policies.create(resource='/table/taxonattachment', action='read') + role.policies.create(resource='/table/taxonattachment', action='create') + role.policies.create(resource='/table/taxonattachment', action='delete') + + role.policies.create(resource='/table/treatmentevent', action='read') + role.policies.create(resource='/table/treatmentevent', action='update') + role.policies.create(resource='/table/treatmenteventattachment', action='read') + role.policies.create(resource='/table/treatmenteventattachment', action='create') + role.policies.create(resource='/table/treatmenteventattachment', action='delete') - # Create each policy and log its creation - for policy in policy_definitions: - for action in policy['actions']: - obj, created = role.policies.get_or_create(resource=policy['resource'], action=action) - if created: - auditlog.insert(obj) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index ae767208d27..69d02382e7a 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -74,7 +74,8 @@ def fix_schema_config(stdout: WriteToStdOut | None = None): usc.remove_collectionobject_parentco, # specify 0029 usc.add_quantities_gift, # specify 0032 usc.update_paleo_desc, # specify 0033 - usc.update_accession_date_fields # specify 0034 + usc.update_accession_date_fields, # specify 0034 + usc.update_version_required, # specify 0035 ] log_and_run(funcs, stdout) diff --git a/specifyweb/specify/migrations/0002_geo.py b/specifyweb/specify/migrations/0002_geo.py index e84e59d5c2e..27940aec820 100644 --- a/specifyweb/specify/migrations/0002_geo.py +++ b/specifyweb/specify/migrations/0002_geo.py @@ -3,19 +3,17 @@ import logging from django.db import migrations, models -# from django.db.models import F +from django.db.models import F import django.utils.timezone from specifyweb.specify.models import ( protect_with_blockers ) -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0002_TABLES as SCHEMA_CONFIG_TABLES -from specifyweb.specify.migration_utils import update_schema_config as usc -from specifyweb.specify.migration_utils.default_cots import ( - create_cogtype_type_picklist, - create_default_collection_types, - create_default_discipline_for_tree_defs, - set_discipline_for_taxon_treedefs, +from specifyweb.specify.migration_utils.update_schema_config import ( + update_table_schema_config_with_defaults, + revert_table_schema_config, ) +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0002_TABLES as SCHEMA_CONFIG_TABLES +from specifyweb.specify.api.utils import create_default_collection_types logger = logging.getLogger(__name__) @@ -30,6 +28,15 @@ # 8. Add discipline relationship to TreeDef tables # 9. Add schema config for new sp7 tables +DEFAULT_COG_TYPES = [ + 'Discrete', + 'Consolidated', + 'Drill Core', +] + +def handle_default_collection_types(apps): + create_default_collection_types(apps) + def revert_default_collection_types(apps): # Reverse handeled by table deletion. pass @@ -38,19 +45,74 @@ def revert_default_cog_types(apps): # Reverse handeled by table deletion pass +def create_default_discipline_for_tree_defs(apps): + Discipline = apps.get_model('specify', 'Discipline') + Institution = apps.get_model('specify', 'Institution') + + for discipline in Discipline.objects.all(): + geography_tree_def = discipline.geographytreedef + geography_tree_def.discipline = discipline + geography_tree_def.save() + + geologic_time_period_tree_def = discipline.geologictimeperiodtreedef + geologic_time_period_tree_def.discipline = discipline + geologic_time_period_tree_def.save() + + lithostrat_tree_def = discipline.lithostrattreedef + lithostrat_tree_def.discipline = discipline + lithostrat_tree_def.save() + + taxon_tree_def = discipline.taxontreedef + taxon_tree_def.discipline = discipline + taxon_tree_def.save() + + for institution in Institution.objects.all(): + storage_tree_def = institution.storagetreedef + storage_tree_def.institution = institution + storage_tree_def.save() + def revert_default_discipline_for_tree_defs(apps): # Reverse handeled by table deletion pass -def revert_cogtype_type_picklist(apps): +def create_table_schema_config_with_defaults(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, desc in SCHEMA_CONFIG_TABLES: + update_table_schema_config_with_defaults(table, discipline.id, desc, apps) + +def revert_table_schema_config_with_defaults(apps): + for table, _ in SCHEMA_CONFIG_TABLES: + revert_table_schema_config(table, apps) + +def create_default_collection_object_types(apps): Collection = apps.get_model('specify', 'Collection') Picklist = apps.get_model('specify', 'Picklist') Picklistitem = apps.get_model('specify', 'Picklistitem') for collection in Collection.objects.all(): - cog_type_picklist_qs = Picklist.objects.filter( + cog_type_picklist = Picklist.objects.create( name='Default Collection Object Group Types', + issystem=False, type=0, + readonly=False, + collection=collection + ) + for cog_type in DEFAULT_COG_TYPES: + Picklistitem.objects.create( + title=cog_type, + value=cog_type, + picklist=cog_type_picklist + ) + +def revert_default_collection_object_types(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + Picklistitem = apps.get_model('specify', 'Picklistitem') + + for collection in Collection.objects.all(): + cog_type_picklist_qs = Picklist.objects.filter( + name='Default Collection Object Group Types', collection=collection ) if cog_type_picklist_qs.exists(): @@ -58,9 +120,18 @@ def revert_cogtype_type_picklist(apps): Picklistitem.objects.filter(picklist=cog_type_picklist).delete() cog_type_picklist.delete() -def revert_geo_table_schema_config_with_defaults(apps): - for table, _ in SCHEMA_CONFIG_TABLES: - usc.revert_table_schema_config(table, apps) +def set_discipline_for_taxon_treedefs(apps): + Collectionobjecttype = apps.get_model('specify', 'Collectionobjecttype') + Taxontreedef = apps.get_model('specify', 'Taxontreedef') + + collection_object_types = Collectionobjecttype.objects.filter( + taxontreedef__discipline__isnull=True + ).annotate( + discipline=F('collection__discipline') + ) + + for cot in collection_object_types: + Taxontreedef.objects.filter(id=cot.taxontreedef_id).update(discipline=cot.discipline) class Migration(migrations.Migration): @@ -71,15 +142,15 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - create_default_collection_types(apps) + handle_default_collection_types(apps) create_default_discipline_for_tree_defs(apps) - usc.create_geo_table_schema_config_with_defaults(apps) - create_cogtype_type_picklist(apps) + create_table_schema_config_with_defaults(apps) + create_default_collection_object_types(apps) set_discipline_for_taxon_treedefs(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_cogtype_type_picklist(apps) - revert_geo_table_schema_config_with_defaults(apps) + revert_default_collection_object_types(apps) + revert_table_schema_config_with_defaults(apps) revert_default_discipline_for_tree_defs(apps) revert_default_collection_types(apps) diff --git a/specifyweb/specify/migrations/0003_cotype_picklist.py b/specifyweb/specify/migrations/0003_cotype_picklist.py index 9d921b074d0..f3da52afc9e 100644 --- a/specifyweb/specify/migrations/0003_cotype_picklist.py +++ b/specifyweb/specify/migrations/0003_cotype_picklist.py @@ -1,28 +1,65 @@ from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc -from specifyweb.specify.migration_utils.default_cots import create_cotype_picklist, COTYPE_PICKLIST_NAME + +PICKLIST_NAME = 'CollectionObjectType' +FIELD_NAME = 'collectionObjectType' +COTYPE_TEXT = 'Collection Object Type' + +def create_cotype_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + # Create a cotype picklist for each collection + for collection in Collection.objects.all(): + Picklist.objects.get_or_create( + name=PICKLIST_NAME, + issystem=True, + readonly=True, + sizelimit=-1, + sorttype=1, + type=1, + tablename='collectionobjecttype', + collection=collection, + formatter=PICKLIST_NAME + ) def revert_cotype_picklist(apps): Picklist = apps.get_model('specify', 'Picklist') - Picklist.objects.filter(name=COTYPE_PICKLIST_NAME).delete() + Picklist.objects.filter(name=PICKLIST_NAME).delete() + + +def create_cotype_splocalecontaineritem(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + # Create a Splocalecontaineritem record for each CollectionObject Splocalecontainer + # NOTE: Each discipline has its own CollectionObject Splocalecontainer + for container in Splocalecontainer.objects.filter(name='collectionobject', schematype=0): + container_item, _ = Splocalecontaineritem.objects.get_or_create( + name=FIELD_NAME, + picklistname=PICKLIST_NAME, + type='ManyToOne', + container=container, + isrequired=True + ) + Splocaleitemstr.objects.get_or_create( + language='en', + text=COTYPE_TEXT, + itemname=container_item + ) + Splocaleitemstr.objects.get_or_create( + language='en', + text=COTYPE_TEXT, + itemdesc=container_item + ) def revert_cotype_splocalecontaineritem(apps): Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - Splocaleitemstr.objects.filter( - text=usc.COT_TEXT, - itemdesc__container__name="collectionobject", - itemdesc__container__schematype=0, - ).delete() - Splocaleitemstr.objects.filter( - text=usc.COT_TEXT, - itemname__container__name="collectionobject", - itemname__container__schematype=0, - ).delete() - Splocalecontaineritem.objects.filter( - name=usc.COT_FIELD_NAME, container__name="collectionobject", container__schematype=0 - ).delete() + Splocaleitemstr.objects.filter(text=COTYPE_TEXT, itemdesc__container__name='collectionobject', itemdesc__container__schematype=0).delete() + Splocaleitemstr.objects.filter(text=COTYPE_TEXT, itemname__container__name='collectionobject', itemname__container__schematype=0).delete() + Splocalecontaineritem.objects.filter(name=FIELD_NAME, container__name='collectionobject', container__schematype=0).delete() + class Migration(migrations.Migration): dependencies = [ @@ -31,7 +68,7 @@ class Migration(migrations.Migration): def apply_migration(apps, schema_editor): create_cotype_picklist(apps) - usc.create_cotype_splocalecontaineritem(apps) + create_cotype_splocalecontaineritem(apps) def revert_migration(apps, schema_editor): revert_cotype_picklist(apps) diff --git a/specifyweb/specify/migrations/0004_stratigraphy_age.py b/specifyweb/specify/migrations/0004_stratigraphy_age.py index eb6636fff99..18493d7da13 100644 --- a/specifyweb/specify/migrations/0004_stratigraphy_age.py +++ b/specifyweb/specify/migrations/0004_stratigraphy_age.py @@ -1,10 +1,44 @@ # Generated by Django 3.2.15 on 2024-08-02 06:04 +from django.apps import apps as specify_apps from django.db import migrations, models import django.db.models.deletion import django.utils.timezone from specifyweb.specify.models import protect_with_blockers -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, revert_table_schema_config, update_table_field_schema_config_with_defaults, update_table_schema_config_with_defaults + +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0004_TABLES as SCHEMA_CONFIG_TABLES, MIGRATION_0004_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + +PICKLIST_NAME = 'AgeType' +DEFAULT_AGE_TYPES = [ + 'Sedimentation', + 'Metamorphism', + 'Erosion', + 'Diagenetic', +] + +def create_agetype_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + PicklistItem = apps.get_model('specify', 'Picklistitem') + # Create a AgeType picklist for each collection + for collection in Collection.objects.all(): + age_type_picklist, _ = Picklist.objects.get_or_create( + name=PICKLIST_NAME, + issystem=False, + readonly=False, + sizelimit=-1, + sorttype=1, + type=0, + collection=collection, + formatter=PICKLIST_NAME + ) + for age_type in DEFAULT_AGE_TYPES: + PicklistItem.objects.create( + title=age_type, + value=age_type, + picklist=age_type_picklist + ) def revert_agetype_picklist(apps): Collection = apps.get_model('specify', 'Collection') @@ -12,12 +46,29 @@ def revert_agetype_picklist(apps): PicklistItem = apps.get_model('specify', 'Picklistitem') for collection in Collection.objects.all(): - age_type_pick_lists = Picklist.objects.filter(name=usc.AGETYPE_PICKLIST_NAME, collection=collection) + age_type_pick_lists = Picklist.objects.filter(name=PICKLIST_NAME, collection=collection) for age_type_pick_list in age_type_pick_lists: PicklistItem.objects.filter(picklist=age_type_pick_list).delete() age_type_pick_list.delete() +def create_table_schema_config_with_defaults(apps): + Discipline = specify_apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, desc in SCHEMA_CONFIG_TABLES: + update_table_schema_config_with_defaults(table, discipline.id, desc, apps) + + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + +def revert_table_schema_config_with_defaults(apps): + for table, _ in SCHEMA_CONFIG_TABLES: + revert_table_schema_config(table, apps) + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + class Migration(migrations.Migration): dependencies = [ @@ -25,11 +76,11 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - usc.create_strat_table_schema_config_with_defaults(apps) - usc.create_agetype_picklist(apps) + create_table_schema_config_with_defaults(apps) + create_agetype_picklist(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - usc.revert_strat_table_schema_config_with_defaults(apps) + revert_table_schema_config_with_defaults(apps) revert_agetype_picklist(apps) operations = [ diff --git a/specifyweb/specify/migrations/0007_schema_config_update.py b/specifyweb/specify/migrations/0007_schema_config_update.py index 53b6a01f8ec..408b232679b 100644 --- a/specifyweb/specify/migrations/0007_schema_config_update.py +++ b/specifyweb/specify/migrations/0007_schema_config_update.py @@ -16,7 +16,122 @@ from django.db import migrations, models import django.db.models.deletion -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0007_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + +PICKLIST_NAME = 'COGTypes' +COGTYPE_FIELD_NAME = 'cogType' +SYSTEM_COGTYPE_PICKLIST_NAME = "SystemCOGTypes" + +def update_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + # Revert COG -> children before adding to avoid duplicates + revert_table_field_schema_config('CollectionObjectGroup', 'children', apps) + # Add StorageTreeDef -> institution and COG -> children + for discipline in Discipline.objects.all(): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + + + # Remove COG -> cojo + revert_table_field_schema_config('CollectionObjectGroup', 'cojo', apps) + + # Remove duplicate CollectionObject -> collectionObjectType + container_items = Splocalecontaineritem.objects.filter(name='collectionObjectType', picklistname=None, container__name='CollectionObject') + for container_item in container_items: + Splocaleitemstr.objects.filter(itemname=container_item).delete() + Splocaleitemstr.objects.filter(itemdesc=container_item).delete() + container_items.delete() + +# NOTE: The reverse function will not re-add the duplicate CO -> coType or COG -> cojo as its unnecessary +def revert_update_fields(apps): + # Remove StorageTreeDef -> institution and COG -> children + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + +def create_cogtype_picklist(apps): + Collection = apps.get_model('specify', 'Collection') + Picklist = apps.get_model('specify', 'Picklist') + + # Create a cogtype picklist for each collection + for collection in Collection.objects.all(): + Picklist.objects.get_or_create( + name=PICKLIST_NAME, + issystem=True, + readonly=True, + sizelimit=-1, + sorttype=1, + type=1, + tablename='collectionobjectgrouptype', + collection=collection, + formatter='CollectionObjectGroupType' + ) + +def revert_cogtype_picklist(apps): + Picklist = apps.get_model('specify', 'Picklist') + + Picklist.objects.filter(name=PICKLIST_NAME).delete() + +# Updates COG -> cogtype to use the type 1 picklist created above +def update_cogtype_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + Splocalecontaineritem.objects.filter(container__name='collectionobjectgroup', container__schematype=0, name=COGTYPE_FIELD_NAME).update( + picklistname=PICKLIST_NAME, + type='ManyToOne', + isrequired=True + ) + +def revert_cogtype_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + Splocalecontaineritem.objects.filter(container__name='collectionobjectgroup', container__schematype=0, name=COGTYPE_FIELD_NAME).update( + picklistname=None, + type=None, + isrequired=None + ) + +def update_systemcogtypes_picklist(apps): + Picklist = apps.get_model('specify', 'Picklist') + + Picklist.objects.filter(name='Default Collection Object Group Types').update( + name=SYSTEM_COGTYPE_PICKLIST_NAME, + type=0, + issystem=True, + readonly=True, + sizelimit=3, + tablename=None + ) + +def revert_systemcogtypes_picklist(apps): + Picklist = apps.get_model('specify', 'Picklist') + + # revert only changes the name and not the other attributes as those were incorrect + Picklist.objects.filter(name=SYSTEM_COGTYPE_PICKLIST_NAME).update( + name='Default Collection Object Group Types', + ) + +# Updates cogtype -> type to use the Default COGType picklist (Drill Core, Discrete, Consolidated) +def update_cogtype_type_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + Splocalecontaineritem.objects.filter(container__name='collectionobjectgrouptype', container__schematype=0, name='type').update( + picklistname=SYSTEM_COGTYPE_PICKLIST_NAME, + isrequired=True + ) + +def revert_cogtype_type_splocalecontaineritem(apps): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + Splocalecontaineritem.objects.filter(container__name='collectionobjectgrouptype', container__schematype=0, name='type').update( + picklistname=None, + isrequired=None + ) class Migration(migrations.Migration): dependencies = [ @@ -24,29 +139,24 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.update_cog_type_fields(apps) - usc.create_cogtype_picklist(apps) - usc.update_cogtype_splocalecontaineritem(apps) - usc.update_systemcogtypes_picklist(apps) - usc.update_cogtype_type_splocalecontaineritem(apps) + update_fields(apps) + create_cogtype_picklist(apps) + update_cogtype_splocalecontaineritem(apps) + update_systemcogtypes_picklist(apps) + update_cogtype_type_splocalecontaineritem(apps) def revert_migration(apps, schema_editor): - usc.revert_cog_type_fields(apps) - usc.revert_cogtype_picklist(apps) - usc.revert_cogtype_splocalecontaineritem(apps) - usc.revert_systemcogtypes_picklist(apps) - usc.revert_cogtype_type_splocalecontaineritem(apps) + revert_update_fields(apps) + revert_cogtype_picklist(apps) + revert_cogtype_splocalecontaineritem(apps) + revert_systemcogtypes_picklist(apps) + revert_cogtype_type_splocalecontaineritem(apps) operations = [ migrations.AlterField( - model_name="collectionobjectgroupjoin", - name="parentcog", - field=models.ForeignKey( - db_column="ParentCOGID", - on_delete=django.db.models.deletion.CASCADE, - related_name="children", - to="specify.collectionobjectgroup", - ), + model_name='collectionobjectgroupjoin', + name='parentcog', + field=models.ForeignKey(db_column='ParentCOGID', on_delete=django.db.models.deletion.CASCADE, related_name='children', to='specify.collectionobjectgroup'), ), - migrations.RunPython(apply_migration, revert_migration, atomic=True), - ] + migrations.RunPython(apply_migration, revert_migration, atomic=True) + ] \ No newline at end of file diff --git a/specifyweb/specify/migrations/0008_ageCitations_fix.py b/specifyweb/specify/migrations/0008_ageCitations_fix.py index 9f058e2dbf3..63543ec6a4e 100644 --- a/specifyweb/specify/migrations/0008_ageCitations_fix.py +++ b/specifyweb/specify/migrations/0008_ageCitations_fix.py @@ -3,7 +3,23 @@ from django.db import migrations, models import specifyweb.specify.models -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0008_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + +def update_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + # Add absoluteAgeCitation -> absoluteAge & Add relativeAgeCitation -> relativeAge + for discipline in Discipline.objects.all(): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + +def revert_update_fields(apps): + # Remove absoluteAgeCitation -> absoluteAge and relativeAgeCitation -> relativeAge + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) class Migration(migrations.Migration): @@ -12,33 +28,21 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.update_relative_age_fields(apps) + update_fields(apps) def revert_migration(apps, schema_editor): - usc.revert_relative_age_fields(apps) + revert_update_fields(apps) operations = [ migrations.AddField( - model_name="absoluteage", - name="absoluteagecitation", - field=models.ForeignKey( - db_column="AbsoluteAgeCitationID", - null=True, - on_delete=specifyweb.specify.models.protect_with_blockers, - related_name="absoluteages", - to="specify.absoluteagecitation", - ), + model_name='absoluteage', + name='absoluteagecitation', + field=models.ForeignKey(db_column='AbsoluteAgeCitationID', null=True, on_delete=specifyweb.specify.models.protect_with_blockers, related_name='absoluteages', to='specify.absoluteagecitation'), ), migrations.AddField( - model_name="relativeage", - name="relativeagecitation", - field=models.ForeignKey( - db_column="RelativeAgeCitationID", - null=True, - on_delete=specifyweb.specify.models.protect_with_blockers, - related_name="relativeages", - to="specify.relativeagecitation", - ), + model_name='relativeage', + name='relativeagecitation', + field=models.ForeignKey(db_column='RelativeAgeCitationID', null=True, on_delete=specifyweb.specify.models.protect_with_blockers, related_name='relativeages', to='specify.relativeagecitation'), ), - migrations.RunPython(apply_migration, revert_migration, atomic=True), + migrations.RunPython(apply_migration, revert_migration, atomic=True) ] diff --git a/specifyweb/specify/migrations/0009_tectonic_ranks.py b/specifyweb/specify/migrations/0009_tectonic_ranks.py index f97c7560a19..5f32ab332c7 100644 --- a/specifyweb/specify/migrations/0009_tectonic_ranks.py +++ b/specifyweb/specify/migrations/0009_tectonic_ranks.py @@ -1,11 +1,122 @@ from django.db import migrations +from specifyweb.specify.models import protect_with_blockers -from specifyweb.specify.migration_utils.tectonic_ranks import ( - create_default_tectonic_ranks, - create_root_tectonic_node, - revert_create_root_tectonic_node, - revert_default_tectonic_ranks, -) + +def create_default_tectonic_ranks(apps): + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) + + root, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Root", + title="Root", + rankid=0, + parent=None, + treedef=tectonic_tree_def, + ) + superstructure, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Superstructure", + title="Superstructure", + rankid=10, + parent=root, + treedef=tectonic_tree_def, + ) + tectonic_domain, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Domain", + title="Tectonic Domain", + rankid=20, + parent=superstructure, + treedef=tectonic_tree_def, + ) + tectonic_subdomain, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Subdomain", + title="Tectonic Subdomain", + rankid=30, + parent=tectonic_domain, + treedef=tectonic_tree_def, + ) + tectonic_unit, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Unit", + title="Tectonic Unit", + rankid=40, + parent=tectonic_subdomain, + treedef=tectonic_tree_def, + ) + tectonic_subunit, _ = TectonicUnitTreeDefItem.objects.get_or_create( + name="Tectonic Subunit", + title="Tectonic Subunit", + rankid=50, + parent=tectonic_unit, + treedef=tectonic_tree_def, + ) + + Discipline.objects.filter(id=discipline.id).update(tectonicunittreedef=tectonic_tree_def) + +def revert_default_tectonic_ranks(apps, schema_editor): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + tectonic_tree_defs = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline) + + for tectonic_tree_def in tectonic_tree_defs: + tectonic_unit_tree_def_items = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).order_by('-id') + + for item in tectonic_unit_tree_def_items: + TectonicUnit.objects.filter(definitionitem=item).delete() + + item.delete() + + Discipline.objects.filter(id=discipline.id).update(tectonicunittreedef=None) + tectonic_tree_def.delete() + +def create_root_tectonic_node(apps): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicUnitTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + + tectonic_tree_def, created = TectonicUnitTreeDef.objects.get_or_create( + name="Tectonic Unit", + discipline=discipline + ) + + tectonic_tree_def_item, create = TectonicUnitTreeDefItem.objects.get_or_create( + name="Root", + treedef=tectonic_tree_def + ) + + root, _ = TectonicUnit.objects.get_or_create( + name="Root", + fullname="Root", + isaccepted=1, + nodenumber=1, + rankid=0, + parent=None, + definition=tectonic_tree_def, + definitionitem=tectonic_tree_def_item + ) + +def revert_create_root_tectonic_node(apps, schema_editor): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for discipline in Discipline.objects.all(): + tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() + + if tectonic_tree_def: + TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() + TectonicUnit.objects.filter( + name="Root" + ).delete() class Migration(migrations.Migration): @@ -18,13 +129,9 @@ def consolidated_python_django_migration_operations(apps, schema_editor): create_root_tectonic_node(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_create_root_tectonic_node(apps, schema_editor) revert_default_tectonic_ranks(apps, schema_editor) - + revert_create_root_tectonic_node(apps, schema_editor) + operations = [ - migrations.RunPython( - consolidated_python_django_migration_operations, - revert_cosolidated_python_django_migration_operations, - atomic=True, - ) + migrations.RunPython(consolidated_python_django_migration_operations, revert_cosolidated_python_django_migration_operations, atomic=True), ] diff --git a/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py b/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py index a289553b43f..2ca78d86333 100644 --- a/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py +++ b/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py @@ -2,7 +2,24 @@ This migration adds COG -> cojo and CO -> cojo to Schema Config. """ from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0012_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + + +def add_cojo_to_schema_config(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults( + table, discipline.id, field, apps) + + +def remove_cojo_from_schema_config(apps): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + class Migration(migrations.Migration): @@ -11,10 +28,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.add_cojo_to_schema_config(apps) + add_cojo_to_schema_config(apps) def revert_migration(apps, schema_editor): - usc.remove_cojo_from_schema_config(apps) + remove_cojo_from_schema_config(apps) operations = [ migrations.RunPython(apply_migration, revert_migration, atomic=True), diff --git a/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py b/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py index 5c0f5f25d4e..490ef8b90a7 100644 --- a/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py +++ b/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py @@ -2,7 +2,34 @@ from django.db import migrations, models import specifyweb.specify.models -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0013_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + + +def update_schema_config(apps): + revert_table_field_schema_config( + 'CollectionObjectGroup', 'parentCojo', apps) + revert_table_field_schema_config( + 'CollectionObjectGroup', 'parentCog', apps) + + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults( + table, discipline.id, field, apps) + + +def revert_update(apps): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + update_table_field_schema_config_with_defaults( + 'CollectionObjectGroup', discipline.id, 'parentCojo', apps) + class Migration(migrations.Migration): @@ -11,10 +38,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.update_cog_schema_config(apps) + update_schema_config(apps) def revert_migration(apps, schema_editor): - usc.revert_update_cog_schema_config(apps) + revert_update(apps) operations = [ migrations.RemoveField( diff --git a/specifyweb/specify/migrations/0015_add_version_to_ages.py b/specifyweb/specify/migrations/0015_add_version_to_ages.py index 2dd5d0651b1..07b9e9eff80 100644 --- a/specifyweb/specify/migrations/0015_add_version_to_ages.py +++ b/specifyweb/specify/migrations/0015_add_version_to_ages.py @@ -1,7 +1,20 @@ # Generated by Django 3.2.15 on 2024-12-03 18:59 from django.db import migrations, models -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults + +def update_schema_config(apps): + # Revert before adding to avoid duplicates + revert_update(apps) + + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + update_table_field_schema_config_with_defaults('AbsoluteAge', discipline.id, 'version', apps) + update_table_field_schema_config_with_defaults('RelativeAge', discipline.id, 'version', apps) + +def revert_update(apps): + revert_table_field_schema_config('AbsoluteAge', 'version', apps) + revert_table_field_schema_config('RelativeAge', 'version', apps) class Migration(migrations.Migration): @@ -10,10 +23,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.update_age_schema_config(apps) + update_schema_config(apps) def revert_migration(apps, schema_editor): - usc.revert_update_age_schema_config(apps) + revert_update(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0017_schemaconfig_fixes.py b/specifyweb/specify/migrations/0017_schemaconfig_fixes.py index f543efeed97..16b9bf605af 100644 --- a/specifyweb/specify/migrations/0017_schemaconfig_fixes.py +++ b/specifyweb/specify/migrations/0017_schemaconfig_fixes.py @@ -1,7 +1,23 @@ # Generated by Django 3.2.15 on 2024-11-21 20:08 from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc +from django.db.models import F + +from specifyweb.specify.migration_utils.update_schema_config import datamodel_type_to_schematype, uncapitilize, camel_to_spaced_title_case +from specifyweb.specify.datamodel import datamodel + +from specifyweb.specify.migration_utils.sp7_schemaconfig import ( + # SpLocaleContainer migration changes + MIGRATION_0002_TABLES, MIGRATION_0004_TABLES, + + # SpLocaleContainerItem migration changes + MIGRATION_0004_FIELDS, MIGRATION_0007_FIELDS, MIGRATION_0008_FIELDS, MIGRATION_0012_FIELDS, MIGRATION_0013_FIELDS +) + +CONTAINER_MIGRATIONS = [MIGRATION_0002_TABLES, MIGRATION_0004_TABLES] + +CONTAINER_ITEM_MIGRATIONS = [ + MIGRATION_0004_FIELDS, MIGRATION_0007_FIELDS, MIGRATION_0008_FIELDS, MIGRATION_0012_FIELDS, MIGRATION_0013_FIELDS] """ This migration fixes two bugs introduced in other migrations by the functions @@ -14,12 +30,70 @@ the proper lables """ + +def fix_table_captions(apps): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for migration in CONTAINER_MIGRATIONS: + for table_name, table_desc in migration: + table = datamodel.get_table_strict(table_name) + containers = Splocalecontainer.objects.filter( + name=table_name.lower(), schematype=0) + + # If needed, correct the label of the table in the schema config + if table_desc is not None: + Splocaleitemstr.objects.filter( + containername__in=containers, text=table_desc).update(text=camel_to_spaced_title_case(uncapitilize(table.name))) + + # Update the types for the fields in the table + items = Splocalecontaineritem.objects.filter( + container__in=containers) + for item in items: + datamodel_field = table.get_field(item.name) + if not datamodel_field: + continue + + item.type = datamodel_type_to_schematype( + datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type + item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired + + item.save() + + +def fix_item_types(apps): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for migration in CONTAINER_ITEM_MIGRATIONS: + for table_name, fields in migration.items(): + table = datamodel.get_table_strict(table_name) + items = Splocalecontaineritem.objects.filter( + container__name=table_name.lower(), container__schematype=0, name__in=fields) + + for item in items: + datamodel_field = table.get_field(item.name) + if not datamodel_field: + continue + + item.type = datamodel_type_to_schematype( + datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type + item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired + + item.save() + + +def schemaconfig_fixes(apps, schema_editor): + fix_table_captions(apps) + fix_item_types(apps) + + class Migration(migrations.Migration): dependencies = [ ('specify', '0016_collectionobjecttype_catalognumformatname'), ] operations = [ - migrations.RunPython(usc.schemaconfig_fixes, + migrations.RunPython(schemaconfig_fixes, migrations.RunPython.noop, atomic=True) ] diff --git a/specifyweb/specify/migrations/0018_cot_catnum_schema.py b/specifyweb/specify/migrations/0018_cot_catnum_schema.py index 7e5c378be6d..6919fbe89cf 100644 --- a/specifyweb/specify/migrations/0018_cot_catnum_schema.py +++ b/specifyweb/specify/migrations/0018_cot_catnum_schema.py @@ -1,7 +1,63 @@ # Generated by Django 3.2.15 on 2024-11-21 20:08 from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc +from django.db.models import Q + +from specifyweb.specify.datamodel import datamodel +from specifyweb.specify.migration_utils.update_schema_config import camel_to_spaced_title_case + + +def add_cot_catnum_to_schema(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + CollectionObjectType_Table = datamodel.get_table_strict( + 'collectionobjecttype') + catalognumber_format_field = CollectionObjectType_Table.get_field_strict( + 'catalogNumberFormatName') + + for container in Splocalecontainer.objects.filter(name='collectionobjecttype', schematype=0): + schema_item, created = Splocalecontaineritem.objects.get_or_create( + name=catalognumber_format_field.name, type=catalognumber_format_field.type, container=container) + if created: + schema_item.version = 0 + + schema_item.isrequired = catalognumber_format_field.required if schema_item.isrequired is None else schema_item.isrequired + + schema_item.save() + + schema_name = camel_to_spaced_title_case( + catalognumber_format_field.name) + Splocaleitemstr.objects.get_or_create( + language='en', text=schema_name, itemname=schema_item) + Splocaleitemstr.objects.get_or_create( + language='en', text=schema_name, itemdesc=schema_item) + + +def remove_cot_catnum_from_schema(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + CollectionObjectType_Table = datamodel.get_table_strict( + 'collectionobjecttype') + catalognumber_format_field = CollectionObjectType_Table.get_field_strict( + 'catalogNumberFormatName') + + containers = Splocalecontainer.objects.filter( + name='collectionobjecttype', schematype=0) + items = Splocalecontaineritem.objects.filter( + name='catalogNumberFormatName', container__in=containers) + + schema_name = camel_to_spaced_title_case(catalognumber_format_field.name) + filters = Q(language='en', text=schema_name) & ( + Q(itemname__in=items) | Q(itemdesc__in=items)) + locale_strings = Splocaleitemstr.objects.filter(filters) + + locale_strings.delete() + items.delete() + class Migration(migrations.Migration): @@ -10,6 +66,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(usc.add_cot_catnum_to_schema, - usc.remove_cot_catnum_from_schema, atomic=True) + migrations.RunPython(add_cot_catnum_to_schema, + remove_cot_catnum_from_schema, atomic=True) ] diff --git a/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py b/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py index 455dd7ee047..2b84618a827 100644 --- a/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py +++ b/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py @@ -2,7 +2,24 @@ This migration adds Tectonic Unit -> Paleo Context in the Schema Config. """ from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0020_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + + +def add_tectonicunit_to_pc_in_schema_config(apps): + Discipline = apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults( + table, discipline.id, field, apps) + + +def remove_tectonicunit_from_pc_schema_config(apps): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) + class Migration(migrations.Migration): @@ -11,10 +28,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.add_tectonicunit_to_pc_in_schema_config(apps) + add_tectonicunit_to_pc_in_schema_config(apps) def revert_migration(apps, schema_editor): - usc.remove_tectonicunit_from_pc_schema_config(apps) + remove_tectonicunit_from_pc_schema_config(apps) operations = [ migrations.RunPython(apply_migration, revert_migration, atomic=True), diff --git a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py index 26a1a2c4c25..bc5738d69d4 100644 --- a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py +++ b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py @@ -3,7 +3,54 @@ """ from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0021_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS +from specifyweb.specify.models_utils.model_extras import GEOLOGY_DISCIPLINES, PALEO_DISCIPLINES + +def fix_hidden_geo_prop(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + + excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + + filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + + for discipline in filtered_disciplines: + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + discipline_id=discipline.id, + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + items.update(ishidden=True) + +def reverse_fix_hidden_geo_prop(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + + excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + + filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + + for discipline in filtered_disciplines: + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + discipline_id=discipline.id, + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + items.update(ishidden=True) class Migration(migrations.Migration): dependencies = [ @@ -11,5 +58,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(usc.fix_hidden_geo_prop, usc.reverse_fix_hidden_geo_prop, atomic=True) + migrations.RunPython(fix_hidden_geo_prop, reverse_fix_hidden_geo_prop, atomic=True) ] diff --git a/specifyweb/specify/migrations/0022_ensure_default_cots.py b/specifyweb/specify/migrations/0022_ensure_default_cots.py index 9f2219e6645..d687d1a9677 100644 --- a/specifyweb/specify/migrations/0022_ensure_default_cots.py +++ b/specifyweb/specify/migrations/0022_ensure_default_cots.py @@ -1,5 +1,5 @@ from django.db import migrations -from specifyweb.specify.migration_utils.default_cots import create_default_collection_types +from specifyweb.specify.api.utils import create_default_collection_types class Migration(migrations.Migration): diff --git a/specifyweb/specify/migrations/0023_update_schema_config_text.py b/specifyweb/specify/migrations/0023_update_schema_config_text.py index 4ddecb4d688..533b9e119ff 100644 --- a/specifyweb/specify/migrations/0023_update_schema_config_text.py +++ b/specifyweb/specify/migrations/0023_update_schema_config_text.py @@ -3,7 +3,109 @@ """ from django.db import migrations -from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0023_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0023_FIELDS_BIS as SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS + +def update_schema_config_field_desc(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + #i.e: Collection Object + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + #i.e: COType + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = new_desc + localized_items_desc.save() + + localized_items_name.text = new_name + localized_items_name.save() + +def update_hidden_prop(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + item.ishidden = True + item.save() + +def reverse_update_hidden_prop(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + for container in containers: + for field_name in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + item.ishidden = False + item.save() + +def reverse_update_schema_config_field_desc(apps, schema_editor): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + containers = Splocalecontainer.objects.filter( + name=table.lower(), + ) + + for container in containers: + for field_name, new_name, new_desc in fields: + items = Splocalecontaineritem.objects.filter( + container=container, + name=field_name.lower() + ) + + for item in items: + localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + + if localized_items_desc is None or localized_items_name is None: + continue + + localized_items_desc.text = item.name + localized_items_desc.save() + + localized_items_name.text = item.name + localized_items_name.save() class Migration(migrations.Migration): dependencies = [ @@ -15,9 +117,9 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython( - usc.update_schema_config_field_desc, usc.reverse_update_schema_config_field_desc, atomic=True + update_schema_config_field_desc, reverse_update_schema_config_field_desc, atomic=True ), migrations.RunPython( - usc.update_hidden_prop, usc.reverse_update_hidden_prop, atomic=True + update_hidden_prop, reverse_update_hidden_prop, atomic=True ), ] \ No newline at end of file diff --git a/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py b/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py index e7b26de5b9e..35cc56b98f9 100644 --- a/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py +++ b/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py @@ -1,6 +1,22 @@ from django.db import migrations, models -from specifyweb.backend.businessrules.uniqueness_rules import DEFAULT_UNIQUENESS_RULES -from specifyweb.specify.migration_utils import update_schema_config as usc + +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0024_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS + +def update_fields(apps): + Discipline = apps.get_model('specify', 'Discipline') + + # Add uniqueIdentifier -> storage + for discipline in Discipline.objects.all(): + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) + +def revert_update_fields(apps): + # Remove uniqueIdentifier -> storage + for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): + for field in fields: + revert_table_field_schema_config(table, field, apps) class Migration(migrations.Migration): @@ -9,10 +25,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - usc.update_storage_unique_id_fields(apps) + update_fields(apps) def revert_migration(apps, schema_editor): - usc.revert_storage_unique_id_fields(apps) + revert_update_fields(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0027_CO_children.py b/specifyweb/specify/migrations/0027_CO_children.py index f7f5f8c18a7..9d7ac906294 100644 --- a/specifyweb/specify/migrations/0027_CO_children.py +++ b/specifyweb/specify/migrations/0027_CO_children.py @@ -1,158 +1,72 @@ # Generated by Django 3.2.15 on 2025-04-11 15:35 -import re - -from django.core.exceptions import MultipleObjectsReturned +from django.apps import apps as specify_apps from django.db import migrations, models -from django.db.models import Q import django.db.models.deletion +from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults +import specifyweb.specify.models -from specifyweb.specify.models import datamodel -from specifyweb.specify.models_utils.load_datamodel import FieldDoesNotExistError - - -MIGRATION_0027_FIELDS = { - 'CollectionObject': ['parentCO', 'children'], -} - -MIGRATION_0027_UPDATE_FIELDS = { - 'CollectionObject': [ - ('parentCO', 'Parent Collection Object', 'Parent CollectionObject'), - ('children', 'Children', 'Children'), - ] -} - -HIDDEN_FIELDS = [ - "timestampcreated", "timestampmodified", "version", "createdbyagent", "modifiedbyagent" -] - - -def datamodel_type_to_schematype(datamodel_type: str) -> str: - return "".join(map(lambda type_part: type_part.lower().capitalize(), datamodel_type.split('-'))) - - -def camel_to_spaced_title_case(camel_case: str) -> str: - return re.sub(r"(? Date: Wed, 20 May 2026 14:21:39 -0500 Subject: [PATCH 04/17] Revert "Make migration edits while preserving PR review fixes" This reverts commit e8a005f092e4bf937299fdc51ed3567d878eed4f. --- .../migrations/0004_catnum_uniquerule.py | 32 +-- .../businessrules/migrations/0005_cojo.py | 2 +- .../0008_fix_global_default_rules.py | 18 ++ .../migrations/0001_restore_separators.py | 1 + .../migrations/0002_fix_accepted_taxon.py | 10 +- .../migrations/0003_coordinate_fields_fix.py | 25 +- ...add_dataset_create_recordset_permission.py | 37 ++- .../0007_add_stats_edit_permission.py | 26 +- .../migrations/0008_attachment_import_role.py | 253 +++++------------- .../commands/run_key_migration_functions.py | 3 +- specifyweb/specify/migrations/0002_geo.py | 107 ++------ .../migrations/0003_cotype_picklist.py | 71 ++--- .../migrations/0004_stratigraphy_age.py | 61 +---- .../migrations/0007_schema_config_update.py | 152 ++--------- .../migrations/0008_ageCitations_fix.py | 48 ++-- .../specify/migrations/0009_tectonic_ranks.py | 133 +-------- .../0012_add_cojo_to_schema_config.py | 23 +- .../0013_collectionobjectgroup_parentcog.py | 33 +-- .../migrations/0015_add_version_to_ages.py | 19 +- .../migrations/0017_schemaconfig_fixes.py | 78 +----- .../migrations/0018_cot_catnum_schema.py | 62 +---- ...add_tectonicunit_to_pc_in_schema_config.py | 23 +- .../0021_update_hidden_geo_tables.py | 51 +--- .../migrations/0022_ensure_default_cots.py | 2 +- .../0023_update_schema_config_text.py | 108 +------- .../0024_add_uniqueIdentifier_storage.py | 24 +- .../specify/migrations/0027_CO_children.py | 178 ++++++++---- .../0029_remove_collectionobject_parentco.py | 130 +-------- .../0031_add_default_for_selectseries.py | 11 +- .../migrations/0032_add_quantities_gift.py | 83 +----- .../migrations/0033_update_paleo_desc.py | 22 +- .../migrations/0034_accession_date_fields.py | 92 +------ .../migrations/0035_version_required.py | 11 + 33 files changed, 437 insertions(+), 1492 deletions(-) create mode 100644 specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py diff --git a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py index fd1e3fe66ec..314a8ad9725 100644 --- a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py +++ b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py @@ -1,36 +1,6 @@ -from typing import Tuple - from django.db import migrations -from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule - - -def catnum_rule_editable(apps, schema_editor): - UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') - - candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__isDatabaseConstraint=True, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) - - candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) - - candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) - candidate_rules.update(isDatabaseConstraint=False) - -def catnum_rule_uneditable(apps, schema_editor): - Discipline = apps.get_model('specify', 'Discipline') - UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') - - for discipline in Discipline.objects.all(): - candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__discipline=discipline.id, uniquenessrule__isDatabaseConstraint=False, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) - - candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) - - candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) - if len(candidate_rules) == 0: - create_uniqueness_rule('Collectionobject', discipline=discipline, is_database_constraint=True, fields=['catalogNumber'], scopes=['collection'], registry=apps) - else: - candidate_rules.update(isDatabaseConstraint=True) +from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable, catnum_rule_uneditable class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/backend/businessrules/migrations/0005_cojo.py b/specifyweb/backend/businessrules/migrations/0005_cojo.py index e9440465fd8..c006d850af6 100644 --- a/specifyweb/backend/businessrules/migrations/0005_cojo.py +++ b/specifyweb/backend/businessrules/migrations/0005_cojo.py @@ -6,7 +6,7 @@ Applies the COJO uniqueness rule to the database. """ def apply_migration(apps, schema_editor): - cojo_rules = DEFAULT_UNIQUENESS_RULES["CollectionObjectGroupJoin"] + cojo_rules = DEFAULT_UNIQUENESS_RULES["Collectionobjectgroupjoin"] Discipline = apps.get_model('specify', 'Discipline') for rule in cojo_rules: diff --git a/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py b/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py new file mode 100644 index 00000000000..49208174a35 --- /dev/null +++ b/specifyweb/backend/businessrules/migrations/0008_fix_global_default_rules.py @@ -0,0 +1,18 @@ +import logging +from django.db import migrations +from specifyweb.backend.businessrules.uniqueness_rules import fix_global_default_rules + +logger = logging.getLogger(__name__) + +def apply_migration(apps, schema_editor): + fix_global_default_rules(apps) + +class Migration(migrations.Migration): + + dependencies = [ + ('businessrules', '0007_more_uniqueness_rules'), + ] + + operations = [ + migrations.RunPython(apply_migration, migrations.RunPython.noop, atomic=True) + ] diff --git a/specifyweb/backend/patches/migrations/0001_restore_separators.py b/specifyweb/backend/patches/migrations/0001_restore_separators.py index 363ee2d5055..cdc34a59a06 100644 --- a/specifyweb/backend/patches/migrations/0001_restore_separators.py +++ b/specifyweb/backend/patches/migrations/0001_restore_separators.py @@ -7,6 +7,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ('specify', '0001_initial'), ] operations = [ diff --git a/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py b/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py index ec0ceae29f9..ce9e4213d4d 100644 --- a/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py +++ b/specifyweb/backend/patches/migrations/0002_fix_accepted_taxon.py @@ -2,6 +2,8 @@ from django.db import migrations +from specifyweb.backend.patches.migration_utils import update_is_accepted + class Migration(migrations.Migration): @@ -13,10 +15,6 @@ class Migration(migrations.Migration): # Fixes an issue due to a bug in the WorkBench prior to 7.9.0 that did not # set accepted nodes to IsAccepted = 1 when they were not synonyms. # https://github.com/specify/specify7/issues/5131 - migrations.RunSQL( - 'UPDATE taxon t1 SET IsAccepted = TRUE WHERE t1.IsAccepted = 0 AND t1.AcceptedId IS NULL', - # This should not need to be reversed, but this allows for rollback without reversing the SQL - reverse_sql='' - ) + migrations.RunPython(update_is_accepted, + migrations.RunPython.noop, atomic=True), ] - diff --git a/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py b/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py index 2813b08ff16..eaccb47f93b 100644 --- a/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py +++ b/specifyweb/backend/patches/migrations/0003_coordinate_fields_fix.py @@ -2,6 +2,8 @@ from django.db import migrations +from specifyweb.backend.patches.migration_utils import update_coordinates + class Migration(migrations.Migration): @@ -14,25 +16,6 @@ class Migration(migrations.Migration): # the accompanying text fields are not populated. In cases where the text fields are not populated, # it appears the records do not have coordinates. This script will populate the text fields with the # decimal values where the text fields are empty but the decimal values are not. - migrations.RunSQL( - """ - UPDATE locality - SET Lat1text = Latitude1 - WHERE Lat1text IS NULL AND Latitude1 IS NOT NULL; - - UPDATE locality - SET Long1text = Longitude1 - WHERE Long1text IS NULL AND Longitude1 IS NOT NULL; - - UPDATE locality - SET Lat2text = Latitude2 - WHERE Lat2text IS NULL AND Latitude2 IS NOT NULL; - - UPDATE locality - SET Long2text = Longitude2 - WHERE Long2text IS NULL AND Longitude2 IS NOT NULL; - """, - # This SQL statment should not be reversed, but this allows for rollback - reverse_sql='' - ) + migrations.RunPython(update_coordinates, + migrations.RunPython.noop, atomic=True), ] diff --git a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py index 702defd3036..936fc821511 100644 --- a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py +++ b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py @@ -2,32 +2,55 @@ from django.db import migrations -def add_permission(apps, schema_editor): +from specifyweb.backend.workbench.upload.auditlog import auditlog + + +def add_permission(apps, schema_editor=None): UserPolicy = apps.get_model('permissions', 'UserPolicy') LibraryRolePolicy = apps.get_model('permissions', 'LibraryRolePolicy') RolePolicy = apps.get_model('permissions', 'RolePolicy') for p in UserPolicy.objects.filter(resource='/workbench/dataset', action='upload'): - UserPolicy.objects.create( + if not UserPolicy.objects.filter( collection=p.collection, specifyuser=p.specifyuser, resource=p.resource, action='create_recordset', - ) + ).exists(): + user_policy = UserPolicy.objects.create( + collection=p.collection, + specifyuser=p.specifyuser, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(user_policy) for p in RolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - RolePolicy.objects.create( + if not RolePolicy.objects.filter( role=p.role, resource=p.resource, action='create_recordset', - ) + ).exists(): + role_policy = RolePolicy.objects.create( + role=p.role, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(role_policy, None) for p in LibraryRolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - LibraryRolePolicy.objects.create( + if not LibraryRolePolicy.objects.filter( role=p.role, resource=p.resource, action='create_recordset', - ) + ).exists(): + library_role_policy = LibraryRolePolicy.objects.create( + role=p.role, + resource=p.resource, + action='create_recordset', + ) + auditlog.insert(library_role_policy, None) + class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py index 6d262b3fc64..a850816b560 100644 --- a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py +++ b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py @@ -1,19 +1,19 @@ from django.db import migrations -from specifyweb.specify.models import Collection # type: ignore -from ..models import Role +from specifyweb.backend.workbench.upload.auditlog import auditlog -def add_stats_edit_permission(apps, schema_editor): - for collection_id in Collection.objects.values_list('id', flat=True): - try: - all_full_access_roles = Role.objects.filter(collection_id=collection_id, name="Full Access - Legacy") - for full_access_role in all_full_access_roles: - full_access_role.policies.create(resource="/preferences" - "/statistics", - action="edit") - except: - print("Failed to assign stats edit permission in collection: ", - collection_id) +def add_stats_edit_permission(apps, schema_editor=None): + Role = apps.get_model('permissions', 'Role') + + all_full_access_roles = Role.objects.filter(name="Full Access - Legacy") + + for full_access_role in all_full_access_roles: + new_policy, created = full_access_role.policies.get_or_create( + resource="/preferences/statistics", + action="edit", + ) + if created: + auditlog.insert(new_policy, None) class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py b/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py index 60ff925f311..a1cabb22429 100644 --- a/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py +++ b/specifyweb/backend/permissions/migrations/0008_attachment_import_role.py @@ -1,4 +1,5 @@ from django.db import migrations +from specifyweb.backend.workbench.upload.auditlog import auditlog def add_attachment_import_role(apps, schema_editor): LibraryRole = apps.get_model('permissions', 'LibraryRole') @@ -6,187 +7,79 @@ def add_attachment_import_role(apps, schema_editor): name="Bulk Attachment Import", description="Gives full access to the Bulk Attachment Import. Allows creating new attachments for any attachment table" ) - # Attachment dataset permissions - role.policies.create(resource='/attachment_import/dataset', action='create') - role.policies.create(resource='/attachment_import/dataset', action='update') - role.policies.create(resource='/attachment_import/dataset', action='delete') - role.policies.create(resource='/attachment_import/dataset', action='upload') - role.policies.create(resource='/attachment_import/dataset', action='rollback') - - # Attachment permissions - role.policies.create(resource='/table/attachment', action='create') - role.policies.create(resource='/table/attachment', action='read') - role.policies.create(resource='/table/attachment', action='delete') - - # Table Specific permissions (Auto-generated) - role.policies.create(resource='/table/accession', action='read') - role.policies.create(resource='/table/accession', action='update') - role.policies.create(resource='/table/accessionattachment', action='read') - role.policies.create(resource='/table/accessionattachment', action='create') - role.policies.create(resource='/table/accessionattachment', action='delete') - - role.policies.create(resource='/table/agent', action='read') - role.policies.create(resource='/table/agent', action='update') - role.policies.create(resource='/table/agentattachment', action='read') - role.policies.create(resource='/table/agentattachment', action='create') - role.policies.create(resource='/table/agentattachment', action='delete') - - role.policies.create(resource='/table/borrow', action='read') - role.policies.create(resource='/table/borrow', action='update') - role.policies.create(resource='/table/borrowattachment', action='read') - role.policies.create(resource='/table/borrowattachment', action='create') - role.policies.create(resource='/table/borrowattachment', action='delete') - - role.policies.create(resource='/table/collectingevent', action='read') - role.policies.create(resource='/table/collectingevent', action='update') - role.policies.create(resource='/table/collectingeventattachment', action='read') - role.policies.create(resource='/table/collectingeventattachment', action='create') - role.policies.create(resource='/table/collectingeventattachment', action='delete') - - role.policies.create(resource='/table/collectingtrip', action='read') - role.policies.create(resource='/table/collectingtrip', action='update') - role.policies.create(resource='/table/collectingtripattachment', action='read') - role.policies.create(resource='/table/collectingtripattachment', action='create') - role.policies.create(resource='/table/collectingtripattachment', action='delete') - - role.policies.create(resource='/table/collectionobject', action='read') - role.policies.create(resource='/table/collectionobject', action='update') - role.policies.create(resource='/table/collectionobjectattachment', action='read') - role.policies.create(resource='/table/collectionobjectattachment', action='create') - role.policies.create(resource='/table/collectionobjectattachment', action='delete') - - role.policies.create(resource='/table/conservdescription', action='read') - role.policies.create(resource='/table/conservdescription', action='update') - role.policies.create(resource='/table/conservdescriptionattachment', action='read') - role.policies.create(resource='/table/conservdescriptionattachment', action='create') - role.policies.create(resource='/table/conservdescriptionattachment', action='delete') - - role.policies.create(resource='/table/conservevent', action='read') - role.policies.create(resource='/table/conservevent', action='update') - role.policies.create(resource='/table/conserveventattachment', action='read') - role.policies.create(resource='/table/conserveventattachment', action='create') - role.policies.create(resource='/table/conserveventattachment', action='delete') - - role.policies.create(resource='/table/dnasequence', action='read') - role.policies.create(resource='/table/dnasequence', action='update') - role.policies.create(resource='/table/dnasequenceattachment', action='read') - role.policies.create(resource='/table/dnasequenceattachment', action='create') - role.policies.create(resource='/table/dnasequenceattachment', action='delete') - - role.policies.create(resource='/table/dnasequencingrun', action='read') - role.policies.create(resource='/table/dnasequencingrun', action='update') - role.policies.create(resource='/table/dnasequencingrunattachment', action='read') - role.policies.create(resource='/table/dnasequencingrunattachment', action='create') - role.policies.create(resource='/table/dnasequencingrunattachment', action='delete') - - role.policies.create(resource='/table/deaccession', action='read') - role.policies.create(resource='/table/deaccession', action='update') - role.policies.create(resource='/table/deaccessionattachment', action='read') - role.policies.create(resource='/table/deaccessionattachment', action='create') - role.policies.create(resource='/table/deaccessionattachment', action='delete') - - role.policies.create(resource='/table/disposal', action='read') - role.policies.create(resource='/table/disposal', action='update') - role.policies.create(resource='/table/disposalattachment', action='read') - role.policies.create(resource='/table/disposalattachment', action='create') - role.policies.create(resource='/table/disposalattachment', action='delete') - - role.policies.create(resource='/table/exchangein', action='read') - role.policies.create(resource='/table/exchangein', action='update') - role.policies.create(resource='/table/exchangeinattachment', action='read') - role.policies.create(resource='/table/exchangeinattachment', action='create') - role.policies.create(resource='/table/exchangeinattachment', action='delete') - - role.policies.create(resource='/table/exchangeout', action='read') - role.policies.create(resource='/table/exchangeout', action='update') - role.policies.create(resource='/table/exchangeoutattachment', action='read') - role.policies.create(resource='/table/exchangeoutattachment', action='create') - role.policies.create(resource='/table/exchangeoutattachment', action='delete') - - role.policies.create(resource='/table/fieldnotebook', action='read') - role.policies.create(resource='/table/fieldnotebook', action='update') - role.policies.create(resource='/table/fieldnotebookattachment', action='read') - role.policies.create(resource='/table/fieldnotebookattachment', action='create') - role.policies.create(resource='/table/fieldnotebookattachment', action='delete') - - role.policies.create(resource='/table/fieldnotebookpage', action='read') - role.policies.create(resource='/table/fieldnotebookpage', action='update') - role.policies.create(resource='/table/fieldnotebookpageattachment', action='read') - role.policies.create(resource='/table/fieldnotebookpageattachment', action='create') - role.policies.create(resource='/table/fieldnotebookpageattachment', action='delete') - - role.policies.create(resource='/table/fieldnotebookpageset', action='read') - role.policies.create(resource='/table/fieldnotebookpageset', action='update') - role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='read') - role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='create') - role.policies.create(resource='/table/fieldnotebookpagesetattachment', action='delete') - - role.policies.create(resource='/table/gift', action='read') - role.policies.create(resource='/table/gift', action='update') - role.policies.create(resource='/table/giftattachment', action='read') - role.policies.create(resource='/table/giftattachment', action='create') - role.policies.create(resource='/table/giftattachment', action='delete') - - role.policies.create(resource='/table/loan', action='read') - role.policies.create(resource='/table/loan', action='update') - role.policies.create(resource='/table/loanattachment', action='read') - role.policies.create(resource='/table/loanattachment', action='create') - role.policies.create(resource='/table/loanattachment', action='delete') - - role.policies.create(resource='/table/locality', action='read') - role.policies.create(resource='/table/locality', action='update') - role.policies.create(resource='/table/localityattachment', action='read') - role.policies.create(resource='/table/localityattachment', action='create') - role.policies.create(resource='/table/localityattachment', action='delete') - - role.policies.create(resource='/table/morphbankview', action='read') - role.policies.create(resource='/table/morphbankview', action='update') - role.policies.create(resource='/table/attachmentimageattribute', action='read') - role.policies.create(resource='/table/attachmentimageattribute', action='create') - role.policies.create(resource='/table/attachmentimageattribute', action='delete') - - role.policies.create(resource='/table/permit', action='read') - role.policies.create(resource='/table/permit', action='update') - role.policies.create(resource='/table/permitattachment', action='read') - role.policies.create(resource='/table/permitattachment', action='create') - role.policies.create(resource='/table/permitattachment', action='delete') - - role.policies.create(resource='/table/preparation', action='read') - role.policies.create(resource='/table/preparation', action='update') - role.policies.create(resource='/table/preparationattachment', action='read') - role.policies.create(resource='/table/preparationattachment', action='create') - role.policies.create(resource='/table/preparationattachment', action='delete') - - role.policies.create(resource='/table/referencework', action='read') - role.policies.create(resource='/table/referencework', action='update') - role.policies.create(resource='/table/referenceworkattachment', action='read') - role.policies.create(resource='/table/referenceworkattachment', action='create') - role.policies.create(resource='/table/referenceworkattachment', action='delete') - - role.policies.create(resource='/table/repositoryagreement', action='read') - role.policies.create(resource='/table/repositoryagreement', action='update') - role.policies.create(resource='/table/repositoryagreementattachment', action='read') - role.policies.create(resource='/table/repositoryagreementattachment', action='create') - role.policies.create(resource='/table/repositoryagreementattachment', action='delete') - - role.policies.create(resource='/table/storage', action='read') - role.policies.create(resource='/table/storage', action='update') - role.policies.create(resource='/table/storageattachment', action='read') - role.policies.create(resource='/table/storageattachment', action='create') - role.policies.create(resource='/table/storageattachment', action='delete') - - role.policies.create(resource='/table/taxon', action='read') - role.policies.create(resource='/table/taxon', action='update') - role.policies.create(resource='/table/taxonattachment', action='read') - role.policies.create(resource='/table/taxonattachment', action='create') - role.policies.create(resource='/table/taxonattachment', action='delete') - - role.policies.create(resource='/table/treatmentevent', action='read') - role.policies.create(resource='/table/treatmentevent', action='update') - role.policies.create(resource='/table/treatmenteventattachment', action='read') - role.policies.create(resource='/table/treatmenteventattachment', action='create') - role.policies.create(resource='/table/treatmenteventattachment', action='delete') + auditlog.insert(role) + + # Define policy sets grouped by resource + policy_definitions = [ + # Attachment dataset permissions + {'resource': '/attachment_import/dataset', 'actions': ['create', 'update', 'delete', 'upload', 'rollback']}, + # Attachment permissions + {'resource': '/table/attachment', 'actions': ['create', 'read', 'delete']}, + # Table Specific permissions + {'resource': '/table/accession', 'actions': ['read', 'update']}, + {'resource': '/table/accessionattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/agent', 'actions': ['read', 'update']}, + {'resource': '/table/agentattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/borrow', 'actions': ['read', 'update']}, + {'resource': '/table/borrowattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/collectingevent', 'actions': ['read', 'update']}, + {'resource': '/table/collectingeventattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/collectingtrip', 'actions': ['read', 'update']}, + {'resource': '/table/collectingtripattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/collectionobject', 'actions': ['read', 'update']}, + {'resource': '/table/collectionobjectattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/conservdescription', 'actions': ['read', 'update']}, + {'resource': '/table/conservdescriptionattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/conservevent', 'actions': ['read', 'update']}, + {'resource': '/table/conserveventattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/dnasequence', 'actions': ['read', 'update']}, + {'resource': '/table/dnasequenceattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/dnasequencingrun', 'actions': ['read', 'update']}, + {'resource': '/table/dnasequencingrunattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/deaccession', 'actions': ['read', 'update']}, + {'resource': '/table/deaccessionattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/disposal', 'actions': ['read', 'update']}, + {'resource': '/table/disposalattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/exchangein', 'actions': ['read', 'update']}, + {'resource': '/table/exchangeinattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/exchangeout', 'actions': ['read', 'update']}, + {'resource': '/table/exchangeoutattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/fieldnotebook', 'actions': ['read', 'update']}, + {'resource': '/table/fieldnotebookattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/fieldnotebookpage', 'actions': ['read', 'update']}, + {'resource': '/table/fieldnotebookpageattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/fieldnotebookpageset', 'actions': ['read', 'update']}, + {'resource': '/table/fieldnotebookpagesetattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/gift', 'actions': ['read', 'update']}, + {'resource': '/table/giftattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/loan', 'actions': ['read', 'update']}, + {'resource': '/table/loanattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/locality', 'actions': ['read', 'update']}, + {'resource': '/table/localityattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/morphbankview', 'actions': ['read', 'update']}, + {'resource': '/table/attachmentimageattribute', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/permit', 'actions': ['read', 'update']}, + {'resource': '/table/permitattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/preparation', 'actions': ['read', 'update']}, + {'resource': '/table/preparationattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/referencework', 'actions': ['read', 'update']}, + {'resource': '/table/referenceworkattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/repositoryagreement', 'actions': ['read', 'update']}, + {'resource': '/table/repositoryagreementattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/storage', 'actions': ['read', 'update']}, + {'resource': '/table/storageattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/taxon', 'actions': ['read', 'update']}, + {'resource': '/table/taxonattachment', 'actions': ['read', 'create', 'delete']}, + {'resource': '/table/treatmentevent', 'actions': ['read', 'update']}, + {'resource': '/table/treatmenteventattachment', 'actions': ['read', 'create', 'delete']}, + ] + # Create each policy and log its creation + for policy in policy_definitions: + for action in policy['actions']: + obj, created = role.policies.get_or_create(resource=policy['resource'], action=action) + if created: + auditlog.insert(obj) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index 69d02382e7a..ae767208d27 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -74,8 +74,7 @@ def fix_schema_config(stdout: WriteToStdOut | None = None): usc.remove_collectionobject_parentco, # specify 0029 usc.add_quantities_gift, # specify 0032 usc.update_paleo_desc, # specify 0033 - usc.update_accession_date_fields, # specify 0034 - usc.update_version_required, # specify 0035 + usc.update_accession_date_fields # specify 0034 ] log_and_run(funcs, stdout) diff --git a/specifyweb/specify/migrations/0002_geo.py b/specifyweb/specify/migrations/0002_geo.py index 27940aec820..e84e59d5c2e 100644 --- a/specifyweb/specify/migrations/0002_geo.py +++ b/specifyweb/specify/migrations/0002_geo.py @@ -3,17 +3,19 @@ import logging from django.db import migrations, models -from django.db.models import F +# from django.db.models import F import django.utils.timezone from specifyweb.specify.models import ( protect_with_blockers ) -from specifyweb.specify.migration_utils.update_schema_config import ( - update_table_schema_config_with_defaults, - revert_table_schema_config, -) from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0002_TABLES as SCHEMA_CONFIG_TABLES -from specifyweb.specify.api.utils import create_default_collection_types +from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.default_cots import ( + create_cogtype_type_picklist, + create_default_collection_types, + create_default_discipline_for_tree_defs, + set_discipline_for_taxon_treedefs, +) logger = logging.getLogger(__name__) @@ -28,15 +30,6 @@ # 8. Add discipline relationship to TreeDef tables # 9. Add schema config for new sp7 tables -DEFAULT_COG_TYPES = [ - 'Discrete', - 'Consolidated', - 'Drill Core', -] - -def handle_default_collection_types(apps): - create_default_collection_types(apps) - def revert_default_collection_types(apps): # Reverse handeled by table deletion. pass @@ -45,67 +38,11 @@ def revert_default_cog_types(apps): # Reverse handeled by table deletion pass -def create_default_discipline_for_tree_defs(apps): - Discipline = apps.get_model('specify', 'Discipline') - Institution = apps.get_model('specify', 'Institution') - - for discipline in Discipline.objects.all(): - geography_tree_def = discipline.geographytreedef - geography_tree_def.discipline = discipline - geography_tree_def.save() - - geologic_time_period_tree_def = discipline.geologictimeperiodtreedef - geologic_time_period_tree_def.discipline = discipline - geologic_time_period_tree_def.save() - - lithostrat_tree_def = discipline.lithostrattreedef - lithostrat_tree_def.discipline = discipline - lithostrat_tree_def.save() - - taxon_tree_def = discipline.taxontreedef - taxon_tree_def.discipline = discipline - taxon_tree_def.save() - - for institution in Institution.objects.all(): - storage_tree_def = institution.storagetreedef - storage_tree_def.institution = institution - storage_tree_def.save() - def revert_default_discipline_for_tree_defs(apps): # Reverse handeled by table deletion pass -def create_table_schema_config_with_defaults(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, desc in SCHEMA_CONFIG_TABLES: - update_table_schema_config_with_defaults(table, discipline.id, desc, apps) - -def revert_table_schema_config_with_defaults(apps): - for table, _ in SCHEMA_CONFIG_TABLES: - revert_table_schema_config(table, apps) - -def create_default_collection_object_types(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - Picklistitem = apps.get_model('specify', 'Picklistitem') - - for collection in Collection.objects.all(): - cog_type_picklist = Picklist.objects.create( - name='Default Collection Object Group Types', - issystem=False, - type=0, - readonly=False, - collection=collection - ) - for cog_type in DEFAULT_COG_TYPES: - Picklistitem.objects.create( - title=cog_type, - value=cog_type, - picklist=cog_type_picklist - ) - -def revert_default_collection_object_types(apps): +def revert_cogtype_type_picklist(apps): Collection = apps.get_model('specify', 'Collection') Picklist = apps.get_model('specify', 'Picklist') Picklistitem = apps.get_model('specify', 'Picklistitem') @@ -113,6 +50,7 @@ def revert_default_collection_object_types(apps): for collection in Collection.objects.all(): cog_type_picklist_qs = Picklist.objects.filter( name='Default Collection Object Group Types', + type=0, collection=collection ) if cog_type_picklist_qs.exists(): @@ -120,18 +58,9 @@ def revert_default_collection_object_types(apps): Picklistitem.objects.filter(picklist=cog_type_picklist).delete() cog_type_picklist.delete() -def set_discipline_for_taxon_treedefs(apps): - Collectionobjecttype = apps.get_model('specify', 'Collectionobjecttype') - Taxontreedef = apps.get_model('specify', 'Taxontreedef') - - collection_object_types = Collectionobjecttype.objects.filter( - taxontreedef__discipline__isnull=True - ).annotate( - discipline=F('collection__discipline') - ) - - for cot in collection_object_types: - Taxontreedef.objects.filter(id=cot.taxontreedef_id).update(discipline=cot.discipline) +def revert_geo_table_schema_config_with_defaults(apps): + for table, _ in SCHEMA_CONFIG_TABLES: + usc.revert_table_schema_config(table, apps) class Migration(migrations.Migration): @@ -142,15 +71,15 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - handle_default_collection_types(apps) + create_default_collection_types(apps) create_default_discipline_for_tree_defs(apps) - create_table_schema_config_with_defaults(apps) - create_default_collection_object_types(apps) + usc.create_geo_table_schema_config_with_defaults(apps) + create_cogtype_type_picklist(apps) set_discipline_for_taxon_treedefs(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_default_collection_object_types(apps) - revert_table_schema_config_with_defaults(apps) + revert_cogtype_type_picklist(apps) + revert_geo_table_schema_config_with_defaults(apps) revert_default_discipline_for_tree_defs(apps) revert_default_collection_types(apps) diff --git a/specifyweb/specify/migrations/0003_cotype_picklist.py b/specifyweb/specify/migrations/0003_cotype_picklist.py index f3da52afc9e..9d921b074d0 100644 --- a/specifyweb/specify/migrations/0003_cotype_picklist.py +++ b/specifyweb/specify/migrations/0003_cotype_picklist.py @@ -1,65 +1,28 @@ from django.db import migrations - -PICKLIST_NAME = 'CollectionObjectType' -FIELD_NAME = 'collectionObjectType' -COTYPE_TEXT = 'Collection Object Type' - -def create_cotype_picklist(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - # Create a cotype picklist for each collection - for collection in Collection.objects.all(): - Picklist.objects.get_or_create( - name=PICKLIST_NAME, - issystem=True, - readonly=True, - sizelimit=-1, - sorttype=1, - type=1, - tablename='collectionobjecttype', - collection=collection, - formatter=PICKLIST_NAME - ) +from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.default_cots import create_cotype_picklist, COTYPE_PICKLIST_NAME def revert_cotype_picklist(apps): Picklist = apps.get_model('specify', 'Picklist') - Picklist.objects.filter(name=PICKLIST_NAME).delete() - - -def create_cotype_splocalecontaineritem(apps): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - # Create a Splocalecontaineritem record for each CollectionObject Splocalecontainer - # NOTE: Each discipline has its own CollectionObject Splocalecontainer - for container in Splocalecontainer.objects.filter(name='collectionobject', schematype=0): - container_item, _ = Splocalecontaineritem.objects.get_or_create( - name=FIELD_NAME, - picklistname=PICKLIST_NAME, - type='ManyToOne', - container=container, - isrequired=True - ) - Splocaleitemstr.objects.get_or_create( - language='en', - text=COTYPE_TEXT, - itemname=container_item - ) - Splocaleitemstr.objects.get_or_create( - language='en', - text=COTYPE_TEXT, - itemdesc=container_item - ) + Picklist.objects.filter(name=COTYPE_PICKLIST_NAME).delete() def revert_cotype_splocalecontaineritem(apps): Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - Splocaleitemstr.objects.filter(text=COTYPE_TEXT, itemdesc__container__name='collectionobject', itemdesc__container__schematype=0).delete() - Splocaleitemstr.objects.filter(text=COTYPE_TEXT, itemname__container__name='collectionobject', itemname__container__schematype=0).delete() - Splocalecontaineritem.objects.filter(name=FIELD_NAME, container__name='collectionobject', container__schematype=0).delete() - + Splocaleitemstr.objects.filter( + text=usc.COT_TEXT, + itemdesc__container__name="collectionobject", + itemdesc__container__schematype=0, + ).delete() + Splocaleitemstr.objects.filter( + text=usc.COT_TEXT, + itemname__container__name="collectionobject", + itemname__container__schematype=0, + ).delete() + Splocalecontaineritem.objects.filter( + name=usc.COT_FIELD_NAME, container__name="collectionobject", container__schematype=0 + ).delete() class Migration(migrations.Migration): dependencies = [ @@ -68,7 +31,7 @@ class Migration(migrations.Migration): def apply_migration(apps, schema_editor): create_cotype_picklist(apps) - create_cotype_splocalecontaineritem(apps) + usc.create_cotype_splocalecontaineritem(apps) def revert_migration(apps, schema_editor): revert_cotype_picklist(apps) diff --git a/specifyweb/specify/migrations/0004_stratigraphy_age.py b/specifyweb/specify/migrations/0004_stratigraphy_age.py index 18493d7da13..eb6636fff99 100644 --- a/specifyweb/specify/migrations/0004_stratigraphy_age.py +++ b/specifyweb/specify/migrations/0004_stratigraphy_age.py @@ -1,44 +1,10 @@ # Generated by Django 3.2.15 on 2024-08-02 06:04 -from django.apps import apps as specify_apps from django.db import migrations, models import django.db.models.deletion import django.utils.timezone from specifyweb.specify.models import protect_with_blockers -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, revert_table_schema_config, update_table_field_schema_config_with_defaults, update_table_schema_config_with_defaults - -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0004_TABLES as SCHEMA_CONFIG_TABLES, MIGRATION_0004_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -PICKLIST_NAME = 'AgeType' -DEFAULT_AGE_TYPES = [ - 'Sedimentation', - 'Metamorphism', - 'Erosion', - 'Diagenetic', -] - -def create_agetype_picklist(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - PicklistItem = apps.get_model('specify', 'Picklistitem') - # Create a AgeType picklist for each collection - for collection in Collection.objects.all(): - age_type_picklist, _ = Picklist.objects.get_or_create( - name=PICKLIST_NAME, - issystem=False, - readonly=False, - sizelimit=-1, - sorttype=1, - type=0, - collection=collection, - formatter=PICKLIST_NAME - ) - for age_type in DEFAULT_AGE_TYPES: - PicklistItem.objects.create( - title=age_type, - value=age_type, - picklist=age_type_picklist - ) +from specifyweb.specify.migration_utils import update_schema_config as usc def revert_agetype_picklist(apps): Collection = apps.get_model('specify', 'Collection') @@ -46,29 +12,12 @@ def revert_agetype_picklist(apps): PicklistItem = apps.get_model('specify', 'Picklistitem') for collection in Collection.objects.all(): - age_type_pick_lists = Picklist.objects.filter(name=PICKLIST_NAME, collection=collection) + age_type_pick_lists = Picklist.objects.filter(name=usc.AGETYPE_PICKLIST_NAME, collection=collection) for age_type_pick_list in age_type_pick_lists: PicklistItem.objects.filter(picklist=age_type_pick_list).delete() age_type_pick_list.delete() -def create_table_schema_config_with_defaults(apps): - Discipline = specify_apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, desc in SCHEMA_CONFIG_TABLES: - update_table_schema_config_with_defaults(table, discipline.id, desc, apps) - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def revert_table_schema_config_with_defaults(apps): - for table, _ in SCHEMA_CONFIG_TABLES: - revert_table_schema_config(table, apps) - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - class Migration(migrations.Migration): dependencies = [ @@ -76,11 +25,11 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - create_table_schema_config_with_defaults(apps) - create_agetype_picklist(apps) + usc.create_strat_table_schema_config_with_defaults(apps) + usc.create_agetype_picklist(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_table_schema_config_with_defaults(apps) + usc.revert_strat_table_schema_config_with_defaults(apps) revert_agetype_picklist(apps) operations = [ diff --git a/specifyweb/specify/migrations/0007_schema_config_update.py b/specifyweb/specify/migrations/0007_schema_config_update.py index 408b232679b..53b6a01f8ec 100644 --- a/specifyweb/specify/migrations/0007_schema_config_update.py +++ b/specifyweb/specify/migrations/0007_schema_config_update.py @@ -16,122 +16,7 @@ from django.db import migrations, models import django.db.models.deletion -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0007_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -PICKLIST_NAME = 'COGTypes' -COGTYPE_FIELD_NAME = 'cogType' -SYSTEM_COGTYPE_PICKLIST_NAME = "SystemCOGTypes" - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - # Revert COG -> children before adding to avoid duplicates - revert_table_field_schema_config('CollectionObjectGroup', 'children', apps) - # Add StorageTreeDef -> institution and COG -> children - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - - - # Remove COG -> cojo - revert_table_field_schema_config('CollectionObjectGroup', 'cojo', apps) - - # Remove duplicate CollectionObject -> collectionObjectType - container_items = Splocalecontaineritem.objects.filter(name='collectionObjectType', picklistname=None, container__name='CollectionObject') - for container_item in container_items: - Splocaleitemstr.objects.filter(itemname=container_item).delete() - Splocaleitemstr.objects.filter(itemdesc=container_item).delete() - container_items.delete() - -# NOTE: The reverse function will not re-add the duplicate CO -> coType or COG -> cojo as its unnecessary -def revert_update_fields(apps): - # Remove StorageTreeDef -> institution and COG -> children - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - -def create_cogtype_picklist(apps): - Collection = apps.get_model('specify', 'Collection') - Picklist = apps.get_model('specify', 'Picklist') - - # Create a cogtype picklist for each collection - for collection in Collection.objects.all(): - Picklist.objects.get_or_create( - name=PICKLIST_NAME, - issystem=True, - readonly=True, - sizelimit=-1, - sorttype=1, - type=1, - tablename='collectionobjectgrouptype', - collection=collection, - formatter='CollectionObjectGroupType' - ) - -def revert_cogtype_picklist(apps): - Picklist = apps.get_model('specify', 'Picklist') - - Picklist.objects.filter(name=PICKLIST_NAME).delete() - -# Updates COG -> cogtype to use the type 1 picklist created above -def update_cogtype_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgroup', container__schematype=0, name=COGTYPE_FIELD_NAME).update( - picklistname=PICKLIST_NAME, - type='ManyToOne', - isrequired=True - ) - -def revert_cogtype_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgroup', container__schematype=0, name=COGTYPE_FIELD_NAME).update( - picklistname=None, - type=None, - isrequired=None - ) - -def update_systemcogtypes_picklist(apps): - Picklist = apps.get_model('specify', 'Picklist') - - Picklist.objects.filter(name='Default Collection Object Group Types').update( - name=SYSTEM_COGTYPE_PICKLIST_NAME, - type=0, - issystem=True, - readonly=True, - sizelimit=3, - tablename=None - ) - -def revert_systemcogtypes_picklist(apps): - Picklist = apps.get_model('specify', 'Picklist') - - # revert only changes the name and not the other attributes as those were incorrect - Picklist.objects.filter(name=SYSTEM_COGTYPE_PICKLIST_NAME).update( - name='Default Collection Object Group Types', - ) - -# Updates cogtype -> type to use the Default COGType picklist (Drill Core, Discrete, Consolidated) -def update_cogtype_type_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgrouptype', container__schematype=0, name='type').update( - picklistname=SYSTEM_COGTYPE_PICKLIST_NAME, - isrequired=True - ) - -def revert_cogtype_type_splocalecontaineritem(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - Splocalecontaineritem.objects.filter(container__name='collectionobjectgrouptype', container__schematype=0, name='type').update( - picklistname=None, - isrequired=None - ) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): dependencies = [ @@ -139,24 +24,29 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_fields(apps) - create_cogtype_picklist(apps) - update_cogtype_splocalecontaineritem(apps) - update_systemcogtypes_picklist(apps) - update_cogtype_type_splocalecontaineritem(apps) + usc.update_cog_type_fields(apps) + usc.create_cogtype_picklist(apps) + usc.update_cogtype_splocalecontaineritem(apps) + usc.update_systemcogtypes_picklist(apps) + usc.update_cogtype_type_splocalecontaineritem(apps) def revert_migration(apps, schema_editor): - revert_update_fields(apps) - revert_cogtype_picklist(apps) - revert_cogtype_splocalecontaineritem(apps) - revert_systemcogtypes_picklist(apps) - revert_cogtype_type_splocalecontaineritem(apps) + usc.revert_cog_type_fields(apps) + usc.revert_cogtype_picklist(apps) + usc.revert_cogtype_splocalecontaineritem(apps) + usc.revert_systemcogtypes_picklist(apps) + usc.revert_cogtype_type_splocalecontaineritem(apps) operations = [ migrations.AlterField( - model_name='collectionobjectgroupjoin', - name='parentcog', - field=models.ForeignKey(db_column='ParentCOGID', on_delete=django.db.models.deletion.CASCADE, related_name='children', to='specify.collectionobjectgroup'), + model_name="collectionobjectgroupjoin", + name="parentcog", + field=models.ForeignKey( + db_column="ParentCOGID", + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="specify.collectionobjectgroup", + ), ), - migrations.RunPython(apply_migration, revert_migration, atomic=True) - ] \ No newline at end of file + migrations.RunPython(apply_migration, revert_migration, atomic=True), + ] diff --git a/specifyweb/specify/migrations/0008_ageCitations_fix.py b/specifyweb/specify/migrations/0008_ageCitations_fix.py index 63543ec6a4e..9f058e2dbf3 100644 --- a/specifyweb/specify/migrations/0008_ageCitations_fix.py +++ b/specifyweb/specify/migrations/0008_ageCitations_fix.py @@ -3,23 +3,7 @@ from django.db import migrations, models import specifyweb.specify.models -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0008_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - # Add absoluteAgeCitation -> absoluteAge & Add relativeAgeCitation -> relativeAge - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def revert_update_fields(apps): - # Remove absoluteAgeCitation -> absoluteAge and relativeAgeCitation -> relativeAge - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -28,21 +12,33 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_fields(apps) + usc.update_relative_age_fields(apps) def revert_migration(apps, schema_editor): - revert_update_fields(apps) + usc.revert_relative_age_fields(apps) operations = [ migrations.AddField( - model_name='absoluteage', - name='absoluteagecitation', - field=models.ForeignKey(db_column='AbsoluteAgeCitationID', null=True, on_delete=specifyweb.specify.models.protect_with_blockers, related_name='absoluteages', to='specify.absoluteagecitation'), + model_name="absoluteage", + name="absoluteagecitation", + field=models.ForeignKey( + db_column="AbsoluteAgeCitationID", + null=True, + on_delete=specifyweb.specify.models.protect_with_blockers, + related_name="absoluteages", + to="specify.absoluteagecitation", + ), ), migrations.AddField( - model_name='relativeage', - name='relativeagecitation', - field=models.ForeignKey(db_column='RelativeAgeCitationID', null=True, on_delete=specifyweb.specify.models.protect_with_blockers, related_name='relativeages', to='specify.relativeagecitation'), + model_name="relativeage", + name="relativeagecitation", + field=models.ForeignKey( + db_column="RelativeAgeCitationID", + null=True, + on_delete=specifyweb.specify.models.protect_with_blockers, + related_name="relativeages", + to="specify.relativeagecitation", + ), ), - migrations.RunPython(apply_migration, revert_migration, atomic=True) + migrations.RunPython(apply_migration, revert_migration, atomic=True), ] diff --git a/specifyweb/specify/migrations/0009_tectonic_ranks.py b/specifyweb/specify/migrations/0009_tectonic_ranks.py index 5f32ab332c7..f97c7560a19 100644 --- a/specifyweb/specify/migrations/0009_tectonic_ranks.py +++ b/specifyweb/specify/migrations/0009_tectonic_ranks.py @@ -1,122 +1,11 @@ from django.db import migrations -from specifyweb.specify.models import protect_with_blockers - -def create_default_tectonic_ranks(apps): - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) - - root, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Root", - title="Root", - rankid=0, - parent=None, - treedef=tectonic_tree_def, - ) - superstructure, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Superstructure", - title="Superstructure", - rankid=10, - parent=root, - treedef=tectonic_tree_def, - ) - tectonic_domain, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Domain", - title="Tectonic Domain", - rankid=20, - parent=superstructure, - treedef=tectonic_tree_def, - ) - tectonic_subdomain, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Subdomain", - title="Tectonic Subdomain", - rankid=30, - parent=tectonic_domain, - treedef=tectonic_tree_def, - ) - tectonic_unit, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Unit", - title="Tectonic Unit", - rankid=40, - parent=tectonic_subdomain, - treedef=tectonic_tree_def, - ) - tectonic_subunit, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Subunit", - title="Tectonic Subunit", - rankid=50, - parent=tectonic_unit, - treedef=tectonic_tree_def, - ) - - Discipline.objects.filter(id=discipline.id).update(tectonicunittreedef=tectonic_tree_def) - -def revert_default_tectonic_ranks(apps, schema_editor): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - tectonic_tree_defs = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline) - - for tectonic_tree_def in tectonic_tree_defs: - tectonic_unit_tree_def_items = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).order_by('-id') - - for item in tectonic_unit_tree_def_items: - TectonicUnit.objects.filter(definitionitem=item).delete() - - item.delete() - - Discipline.objects.filter(id=discipline.id).update(tectonicunittreedef=None) - tectonic_tree_def.delete() - -def create_root_tectonic_node(apps): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicUnitTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - - tectonic_tree_def, created = TectonicUnitTreeDef.objects.get_or_create( - name="Tectonic Unit", - discipline=discipline - ) - - tectonic_tree_def_item, create = TectonicUnitTreeDefItem.objects.get_or_create( - name="Root", - treedef=tectonic_tree_def - ) - - root, _ = TectonicUnit.objects.get_or_create( - name="Root", - fullname="Root", - isaccepted=1, - nodenumber=1, - rankid=0, - parent=None, - definition=tectonic_tree_def, - definitionitem=tectonic_tree_def_item - ) - -def revert_create_root_tectonic_node(apps, schema_editor): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() - - if tectonic_tree_def: - TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() - TectonicUnit.objects.filter( - name="Root" - ).delete() +from specifyweb.specify.migration_utils.tectonic_ranks import ( + create_default_tectonic_ranks, + create_root_tectonic_node, + revert_create_root_tectonic_node, + revert_default_tectonic_ranks, +) class Migration(migrations.Migration): @@ -129,9 +18,13 @@ def consolidated_python_django_migration_operations(apps, schema_editor): create_root_tectonic_node(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_default_tectonic_ranks(apps, schema_editor) revert_create_root_tectonic_node(apps, schema_editor) - + revert_default_tectonic_ranks(apps, schema_editor) + operations = [ - migrations.RunPython(consolidated_python_django_migration_operations, revert_cosolidated_python_django_migration_operations, atomic=True), + migrations.RunPython( + consolidated_python_django_migration_operations, + revert_cosolidated_python_django_migration_operations, + atomic=True, + ) ] diff --git a/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py b/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py index 2ca78d86333..a289553b43f 100644 --- a/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py +++ b/specifyweb/specify/migrations/0012_add_cojo_to_schema_config.py @@ -2,24 +2,7 @@ This migration adds COG -> cojo and CO -> cojo to Schema Config. """ from django.db import migrations -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0012_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - - -def add_cojo_to_schema_config(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults( - table, discipline.id, field, apps) - - -def remove_cojo_from_schema_config(apps): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -28,10 +11,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - add_cojo_to_schema_config(apps) + usc.add_cojo_to_schema_config(apps) def revert_migration(apps, schema_editor): - remove_cojo_from_schema_config(apps) + usc.remove_cojo_from_schema_config(apps) operations = [ migrations.RunPython(apply_migration, revert_migration, atomic=True), diff --git a/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py b/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py index 490ef8b90a7..5c0f5f25d4e 100644 --- a/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py +++ b/specifyweb/specify/migrations/0013_collectionobjectgroup_parentcog.py @@ -2,34 +2,7 @@ from django.db import migrations, models import specifyweb.specify.models -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0013_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - - -def update_schema_config(apps): - revert_table_field_schema_config( - 'CollectionObjectGroup', 'parentCojo', apps) - revert_table_field_schema_config( - 'CollectionObjectGroup', 'parentCog', apps) - - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults( - table, discipline.id, field, apps) - - -def revert_update(apps): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - update_table_field_schema_config_with_defaults( - 'CollectionObjectGroup', discipline.id, 'parentCojo', apps) - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -38,10 +11,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_schema_config(apps) + usc.update_cog_schema_config(apps) def revert_migration(apps, schema_editor): - revert_update(apps) + usc.revert_update_cog_schema_config(apps) operations = [ migrations.RemoveField( diff --git a/specifyweb/specify/migrations/0015_add_version_to_ages.py b/specifyweb/specify/migrations/0015_add_version_to_ages.py index 07b9e9eff80..2dd5d0651b1 100644 --- a/specifyweb/specify/migrations/0015_add_version_to_ages.py +++ b/specifyweb/specify/migrations/0015_add_version_to_ages.py @@ -1,20 +1,7 @@ # Generated by Django 3.2.15 on 2024-12-03 18:59 from django.db import migrations, models -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults - -def update_schema_config(apps): - # Revert before adding to avoid duplicates - revert_update(apps) - - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - update_table_field_schema_config_with_defaults('AbsoluteAge', discipline.id, 'version', apps) - update_table_field_schema_config_with_defaults('RelativeAge', discipline.id, 'version', apps) - -def revert_update(apps): - revert_table_field_schema_config('AbsoluteAge', 'version', apps) - revert_table_field_schema_config('RelativeAge', 'version', apps) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -23,10 +10,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_schema_config(apps) + usc.update_age_schema_config(apps) def revert_migration(apps, schema_editor): - revert_update(apps) + usc.revert_update_age_schema_config(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0017_schemaconfig_fixes.py b/specifyweb/specify/migrations/0017_schemaconfig_fixes.py index 16b9bf605af..f543efeed97 100644 --- a/specifyweb/specify/migrations/0017_schemaconfig_fixes.py +++ b/specifyweb/specify/migrations/0017_schemaconfig_fixes.py @@ -1,23 +1,7 @@ # Generated by Django 3.2.15 on 2024-11-21 20:08 from django.db import migrations -from django.db.models import F - -from specifyweb.specify.migration_utils.update_schema_config import datamodel_type_to_schematype, uncapitilize, camel_to_spaced_title_case -from specifyweb.specify.datamodel import datamodel - -from specifyweb.specify.migration_utils.sp7_schemaconfig import ( - # SpLocaleContainer migration changes - MIGRATION_0002_TABLES, MIGRATION_0004_TABLES, - - # SpLocaleContainerItem migration changes - MIGRATION_0004_FIELDS, MIGRATION_0007_FIELDS, MIGRATION_0008_FIELDS, MIGRATION_0012_FIELDS, MIGRATION_0013_FIELDS -) - -CONTAINER_MIGRATIONS = [MIGRATION_0002_TABLES, MIGRATION_0004_TABLES] - -CONTAINER_ITEM_MIGRATIONS = [ - MIGRATION_0004_FIELDS, MIGRATION_0007_FIELDS, MIGRATION_0008_FIELDS, MIGRATION_0012_FIELDS, MIGRATION_0013_FIELDS] +from specifyweb.specify.migration_utils import update_schema_config as usc """ This migration fixes two bugs introduced in other migrations by the functions @@ -30,70 +14,12 @@ the proper lables """ - -def fix_table_captions(apps): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for migration in CONTAINER_MIGRATIONS: - for table_name, table_desc in migration: - table = datamodel.get_table_strict(table_name) - containers = Splocalecontainer.objects.filter( - name=table_name.lower(), schematype=0) - - # If needed, correct the label of the table in the schema config - if table_desc is not None: - Splocaleitemstr.objects.filter( - containername__in=containers, text=table_desc).update(text=camel_to_spaced_title_case(uncapitilize(table.name))) - - # Update the types for the fields in the table - items = Splocalecontaineritem.objects.filter( - container__in=containers) - for item in items: - datamodel_field = table.get_field(item.name) - if not datamodel_field: - continue - - item.type = datamodel_type_to_schematype( - datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type - item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired - - item.save() - - -def fix_item_types(apps): - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for migration in CONTAINER_ITEM_MIGRATIONS: - for table_name, fields in migration.items(): - table = datamodel.get_table_strict(table_name) - items = Splocalecontaineritem.objects.filter( - container__name=table_name.lower(), container__schematype=0, name__in=fields) - - for item in items: - datamodel_field = table.get_field(item.name) - if not datamodel_field: - continue - - item.type = datamodel_type_to_schematype( - datamodel_field.type) if datamodel_field.is_relationship else datamodel_field.type - item.isrequired = datamodel_field.required if item.isrequired is None else item.isrequired - - item.save() - - -def schemaconfig_fixes(apps, schema_editor): - fix_table_captions(apps) - fix_item_types(apps) - - class Migration(migrations.Migration): dependencies = [ ('specify', '0016_collectionobjecttype_catalognumformatname'), ] operations = [ - migrations.RunPython(schemaconfig_fixes, + migrations.RunPython(usc.schemaconfig_fixes, migrations.RunPython.noop, atomic=True) ] diff --git a/specifyweb/specify/migrations/0018_cot_catnum_schema.py b/specifyweb/specify/migrations/0018_cot_catnum_schema.py index 6919fbe89cf..7e5c378be6d 100644 --- a/specifyweb/specify/migrations/0018_cot_catnum_schema.py +++ b/specifyweb/specify/migrations/0018_cot_catnum_schema.py @@ -1,63 +1,7 @@ # Generated by Django 3.2.15 on 2024-11-21 20:08 from django.db import migrations -from django.db.models import Q - -from specifyweb.specify.datamodel import datamodel -from specifyweb.specify.migration_utils.update_schema_config import camel_to_spaced_title_case - - -def add_cot_catnum_to_schema(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - CollectionObjectType_Table = datamodel.get_table_strict( - 'collectionobjecttype') - catalognumber_format_field = CollectionObjectType_Table.get_field_strict( - 'catalogNumberFormatName') - - for container in Splocalecontainer.objects.filter(name='collectionobjecttype', schematype=0): - schema_item, created = Splocalecontaineritem.objects.get_or_create( - name=catalognumber_format_field.name, type=catalognumber_format_field.type, container=container) - if created: - schema_item.version = 0 - - schema_item.isrequired = catalognumber_format_field.required if schema_item.isrequired is None else schema_item.isrequired - - schema_item.save() - - schema_name = camel_to_spaced_title_case( - catalognumber_format_field.name) - Splocaleitemstr.objects.get_or_create( - language='en', text=schema_name, itemname=schema_item) - Splocaleitemstr.objects.get_or_create( - language='en', text=schema_name, itemdesc=schema_item) - - -def remove_cot_catnum_from_schema(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - CollectionObjectType_Table = datamodel.get_table_strict( - 'collectionobjecttype') - catalognumber_format_field = CollectionObjectType_Table.get_field_strict( - 'catalogNumberFormatName') - - containers = Splocalecontainer.objects.filter( - name='collectionobjecttype', schematype=0) - items = Splocalecontaineritem.objects.filter( - name='catalogNumberFormatName', container__in=containers) - - schema_name = camel_to_spaced_title_case(catalognumber_format_field.name) - filters = Q(language='en', text=schema_name) & ( - Q(itemname__in=items) | Q(itemdesc__in=items)) - locale_strings = Splocaleitemstr.objects.filter(filters) - - locale_strings.delete() - items.delete() - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -66,6 +10,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(add_cot_catnum_to_schema, - remove_cot_catnum_from_schema, atomic=True) + migrations.RunPython(usc.add_cot_catnum_to_schema, + usc.remove_cot_catnum_from_schema, atomic=True) ] diff --git a/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py b/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py index 2b84618a827..455dd7ee047 100644 --- a/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py +++ b/specifyweb/specify/migrations/0020_add_tectonicunit_to_pc_in_schema_config.py @@ -2,24 +2,7 @@ This migration adds Tectonic Unit -> Paleo Context in the Schema Config. """ from django.db import migrations -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0020_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - - -def add_tectonicunit_to_pc_in_schema_config(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults( - table, discipline.id, field, apps) - - -def remove_tectonicunit_from_pc_schema_config(apps): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) - +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -28,10 +11,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - add_tectonicunit_to_pc_in_schema_config(apps) + usc.add_tectonicunit_to_pc_in_schema_config(apps) def revert_migration(apps, schema_editor): - remove_tectonicunit_from_pc_schema_config(apps) + usc.remove_tectonicunit_from_pc_schema_config(apps) operations = [ migrations.RunPython(apply_migration, revert_migration, atomic=True), diff --git a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py index bc5738d69d4..26a1a2c4c25 100644 --- a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py +++ b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py @@ -3,54 +3,7 @@ """ from django.db import migrations -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0021_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS -from specifyweb.specify.models_utils.model_extras import GEOLOGY_DISCIPLINES, PALEO_DISCIPLINES - -def fix_hidden_geo_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Discipline = apps.get_model('specify', 'Discipline') - - excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES - - filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) - - for discipline in filtered_disciplines: - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - discipline_id=discipline.id, - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - items.update(ishidden=True) - -def reverse_fix_hidden_geo_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Discipline = apps.get_model('specify', 'Discipline') - - excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES - - filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) - - for discipline in filtered_disciplines: - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - discipline_id=discipline.id, - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - items.update(ishidden=True) +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): dependencies = [ @@ -58,5 +11,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(fix_hidden_geo_prop, reverse_fix_hidden_geo_prop, atomic=True) + migrations.RunPython(usc.fix_hidden_geo_prop, usc.reverse_fix_hidden_geo_prop, atomic=True) ] diff --git a/specifyweb/specify/migrations/0022_ensure_default_cots.py b/specifyweb/specify/migrations/0022_ensure_default_cots.py index d687d1a9677..9f2219e6645 100644 --- a/specifyweb/specify/migrations/0022_ensure_default_cots.py +++ b/specifyweb/specify/migrations/0022_ensure_default_cots.py @@ -1,5 +1,5 @@ from django.db import migrations -from specifyweb.specify.api.utils import create_default_collection_types +from specifyweb.specify.migration_utils.default_cots import create_default_collection_types class Migration(migrations.Migration): diff --git a/specifyweb/specify/migrations/0023_update_schema_config_text.py b/specifyweb/specify/migrations/0023_update_schema_config_text.py index 533b9e119ff..4ddecb4d688 100644 --- a/specifyweb/specify/migrations/0023_update_schema_config_text.py +++ b/specifyweb/specify/migrations/0023_update_schema_config_text.py @@ -3,109 +3,7 @@ """ from django.db import migrations -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0023_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0023_FIELDS_BIS as SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS - -def update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - #i.e: Collection Object - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - #i.e: COType - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = new_desc - localized_items_desc.save() - - localized_items_name.text = new_name - localized_items_name.save() - -def update_hidden_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - item.ishidden = True - item.save() - -def reverse_update_hidden_prop(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS_BIS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - for container in containers: - for field_name in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - item.ishidden = False - item.save() - -def reverse_update_schema_config_field_desc(apps, schema_editor): - Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') - Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') - - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - containers = Splocalecontainer.objects.filter( - name=table.lower(), - ) - - for container in containers: - for field_name, new_name, new_desc in fields: - items = Splocalecontaineritem.objects.filter( - container=container, - name=field_name.lower() - ) - - for item in items: - localized_items_desc = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - localized_items_name = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - - if localized_items_desc is None or localized_items_name is None: - continue - - localized_items_desc.text = item.name - localized_items_desc.save() - - localized_items_name.text = item.name - localized_items_name.save() +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): dependencies = [ @@ -117,9 +15,9 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython( - update_schema_config_field_desc, reverse_update_schema_config_field_desc, atomic=True + usc.update_schema_config_field_desc, usc.reverse_update_schema_config_field_desc, atomic=True ), migrations.RunPython( - update_hidden_prop, reverse_update_hidden_prop, atomic=True + usc.update_hidden_prop, usc.reverse_update_hidden_prop, atomic=True ), ] \ No newline at end of file diff --git a/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py b/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py index 35cc56b98f9..e7b26de5b9e 100644 --- a/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py +++ b/specifyweb/specify/migrations/0024_add_uniqueIdentifier_storage.py @@ -1,22 +1,6 @@ from django.db import migrations, models - -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0024_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS - -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - - # Add uniqueIdentifier -> storage - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) - -def revert_update_fields(apps): - # Remove uniqueIdentifier -> storage - for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): - for field in fields: - revert_table_field_schema_config(table, field, apps) +from specifyweb.backend.businessrules.uniqueness_rules import DEFAULT_UNIQUENESS_RULES +from specifyweb.specify.migration_utils import update_schema_config as usc class Migration(migrations.Migration): @@ -25,10 +9,10 @@ class Migration(migrations.Migration): ] def apply_migration(apps, schema_editor): - update_fields(apps) + usc.update_storage_unique_id_fields(apps) def revert_migration(apps, schema_editor): - revert_update_fields(apps) + usc.revert_storage_unique_id_fields(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0027_CO_children.py b/specifyweb/specify/migrations/0027_CO_children.py index 9d7ac906294..f7f5f8c18a7 100644 --- a/specifyweb/specify/migrations/0027_CO_children.py +++ b/specifyweb/specify/migrations/0027_CO_children.py @@ -1,72 +1,158 @@ # Generated by Django 3.2.15 on 2025-04-11 15:35 -from django.apps import apps as specify_apps +import re + +from django.core.exceptions import MultipleObjectsReturned from django.db import migrations, models +from django.db.models import Q import django.db.models.deletion -from specifyweb.specify.migration_utils.update_schema_config import revert_table_field_schema_config, update_table_field_schema_config_with_defaults -import specifyweb.specify.models -from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0027_FIELDS as SCHEMA_CONFIG_TABLE_FIELDS, MIGRATION_0027_UPDATE_FIELDS as SCHEMA_CONFIG_CO_TABLE_FIELDS +from specifyweb.specify.models import datamodel +from specifyweb.specify.models_utils.load_datamodel import FieldDoesNotExistError -def update_fields(apps): - Discipline = apps.get_model('specify', 'Discipline') - for discipline in Discipline.objects.all(): - for table, fields in SCHEMA_CONFIG_TABLE_FIELDS.items(): - for field in fields: - update_table_field_schema_config_with_defaults(table, discipline.id, field, apps) +MIGRATION_0027_FIELDS = { + 'CollectionObject': ['parentCO', 'children'], +} + +MIGRATION_0027_UPDATE_FIELDS = { + 'CollectionObject': [ + ('parentCO', 'Parent Collection Object', 'Parent CollectionObject'), + ('children', 'Children', 'Children'), + ] +} + +HIDDEN_FIELDS = [ + "timestampcreated", "timestampmodified", "version", "createdbyagent", "modifiedbyagent" +] + + +def datamodel_type_to_schematype(datamodel_type: str) -> str: + return "".join(map(lambda type_part: type_part.lower().capitalize(), datamodel_type.split('-'))) + + +def camel_to_spaced_title_case(camel_case: str) -> str: + return re.sub(r"(? Date: Wed, 20 May 2026 14:21:46 -0500 Subject: [PATCH 05/17] Revert "Fix PR review issues for key migration pipeline" This reverts commit f2a35168af06431dffa2e961eddd2cb738e7ddd4. --- .../backend/businessrules/migration_utils.py | 8 +- .../migrations/0004_catnum_uniquerule.py | 29 ++++ .../backend/businessrules/uniqueness_rules.py | 91 +++++----- specifyweb/backend/patches/migration_utils.py | 12 +- specifyweb/backend/permissions/initialize.py | 24 +-- ...add_dataset_create_recordset_permission.py | 49 +----- .../0007_add_stats_edit_permission.py | 16 +- .../backend/stored_queries/execution.py | 8 +- .../backend/stored_queries/geology_time.py | 4 +- specifyweb/backend/stored_queries/utils.py | 10 +- .../commands/run_key_migration_functions.py | 13 +- .../specify/migration_utils/default_cots.py | 41 ++--- .../migration_utils/sp7_schemaconfig.py | 2 +- .../specify/migration_utils/tectonic_ranks.py | 37 ++-- .../migration_utils/update_schema_config.py | 56 +++--- specifyweb/specify/migrations/0002_geo.py | 1 + .../migrations/0008_ageCitations_fix.py | 2 +- .../specify/migrations/0009_tectonic_ranks.py | 2 +- .../0021_update_hidden_geo_tables.py | 49 ++++++ .../specify/migrations/0027_CO_children.py | 162 +----------------- 20 files changed, 224 insertions(+), 392 deletions(-) diff --git a/specifyweb/backend/businessrules/migration_utils.py b/specifyweb/backend/businessrules/migration_utils.py index f5d689b9853..38265a6b20c 100644 --- a/specifyweb/backend/businessrules/migration_utils.py +++ b/specifyweb/backend/businessrules/migration_utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Tuple, List from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule @@ -47,7 +47,6 @@ def catnum_rule_uneditable(apps, schema_editor=None): model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", discipline_id=discipline.id, isDatabaseConstraint=False) has_catalognumber_rule = False - matching_rule_ids: List[int] = [] for rule in model_rules: rule_fields = rule.uniquenessrulefield_set.all() @@ -60,11 +59,8 @@ def catnum_rule_uneditable(apps, schema_editor=None): # exception if more than one result is returned if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): has_catalognumber_rule = True - matching_rule_ids.append(rule.id) - if has_catalognumber_rule: - UniquenessRule.objects.filter(id__in=matching_rule_ids).update(isDatabaseConstraint=True) - else: + if not has_catalognumber_rule: create_uniqueness_rule( "Collectionobject", discipline=discipline, diff --git a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py index 314a8ad9725..33f2798cb81 100644 --- a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py +++ b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py @@ -1,6 +1,35 @@ from django.db import migrations from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable, catnum_rule_uneditable +from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule + + +def catnum_rule_editable(apps, schema_editor): + UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') + + candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__isDatabaseConstraint=True, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) + + candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) + + candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) + candidate_rules.update(isDatabaseConstraint=False) + +def catnum_rule_uneditable(apps, schema_editor): + Discipline = apps.get_model('specify', 'Discipline') + UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') + + for discipline in Discipline.objects.all(): + candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__discipline=discipline.id, uniquenessrule__isDatabaseConstraint=False, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) + + candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) + + candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) + if len(candidate_rules) == 0: + create_uniqueness_rule('Collectionobject', discipline=discipline, is_database_constraint=True, fields=['catalogNumber'], scopes=['collection'], registry=apps) + else: + candidate_rules.update(isDatabaseConstraint=True) class Migration(migrations.Migration): dependencies = [ diff --git a/specifyweb/backend/businessrules/uniqueness_rules.py b/specifyweb/backend/businessrules/uniqueness_rules.py index dec36a13a7e..fb3f055dbce 100644 --- a/specifyweb/backend/businessrules/uniqueness_rules.py +++ b/specifyweb/backend/businessrules/uniqueness_rules.py @@ -7,7 +7,7 @@ from django.apps import apps from django.db import connections, transaction -from django.db.models import Q, Count +from django.db.models import Q, Count, Exists, OuterRef from django.db.migrations.recorder import MigrationRecorder from django.core.exceptions import ObjectDoesNotExist from specifyweb.specify.api.crud import get_model @@ -266,19 +266,13 @@ def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f candidate_rules = UniquenessRule.objects.filter(modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) - expected_fields = set(fields) - expected_scopes = set(scopes) for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() - existing_fields = set( - all_fields.filter(isScope=False).values_list("fieldPath", flat=True) - ) - existing_scopes = set( - all_fields.filter(isScope=True).values_list("fieldPath", flat=True) - ) + matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) + matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) # If the rule already exists, skip creating the rule - if existing_fields == expected_fields and existing_scopes == expected_scopes: + if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): return logger.info(f"Creating uniqueness rule on {model_name} with fields {fields} and scopes {scopes} for the discipline {discipline.name if discipline else 'Global'}") @@ -301,19 +295,13 @@ def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f candidate_rules = UniquenessRule.objects.filter( modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) - expected_fields = set(fields) - expected_scopes = set(scopes) rule_ids = [] for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() - existing_fields = set( - all_fields.filter(isScope=False).values_list("fieldPath", flat=True) - ) - existing_scopes = set( - all_fields.filter(isScope=True).values_list("fieldPath", flat=True) - ) + matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) + matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) # If the rule exists, add it to the list of rules to be deleted - if existing_fields == expected_fields and existing_scopes == expected_scopes: + if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): rule_ids.append(rule.id) UniquenessRuleField.objects.filter( @@ -335,19 +323,24 @@ def fix_global_default_rules(registry=None): UniquenessRule = registry.get_model('businessrules', 'UniquenessRule') \ if registry \ else models.UniquenessRule + UniquenessRuleField = registry.get_model('businessrules', 'UniquenessRuleField') \ + if registry \ + else models.UniquenessRuleField + + global_rule_fields = UniquenessRuleField.objects.filter( + uniquenessrule__discipline__isnull=True + ).values( + "uniquenessrule__modelName", + "uniquenessrule__isDatabaseConstraint", + "fieldPath", + "isScope", + ) - global_rule_signatures = { - ( - rule.modelName, - rule.isDatabaseConstraint, - frozenset( - rule.uniquenessrulefield_set.values_list("fieldPath", "isScope") - ), - ) - for rule in UniquenessRule.objects.filter( - discipline__isnull=True - ).prefetch_related("uniquenessrulefield_set") - } + global_rule_exists = UniquenessRule.objects.filter( + discipline__isnull=True, + modelName=OuterRef("modelName"), + isDatabaseConstraint=OuterRef("isDatabaseConstraint"), + ) discipline_ids = ( UniquenessRule.objects.exclude(discipline__isnull=True) @@ -357,16 +350,28 @@ def fix_global_default_rules(registry=None): for discipline_id in discipline_ids: with transaction.atomic(): - for rule in UniquenessRule.objects.filter( - discipline_id=discipline_id - ).prefetch_related("uniquenessrulefield_set"): - signature = ( - rule.modelName, - rule.isDatabaseConstraint, - frozenset( - rule.uniquenessrulefield_set.values_list("fieldPath", "isScope") - ), + # Delete matching fields for this discipline + matching_fields_qs = UniquenessRuleField.objects.filter( + uniquenessrule__discipline_id=discipline_id + ).filter( + Exists( + global_rule_fields.filter( + **{ + "uniquenessrule__modelName": OuterRef("uniquenessrule__modelName"), + "uniquenessrule__isDatabaseConstraint": OuterRef("uniquenessrule__isDatabaseConstraint"), + "fieldPath": OuterRef("fieldPath"), + "isScope": OuterRef("isScope"), + } + ) ) - if signature in global_rule_signatures: - rule.uniquenessrulefield_set.all().delete() - rule.delete() + ) + matching_fields_qs.delete() + + # Delete UniquenessRule rows for this discipline that are now empty + empty_rules_qs = ( + UniquenessRule.objects.filter(discipline_id=discipline_id) + .annotate(field_count=Count("uniquenessrulefield")) + .filter(field_count=0) # now empty after field deletions + .filter(Exists(global_rule_exists)) + ) + empty_rules_qs.delete() diff --git a/specifyweb/backend/patches/migration_utils.py b/specifyweb/backend/patches/migration_utils.py index 4dfdc463387..a773cb10ad0 100644 --- a/specifyweb/backend/patches/migration_utils.py +++ b/specifyweb/backend/patches/migration_utils.py @@ -10,7 +10,6 @@ def apply_migrations(app_registry, schema_editor=None): update_coordinates(app_registry, schema_editor) def update_is_accepted(app_registry, schema_editor=None): - db_alias = schema_editor.connection.alias if schema_editor is not None else "default" for tree in SPECIFY_TREES: tree_filters = { "isaccepted": False, @@ -18,21 +17,20 @@ def update_is_accepted(app_registry, schema_editor=None): } tree_model = app_registry.get_model("specify", tree) - tree_model._base_manager.using(db_alias).filter(**tree_filters).update(isaccepted=True) + tree_model.objects.filter(**tree_filters).update(isaccepted=True) def update_coordinates(app_registry, schema_editor=None): - db_alias = schema_editor.connection.alias if schema_editor is not None else "default" Locality = app_registry.get_model("specify", "Locality") - Locality._base_manager.using(db_alias).filter(lat1text__isnull=True, latitude1__isnull=False) \ + Locality.objects.filter(lat1text__isnull=True, latitude1__isnull=False) \ .update(lat1text=F("latitude1")) - Locality._base_manager.using(db_alias).filter(long1text__isnull=True, longitude1__isnull=False) \ + Locality.objects.filter(long1text__isnull=True, longitude1__isnull=False) \ .update(long1text=F("longitude1")) - Locality._base_manager.using(db_alias).filter(lat2text__isnull=True, latitude2__isnull=False) \ + Locality.objects.filter(lat2text__isnull=True, latitude2__isnull=False) \ .update(lat2text=F("latitude2")) - Locality._base_manager.using(db_alias).filter(long2text__isnull=True, longitude2__isnull=False) \ + Locality.objects.filter(long2text__isnull=True, longitude2__isnull=False) \ .update(long2text=F("longitude2")) diff --git a/specifyweb/backend/permissions/initialize.py b/specifyweb/backend/permissions/initialize.py index fe04beac578..fb8d981891c 100644 --- a/specifyweb/backend/permissions/initialize.py +++ b/specifyweb/backend/permissions/initialize.py @@ -47,14 +47,13 @@ def create_admins(apps=apps) -> None: UserPolicy = apps.get_model('permissions', 'UserPolicy') Specifyuser = apps.get_model('specify', 'Specifyuser') + if UserPolicy.objects.filter(collection__isnull=True, resource='%', action='%').exists(): + # don't do anything if there is already any admin. + return + users = Specifyuser.objects.all() for user in users: - if UserPolicy.objects.filter( - collection__isnull=True, - specifyuser_id=user.id, - resource="%", - action="%", - ).exists(): + if is_sp6_user_permissions_migrated(user, apps): continue if is_legacy_admin(user): UserPolicy.objects.get_or_create( @@ -98,12 +97,17 @@ def assign_users_to_roles(apps=apps) -> None: JOIN spprincipal p ON p.SpPrincipalID = up.SpPrincipalID JOIN collection c ON c.UserGroupScopeId = p.userGroupScopeID WHERE p.groupType IS NULL - AND NOT EXISTS ( - SELECT 1 + AND u.SpecifyUserID NOT IN ( + SELECT ur.specifyuser_id FROM spuserrole ur JOIN sprole r ON r.id = ur.role_id - WHERE r.collection_id = c.UserGroupScopeId - AND ur.specifyuser_id = u.SpecifyUserID + WHERE r.collection_id = p.usergroupscopeid + ) + AND c.UserGroupScopeId NOT IN ( + SELECT DISTINCT r.collection_id + FROM spuserrole ur + JOIN sprole r ON r.id = ur.role_id + JOIN collection c ON c.UserGroupScopeId = r.collection_id ); """) diff --git a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py index 936fc821511..b9d092009af 100644 --- a/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py +++ b/specifyweb/backend/permissions/migrations/0006_add_dataset_create_recordset_permission.py @@ -2,54 +2,7 @@ from django.db import migrations -from specifyweb.backend.workbench.upload.auditlog import auditlog - - -def add_permission(apps, schema_editor=None): - UserPolicy = apps.get_model('permissions', 'UserPolicy') - LibraryRolePolicy = apps.get_model('permissions', 'LibraryRolePolicy') - RolePolicy = apps.get_model('permissions', 'RolePolicy') - - for p in UserPolicy.objects.filter(resource='/workbench/dataset', action='upload'): - if not UserPolicy.objects.filter( - collection=p.collection, - specifyuser=p.specifyuser, - resource=p.resource, - action='create_recordset', - ).exists(): - user_policy = UserPolicy.objects.create( - collection=p.collection, - specifyuser=p.specifyuser, - resource=p.resource, - action='create_recordset', - ) - auditlog.insert(user_policy) - - for p in RolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - if not RolePolicy.objects.filter( - role=p.role, - resource=p.resource, - action='create_recordset', - ).exists(): - role_policy = RolePolicy.objects.create( - role=p.role, - resource=p.resource, - action='create_recordset', - ) - auditlog.insert(role_policy, None) - - for p in LibraryRolePolicy.objects.filter(resource='/workbench/dataset', action='upload'): - if not LibraryRolePolicy.objects.filter( - role=p.role, - resource=p.resource, - action='create_recordset', - ).exists(): - library_role_policy = LibraryRolePolicy.objects.create( - role=p.role, - resource=p.resource, - action='create_recordset', - ) - auditlog.insert(library_role_policy, None) +from specifyweb.permissions.migration_utils.edit_permissions import add_permission class Migration(migrations.Migration): diff --git a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py index a850816b560..fcb886f2dd2 100644 --- a/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py +++ b/specifyweb/backend/permissions/migrations/0007_add_stats_edit_permission.py @@ -1,19 +1,5 @@ from django.db import migrations -from specifyweb.backend.workbench.upload.auditlog import auditlog - - -def add_stats_edit_permission(apps, schema_editor=None): - Role = apps.get_model('permissions', 'Role') - - all_full_access_roles = Role.objects.filter(name="Full Access - Legacy") - - for full_access_role in all_full_access_roles: - new_policy, created = full_access_role.policies.get_or_create( - resource="/preferences/statistics", - action="edit", - ) - if created: - auditlog.insert(new_policy, None) +from specifyweb.permissions.migration_utils.edit_permissions import add_stats_edit_permission class Migration(migrations.Migration): diff --git a/specifyweb/backend/stored_queries/execution.py b/specifyweb/backend/stored_queries/execution.py index 9e611590c49..14121c0d676 100644 --- a/specifyweb/backend/stored_queries/execution.py +++ b/specifyweb/backend/stored_queries/execution.py @@ -852,8 +852,7 @@ def execute( - if settings.DEBUG: - log_sqlalchemy_query(query) # Debugging + log_sqlalchemy_query(query) # Debugging return {"results": apply_special_post_query_processing(query, tableid, field_specs, collection, user)} def build_query( @@ -1060,8 +1059,7 @@ def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" - if settings.DEBUG: - log_sqlalchemy_query(query) # Debugging + log_sqlalchemy_query(query) # Debugging def parse_catalog_for_comparing(s): def check_for_decimal(s): @@ -1186,4 +1184,4 @@ def apply_special_post_query_processing(query, tableid, field_specs, collection, if should_list_query: return list(query) - return query + return query \ No newline at end of file diff --git a/specifyweb/backend/stored_queries/geology_time.py b/specifyweb/backend/stored_queries/geology_time.py index 8946241b0f2..0a1a4ffe9d8 100644 --- a/specifyweb/backend/stored_queries/geology_time.py +++ b/specifyweb/backend/stored_queries/geology_time.py @@ -1,6 +1,5 @@ import logging import os -from django.conf import settings from django.db.models import Case, FloatField, F, Q, Value, When from django.db.models.functions import Coalesce, Greatest, Least, Cast from specifyweb.backend.stored_queries.utils import log_sqlalchemy_query @@ -978,6 +977,5 @@ def modify_query_add_meta_age_range(query, start_time, end_time, require_full_ov ).label("age") new_query = new_query.add_columns(age_expr) - if settings.DEBUG: - log_sqlalchemy_query(new_query) + log_sqlalchemy_query(new_query) return new_query diff --git a/specifyweb/backend/stored_queries/utils.py b/specifyweb/backend/stored_queries/utils.py index 1ab1dbcc3cb..218423c48f0 100644 --- a/specifyweb/backend/stored_queries/utils.py +++ b/specifyweb/backend/stored_queries/utils.py @@ -5,15 +5,11 @@ def log_sqlalchemy_query(query): # Call this function to debug the raw SQL query generated by SQLAlchemy # TODO: verify theis import - from django.conf import settings from sqlalchemy.dialects import mysql - if not settings.DEBUG: - return - compiled_query = query.statement.compile(dialect=mysql.dialect()) + compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) raw_sql = str(compiled_query).replace('\n', ' ') + ';' logger.debug('='.join(['' for _ in range(80)])) - logger.debug("SQL: %s", raw_sql) - logger.debug("Params: %s", compiled_query.params) + logger.debug(raw_sql) logger.debug('='.join(['' for _ in range(80)])) # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 - # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) + # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) \ No newline at end of file diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index ae767208d27..9aeeb3c8320 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -4,7 +4,6 @@ from django.core.management.base import BaseCommand from django.apps import apps from django.db import transaction -from django.db.models import Subquery from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable from specifyweb.backend.businessrules.uniqueness_rules import ( apply_default_uniqueness_rules, @@ -82,12 +81,8 @@ def apply_default_uniqueness_rules_to_disciplines(apps): Discipline = apps.get_model('specify', 'Discipline') UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - disciplines_with_rules = UniquenessRule.objects.exclude( - discipline_id__isnull=True - ).values("discipline_id").distinct() for discipline in Discipline.objects.exclude( - id__in=Subquery(disciplines_with_rules) - ): + id__in=set(UniquenessRule.objects.values_list('discipline_id', flat=True).distinct())): apply_default_uniqueness_rules(discipline, registry=apps) @@ -143,7 +138,7 @@ def add_arguments(self, parser): nargs="*", type=str, choices=tuple(self.funcs.keys()), - help="Optional: specify one or more functions to run", + help=f"Optional: specify one or more functions to run", ) parser.add_argument( "--verbose", @@ -176,6 +171,6 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS(f"Applying {func_name}...")) func(self.stdout.write if verbose else None) self.stdout.write(self.style.SUCCESS(f"Applied {func_name}")) - except Exception: - logger.exception("An error occurred while running key migrations") + except Exception as e: + logger.error(f"An error occurred: {e}") raise diff --git a/specifyweb/specify/migration_utils/default_cots.py b/specifyweb/specify/migration_utils/default_cots.py index 1b2c916131c..e9387b5ca8e 100644 --- a/specifyweb/specify/migration_utils/default_cots.py +++ b/specifyweb/specify/migration_utils/default_cots.py @@ -135,25 +135,22 @@ def fix_tectonic_unit_treedef_discipline_links(apps): Discipline = apps.get_model('specify', 'Discipline') Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef') - empty_tectonic_unit_treedefs = list( - Tectonicunittreedef.objects.filter(discipline__isnull=True) - ) - empty_disciplines = list( - Discipline.objects.filter(tectonicunittreedef__isnull=True) - ) - - for discipline, tectonic_unit_treedef in zip( - empty_disciplines, empty_tectonic_unit_treedefs - ): - tectonic_unit_treedef.discipline = discipline - tectonic_unit_treedef.save() - discipline.tectonicunittreedef = tectonic_unit_treedef - discipline.save() - - for discipline in empty_disciplines[len(empty_tectonic_unit_treedefs):]: - tectonic_unit_treedef = Tectonicunittreedef.objects.create( - name=f'{discipline.name} Tectonic Unit Tree', - discipline=discipline - ) - discipline.tectonicunittreedef = tectonic_unit_treedef - discipline.save() + empty_tectonic_unit_treedefs = Tectonicunittreedef.objects.filter(discipline__isnull=True) + empty_disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) + for empty_discipline in empty_disciplines: + if not empty_tectonic_unit_treedefs.exists(): + new_tectonic_unit_treedef = Tectonicunittreedef.objects.create( + name=f'{empty_discipline.name} Tectonic Unit Tree', + discipline=empty_discipline + ) + else: + empty_discipline.tectonicunittreedef = empty_tectonic_unit_treedefs.first() + empty_discipline.save() + + for empty_tectonic_unit_treedef in empty_tectonic_unit_treedefs: + if empty_disciplines.exists(): + empty_tectonic_unit_treedef.discipline = empty_disciplines.first() + empty_tectonic_unit_treedef.save() + else: + empty_tectonic_unit_treedef.discipline = empty_disciplines.last() + empty_tectonic_unit_treedef.save() diff --git a/specifyweb/specify/migration_utils/sp7_schemaconfig.py b/specifyweb/specify/migration_utils/sp7_schemaconfig.py index 81d28bca732..d870be61399 100644 --- a/specifyweb/specify/migration_utils/sp7_schemaconfig.py +++ b/specifyweb/specify/migration_utils/sp7_schemaconfig.py @@ -148,7 +148,7 @@ 'RelativeAge': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'], 'CollectionObject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'], 'AbsoluteAgeCitation': ['collectionMember', 'absoluteAgeCitationId'], - 'RelativeAgeCitation': ['relativeAgeCitationId', 'collectionMember'], + 'RelativeAgeCitation': ['absoluteAgeCitationId', 'collectionMember'], 'TectonicUnit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'], 'TectonicUnitTreeDefItem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'], 'TectonicUnitTreeDef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId'] diff --git a/specifyweb/specify/migration_utils/tectonic_ranks.py b/specifyweb/specify/migration_utils/tectonic_ranks.py index 1288236778a..c2afe88c6fa 100644 --- a/specifyweb/specify/migration_utils/tectonic_ranks.py +++ b/specifyweb/specify/migration_utils/tectonic_ranks.py @@ -6,13 +6,14 @@ def create_default_tectonic_ranks(apps): TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') Discipline = apps.get_model('specify', 'Discipline') - disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) + disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True).exclude( + id__in=TectonicTreeDef.objects.values_list('discipline_id', flat=True) + ) for discipline in disciplines: - tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create( - name="Tectonic Unit", - discipline=discipline, - ) + tectonic_tree_def = TectonicTreeDef.objects.filter(discipline=discipline).first() + if not tectonic_tree_def: + tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) root, _ = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", @@ -92,26 +93,16 @@ def create_root_tectonic_node(apps): tectonic_tree_def = TectonicUnitTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() if not tectonic_tree_def: - tectonic_tree_def, _ = TectonicUnitTreeDef.objects.get_or_create( + tectonic_tree_def, is_created = TectonicUnitTreeDef.objects.get_or_create( name="Tectonic Unit", discipline=discipline ) - tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter( - treedef=tectonic_tree_def, - name="Root", - ).first() - if tectonic_tree_def_item: - tectonic_tree_def_item.rankid = 0 - tectonic_tree_def_item.parent = None - tectonic_tree_def_item.isenforced = True - tectonic_tree_def_item.save() - else: - tectonic_tree_def_item, _ = TectonicUnitTreeDefItem.objects.get_or_create( + tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, name="Root").first() + if not tectonic_tree_def_item: + tectonic_tree_def_item, is_created = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", title="Root", - rankid=0, - parent=None, treedef=tectonic_tree_def, isenforced=True ) @@ -144,9 +135,7 @@ def revert_create_root_tectonic_node(apps, schema_editor=None): tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() if tectonic_tree_def: - TectonicUnit.objects.filter( - name="Root", - definition=tectonic_tree_def, - parent__isnull=True, - ).delete() TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() + TectonicUnit.objects.filter( + name="Root" + ).delete() \ No newline at end of file diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index bffe878184e..775299a8d07 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -121,20 +121,21 @@ def update_table_schema_config_with_defaults( issystem=table.system, version=0, ) + if not is_new: + return # If the container already exists, we don't need to update it - if is_new: - # Create a Splocaleitemstr for the table name and description - for k, text in { - "containername": camel_to_spaced_title_case(uncapitilize(table.name)), - "containerdesc": table_config.description, - }.items(): - item_str = { - "text": text, - "language": "en", - "version": 0, - } - item_str[k] = sp_local_container - Splocaleitemstr.objects.get_or_create(**item_str) + # Create a Splocaleitemstr for the table name and description + for k, text in { + "containername": camel_to_spaced_title_case(uncapitilize(table.name)), + "containerdesc": table_config.description, + }.items(): + item_str = { + "text": text, + "language": "en", + "version": 0, + } + item_str[k] = sp_local_container + Splocaleitemstr.objects.get_or_create(**item_str) for field in table.all_fields: update_table_field_schema_config_with_defaults(table_name, discipline_id, field.name, apps) @@ -469,7 +470,7 @@ def update_cog_type_fields(apps): container_items = Splocalecontaineritem.objects.filter( name="collectionObjectType", picklistname=None, - container__name="collectionobject", + container__name="CollectionObject", ) for container_item in container_items: Splocaleitemstr.objects.filter(itemname=container_item).delete() @@ -881,7 +882,7 @@ def update_schema_config_field_desc(apps, schema_editor=None): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name__iexact=field_name + name=field_name.lower() ) for item in items: @@ -962,7 +963,7 @@ def reverse_update_schema_config_field_desc(apps, schema_editor=None): for field_name, new_name, new_desc in fields: items = Splocalecontaineritem.objects.filter( container=container, - name__iexact=field_name + name=field_name.lower() ) for item in items: @@ -1026,7 +1027,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name__iexact=field_name + name=field_name.lower() ) for item in items: @@ -1102,7 +1103,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name__iexact=field_name + name=field_name.lower() ) for item in items: @@ -1321,7 +1322,7 @@ def update_0034_schema_config_field_desc(apps): for (field_name, new_name, new_desc) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name__iexact=field_name + name=field_name.lower() ) for item in items: item.ishidden = True @@ -1353,27 +1354,18 @@ def revert_0034_schema_config_field_desc(apps): """ Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') for table, fields in MIGRATION_0034_UPDATE_FIELDS.items(): containers = Splocalecontainer.objects.filter(name=table.lower()) for container in containers: - for (field_name, original_label, original_description) in fields: + for (field_name, _, _) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name__iexact=field_name + name=field_name.lower() ) for item in items: - item.ishidden = False - item.save() - desc_str = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() - name_str = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - if desc_str is not None: - desc_str.text = original_description - desc_str.save() - if name_str is not None: - name_str.text = original_label - name_str.save() + # If needed, reset ishidden or revert text + pass revert_0034_fields(apps) revert_0034_schema_config_field_desc(apps) diff --git a/specifyweb/specify/migrations/0002_geo.py b/specifyweb/specify/migrations/0002_geo.py index e84e59d5c2e..442fca149c4 100644 --- a/specifyweb/specify/migrations/0002_geo.py +++ b/specifyweb/specify/migrations/0002_geo.py @@ -16,6 +16,7 @@ create_default_discipline_for_tree_defs, set_discipline_for_taxon_treedefs, ) +from specifyweb.specify.api.utils import create_default_collection_types logger = logging.getLogger(__name__) diff --git a/specifyweb/specify/migrations/0008_ageCitations_fix.py b/specifyweb/specify/migrations/0008_ageCitations_fix.py index 9f058e2dbf3..479d9dae005 100644 --- a/specifyweb/specify/migrations/0008_ageCitations_fix.py +++ b/specifyweb/specify/migrations/0008_ageCitations_fix.py @@ -15,7 +15,7 @@ def apply_migration(apps, schema_editor): usc.update_relative_age_fields(apps) def revert_migration(apps, schema_editor): - usc.revert_relative_age_fields(apps) + usc.update_relative_age_fields(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0009_tectonic_ranks.py b/specifyweb/specify/migrations/0009_tectonic_ranks.py index f97c7560a19..54182060ebe 100644 --- a/specifyweb/specify/migrations/0009_tectonic_ranks.py +++ b/specifyweb/specify/migrations/0009_tectonic_ranks.py @@ -18,8 +18,8 @@ def consolidated_python_django_migration_operations(apps, schema_editor): create_root_tectonic_node(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_create_root_tectonic_node(apps, schema_editor) revert_default_tectonic_ranks(apps, schema_editor) + revert_create_root_tectonic_node(apps, schema_editor) operations = [ migrations.RunPython( diff --git a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py index 26a1a2c4c25..26dd616c1de 100644 --- a/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py +++ b/specifyweb/specify/migrations/0021_update_hidden_geo_tables.py @@ -4,6 +4,54 @@ from django.db import migrations from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.sp7_schemaconfig import MIGRATION_0021_FIELDS as SCHEMA_CONFIG_MOD_TABLE_FIELDS +from specifyweb.specify.models_utils.model_extras import GEOLOGY_DISCIPLINES, PALEO_DISCIPLINES + +# def fix_hidden_geo_prop(apps, schema_editor): +# Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') +# Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') +# Discipline = apps.get_model('specify', 'Discipline') + +# excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + +# filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + +# for discipline in filtered_disciplines: +# for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): +# containers = Splocalecontainer.objects.filter( +# name=table.lower(), +# discipline_id=discipline.id, +# ) +# for container in containers: +# for field_name in fields: +# items = Splocalecontaineritem.objects.filter( +# container=container, +# name=field_name.lower() +# ) +# items.update(ishidden=True) + +# def reverse_fix_hidden_geo_prop(apps, schema_editor): +# Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') +# Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') +# Discipline = apps.get_model('specify', 'Discipline') + +# excluded_disciplines = PALEO_DISCIPLINES | GEOLOGY_DISCIPLINES + +# filtered_disciplines = Discipline.objects.exclude(type__in=excluded_disciplines) + +# for discipline in filtered_disciplines: +# for table, fields in SCHEMA_CONFIG_MOD_TABLE_FIELDS.items(): +# containers = Splocalecontainer.objects.filter( +# name=table.lower(), +# discipline_id=discipline.id, +# ) +# for container in containers: +# for field_name in fields: +# items = Splocalecontaineritem.objects.filter( +# container=container, +# name=field_name.lower() +# ) +# items.update(ishidden=True) class Migration(migrations.Migration): dependencies = [ @@ -13,3 +61,4 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(usc.fix_hidden_geo_prop, usc.reverse_fix_hidden_geo_prop, atomic=True) ] +# migrations.RunPython(fix_hidden_geo_prop, reverse_fix_hidden_geo_prop, atomic=True) diff --git a/specifyweb/specify/migrations/0027_CO_children.py b/specifyweb/specify/migrations/0027_CO_children.py index f7f5f8c18a7..0dc7ccadad1 100644 --- a/specifyweb/specify/migrations/0027_CO_children.py +++ b/specifyweb/specify/migrations/0027_CO_children.py @@ -1,162 +1,8 @@ # Generated by Django 3.2.15 on 2025-04-11 15:35 -import re - -from django.core.exceptions import MultipleObjectsReturned +from django.apps import apps as specify_apps from django.db import migrations, models -from django.db.models import Q import django.db.models.deletion - -from specifyweb.specify.models import datamodel -from specifyweb.specify.models_utils.load_datamodel import FieldDoesNotExistError - - -MIGRATION_0027_FIELDS = { - 'CollectionObject': ['parentCO', 'children'], -} - -MIGRATION_0027_UPDATE_FIELDS = { - 'CollectionObject': [ - ('parentCO', 'Parent Collection Object', 'Parent CollectionObject'), - ('children', 'Children', 'Children'), - ] -} - -HIDDEN_FIELDS = [ - "timestampcreated", "timestampmodified", "version", "createdbyagent", "modifiedbyagent" -] - - -def datamodel_type_to_schematype(datamodel_type: str) -> str: - return "".join(map(lambda type_part: type_part.lower().capitalize(), datamodel_type.split('-'))) - - -def camel_to_spaced_title_case(camel_case: str) -> str: - return re.sub(r"(? Date: Wed, 20 May 2026 16:00:51 -0500 Subject: [PATCH 06/17] Mark existing catalog number rules as database constraints --- specifyweb/backend/businessrules/migration_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specifyweb/backend/businessrules/migration_utils.py b/specifyweb/backend/businessrules/migration_utils.py index 38265a6b20c..f5d689b9853 100644 --- a/specifyweb/backend/businessrules/migration_utils.py +++ b/specifyweb/backend/businessrules/migration_utils.py @@ -1,4 +1,4 @@ -from typing import Tuple, List +from typing import List from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule @@ -47,6 +47,7 @@ def catnum_rule_uneditable(apps, schema_editor=None): model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", discipline_id=discipline.id, isDatabaseConstraint=False) has_catalognumber_rule = False + matching_rule_ids: List[int] = [] for rule in model_rules: rule_fields = rule.uniquenessrulefield_set.all() @@ -59,8 +60,11 @@ def catnum_rule_uneditable(apps, schema_editor=None): # exception if more than one result is returned if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): has_catalognumber_rule = True + matching_rule_ids.append(rule.id) - if not has_catalognumber_rule: + if has_catalognumber_rule: + UniquenessRule.objects.filter(id__in=matching_rule_ids).update(isDatabaseConstraint=True) + else: create_uniqueness_rule( "Collectionobject", discipline=discipline, From 150237d2212479e172364c6187cfa50a8e21d8bd Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:51 -0500 Subject: [PATCH 07/17] Match uniqueness rules by exact field signatures --- .../backend/businessrules/uniqueness_rules.py | 91 +++++++++---------- 1 file changed, 43 insertions(+), 48 deletions(-) diff --git a/specifyweb/backend/businessrules/uniqueness_rules.py b/specifyweb/backend/businessrules/uniqueness_rules.py index fb3f055dbce..dec36a13a7e 100644 --- a/specifyweb/backend/businessrules/uniqueness_rules.py +++ b/specifyweb/backend/businessrules/uniqueness_rules.py @@ -7,7 +7,7 @@ from django.apps import apps from django.db import connections, transaction -from django.db.models import Q, Count, Exists, OuterRef +from django.db.models import Q, Count from django.db.migrations.recorder import MigrationRecorder from django.core.exceptions import ObjectDoesNotExist from specifyweb.specify.api.crud import get_model @@ -266,13 +266,19 @@ def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f candidate_rules = UniquenessRule.objects.filter(modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) + expected_fields = set(fields) + expected_scopes = set(scopes) for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() - matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) - matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) + existing_fields = set( + all_fields.filter(isScope=False).values_list("fieldPath", flat=True) + ) + existing_scopes = set( + all_fields.filter(isScope=True).values_list("fieldPath", flat=True) + ) # If the rule already exists, skip creating the rule - if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): + if existing_fields == expected_fields and existing_scopes == expected_scopes: return logger.info(f"Creating uniqueness rule on {model_name} with fields {fields} and scopes {scopes} for the discipline {discipline.name if discipline else 'Global'}") @@ -295,13 +301,19 @@ def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f candidate_rules = UniquenessRule.objects.filter( modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline) + expected_fields = set(fields) + expected_scopes = set(scopes) rule_ids = [] for rule in candidate_rules: all_fields = rule.uniquenessrulefield_set.all() - matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False) - matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True) + existing_fields = set( + all_fields.filter(isScope=False).values_list("fieldPath", flat=True) + ) + existing_scopes = set( + all_fields.filter(isScope=True).values_list("fieldPath", flat=True) + ) # If the rule exists, add it to the list of rules to be deleted - if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): + if existing_fields == expected_fields and existing_scopes == expected_scopes: rule_ids.append(rule.id) UniquenessRuleField.objects.filter( @@ -323,24 +335,19 @@ def fix_global_default_rules(registry=None): UniquenessRule = registry.get_model('businessrules', 'UniquenessRule') \ if registry \ else models.UniquenessRule - UniquenessRuleField = registry.get_model('businessrules', 'UniquenessRuleField') \ - if registry \ - else models.UniquenessRuleField - - global_rule_fields = UniquenessRuleField.objects.filter( - uniquenessrule__discipline__isnull=True - ).values( - "uniquenessrule__modelName", - "uniquenessrule__isDatabaseConstraint", - "fieldPath", - "isScope", - ) - global_rule_exists = UniquenessRule.objects.filter( - discipline__isnull=True, - modelName=OuterRef("modelName"), - isDatabaseConstraint=OuterRef("isDatabaseConstraint"), - ) + global_rule_signatures = { + ( + rule.modelName, + rule.isDatabaseConstraint, + frozenset( + rule.uniquenessrulefield_set.values_list("fieldPath", "isScope") + ), + ) + for rule in UniquenessRule.objects.filter( + discipline__isnull=True + ).prefetch_related("uniquenessrulefield_set") + } discipline_ids = ( UniquenessRule.objects.exclude(discipline__isnull=True) @@ -350,28 +357,16 @@ def fix_global_default_rules(registry=None): for discipline_id in discipline_ids: with transaction.atomic(): - # Delete matching fields for this discipline - matching_fields_qs = UniquenessRuleField.objects.filter( - uniquenessrule__discipline_id=discipline_id - ).filter( - Exists( - global_rule_fields.filter( - **{ - "uniquenessrule__modelName": OuterRef("uniquenessrule__modelName"), - "uniquenessrule__isDatabaseConstraint": OuterRef("uniquenessrule__isDatabaseConstraint"), - "fieldPath": OuterRef("fieldPath"), - "isScope": OuterRef("isScope"), - } - ) + for rule in UniquenessRule.objects.filter( + discipline_id=discipline_id + ).prefetch_related("uniquenessrulefield_set"): + signature = ( + rule.modelName, + rule.isDatabaseConstraint, + frozenset( + rule.uniquenessrulefield_set.values_list("fieldPath", "isScope") + ), ) - ) - matching_fields_qs.delete() - - # Delete UniquenessRule rows for this discipline that are now empty - empty_rules_qs = ( - UniquenessRule.objects.filter(discipline_id=discipline_id) - .annotate(field_count=Count("uniquenessrulefield")) - .filter(field_count=0) # now empty after field deletions - .filter(Exists(global_rule_exists)) - ) - empty_rules_qs.delete() + if signature in global_rule_signatures: + rule.uniquenessrulefield_set.all().delete() + rule.delete() From e31e8382c120be75fa65ed7cd4db4a6d77824edb Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:51 -0500 Subject: [PATCH 08/17] Use migration database alias in patch helpers --- specifyweb/backend/patches/migration_utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specifyweb/backend/patches/migration_utils.py b/specifyweb/backend/patches/migration_utils.py index a773cb10ad0..4dfdc463387 100644 --- a/specifyweb/backend/patches/migration_utils.py +++ b/specifyweb/backend/patches/migration_utils.py @@ -10,6 +10,7 @@ def apply_migrations(app_registry, schema_editor=None): update_coordinates(app_registry, schema_editor) def update_is_accepted(app_registry, schema_editor=None): + db_alias = schema_editor.connection.alias if schema_editor is not None else "default" for tree in SPECIFY_TREES: tree_filters = { "isaccepted": False, @@ -17,20 +18,21 @@ def update_is_accepted(app_registry, schema_editor=None): } tree_model = app_registry.get_model("specify", tree) - tree_model.objects.filter(**tree_filters).update(isaccepted=True) + tree_model._base_manager.using(db_alias).filter(**tree_filters).update(isaccepted=True) def update_coordinates(app_registry, schema_editor=None): + db_alias = schema_editor.connection.alias if schema_editor is not None else "default" Locality = app_registry.get_model("specify", "Locality") - Locality.objects.filter(lat1text__isnull=True, latitude1__isnull=False) \ + Locality._base_manager.using(db_alias).filter(lat1text__isnull=True, latitude1__isnull=False) \ .update(lat1text=F("latitude1")) - Locality.objects.filter(long1text__isnull=True, longitude1__isnull=False) \ + Locality._base_manager.using(db_alias).filter(long1text__isnull=True, longitude1__isnull=False) \ .update(long1text=F("longitude1")) - Locality.objects.filter(lat2text__isnull=True, latitude2__isnull=False) \ + Locality._base_manager.using(db_alias).filter(lat2text__isnull=True, latitude2__isnull=False) \ .update(lat2text=F("latitude2")) - Locality.objects.filter(long2text__isnull=True, longitude2__isnull=False) \ + Locality._base_manager.using(db_alias).filter(long2text__isnull=True, longitude2__isnull=False) \ .update(long2text=F("longitude2")) From 244f6cd5a70c79c714e4d28cafaa36323d9ebbbb Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:51 -0500 Subject: [PATCH 09/17] Backfill permissions per user and collection --- specifyweb/backend/permissions/initialize.py | 24 ++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/specifyweb/backend/permissions/initialize.py b/specifyweb/backend/permissions/initialize.py index fb8d981891c..97b650b19b2 100644 --- a/specifyweb/backend/permissions/initialize.py +++ b/specifyweb/backend/permissions/initialize.py @@ -47,13 +47,14 @@ def create_admins(apps=apps) -> None: UserPolicy = apps.get_model('permissions', 'UserPolicy') Specifyuser = apps.get_model('specify', 'Specifyuser') - if UserPolicy.objects.filter(collection__isnull=True, resource='%', action='%').exists(): - # don't do anything if there is already any admin. - return - users = Specifyuser.objects.all() for user in users: - if is_sp6_user_permissions_migrated(user, apps): + if UserPolicy.objects.filter( + collection__isnull=True, + specifyuser_id=user.id, + resource="%", + action="%", + ).exists(): continue if is_legacy_admin(user): UserPolicy.objects.get_or_create( @@ -97,17 +98,12 @@ def assign_users_to_roles(apps=apps) -> None: JOIN spprincipal p ON p.SpPrincipalID = up.SpPrincipalID JOIN collection c ON c.UserGroupScopeId = p.userGroupScopeID WHERE p.groupType IS NULL - AND u.SpecifyUserID NOT IN ( - SELECT ur.specifyuser_id - FROM spuserrole ur - JOIN sprole r ON r.id = ur.role_id - WHERE r.collection_id = p.usergroupscopeid - ) - AND c.UserGroupScopeId NOT IN ( - SELECT DISTINCT r.collection_id + AND NOT EXISTS ( + SELECT 1 FROM spuserrole ur JOIN sprole r ON r.id = ur.role_id - JOIN collection c ON c.UserGroupScopeId = r.collection_id + WHERE r.collection_id = c.UserGroupScopeId + AND ur.specifyuser_id = u.SpecifyUserID ); """) From 6999a4d641bfd0bb3a98c74ebd4f528d90b66ec2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 10/17] Gate stored query SQL debug logging --- specifyweb/backend/stored_queries/execution.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specifyweb/backend/stored_queries/execution.py b/specifyweb/backend/stored_queries/execution.py index 14121c0d676..eee870dd139 100644 --- a/specifyweb/backend/stored_queries/execution.py +++ b/specifyweb/backend/stored_queries/execution.py @@ -852,7 +852,8 @@ def execute( - log_sqlalchemy_query(query) # Debugging + if settings.DEBUG: + log_sqlalchemy_query(query) return {"results": apply_special_post_query_processing(query, tableid, field_specs, collection, user)} def build_query( @@ -1059,7 +1060,8 @@ def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" - log_sqlalchemy_query(query) # Debugging + if settings.DEBUG: + log_sqlalchemy_query(query) def parse_catalog_for_comparing(s): def check_for_decimal(s): @@ -1184,4 +1186,4 @@ def apply_special_post_query_processing(query, tableid, field_specs, collection, if should_list_query: return list(query) - return query \ No newline at end of file + return query From 056f2116c622e22c8f6818924e5c34cd582431a0 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 11/17] Gate geology time SQL debug logging --- specifyweb/backend/stored_queries/geology_time.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specifyweb/backend/stored_queries/geology_time.py b/specifyweb/backend/stored_queries/geology_time.py index 0a1a4ffe9d8..8946241b0f2 100644 --- a/specifyweb/backend/stored_queries/geology_time.py +++ b/specifyweb/backend/stored_queries/geology_time.py @@ -1,5 +1,6 @@ import logging import os +from django.conf import settings from django.db.models import Case, FloatField, F, Q, Value, When from django.db.models.functions import Coalesce, Greatest, Least, Cast from specifyweb.backend.stored_queries.utils import log_sqlalchemy_query @@ -977,5 +978,6 @@ def modify_query_add_meta_age_range(query, start_time, end_time, require_full_ov ).label("age") new_query = new_query.add_columns(age_expr) - log_sqlalchemy_query(new_query) + if settings.DEBUG: + log_sqlalchemy_query(new_query) return new_query From 34806661eac9fbcbff2a90d0e142c1886409a2a8 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 12/17] Tighten key migration command queries and logging --- .../commands/run_key_migration_functions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index 9aeeb3c8320..ae767208d27 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -4,6 +4,7 @@ from django.core.management.base import BaseCommand from django.apps import apps from django.db import transaction +from django.db.models import Subquery from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable from specifyweb.backend.businessrules.uniqueness_rules import ( apply_default_uniqueness_rules, @@ -81,8 +82,12 @@ def apply_default_uniqueness_rules_to_disciplines(apps): Discipline = apps.get_model('specify', 'Discipline') UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') + disciplines_with_rules = UniquenessRule.objects.exclude( + discipline_id__isnull=True + ).values("discipline_id").distinct() for discipline in Discipline.objects.exclude( - id__in=set(UniquenessRule.objects.values_list('discipline_id', flat=True).distinct())): + id__in=Subquery(disciplines_with_rules) + ): apply_default_uniqueness_rules(discipline, registry=apps) @@ -138,7 +143,7 @@ def add_arguments(self, parser): nargs="*", type=str, choices=tuple(self.funcs.keys()), - help=f"Optional: specify one or more functions to run", + help="Optional: specify one or more functions to run", ) parser.add_argument( "--verbose", @@ -171,6 +176,6 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS(f"Applying {func_name}...")) func(self.stdout.write if verbose else None) self.stdout.write(self.style.SUCCESS(f"Applied {func_name}")) - except Exception as e: - logger.error(f"An error occurred: {e}") + except Exception: + logger.exception("An error occurred while running key migrations") raise From 5b8d98b1a80f9c2dce4edfd92d04934b56fd7bab Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 13/17] Pair tectonic tree defs with disciplines one-to-one --- .../specify/migration_utils/default_cots.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/specifyweb/specify/migration_utils/default_cots.py b/specifyweb/specify/migration_utils/default_cots.py index e9387b5ca8e..1b2c916131c 100644 --- a/specifyweb/specify/migration_utils/default_cots.py +++ b/specifyweb/specify/migration_utils/default_cots.py @@ -135,22 +135,25 @@ def fix_tectonic_unit_treedef_discipline_links(apps): Discipline = apps.get_model('specify', 'Discipline') Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef') - empty_tectonic_unit_treedefs = Tectonicunittreedef.objects.filter(discipline__isnull=True) - empty_disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) - for empty_discipline in empty_disciplines: - if not empty_tectonic_unit_treedefs.exists(): - new_tectonic_unit_treedef = Tectonicunittreedef.objects.create( - name=f'{empty_discipline.name} Tectonic Unit Tree', - discipline=empty_discipline - ) - else: - empty_discipline.tectonicunittreedef = empty_tectonic_unit_treedefs.first() - empty_discipline.save() - - for empty_tectonic_unit_treedef in empty_tectonic_unit_treedefs: - if empty_disciplines.exists(): - empty_tectonic_unit_treedef.discipline = empty_disciplines.first() - empty_tectonic_unit_treedef.save() - else: - empty_tectonic_unit_treedef.discipline = empty_disciplines.last() - empty_tectonic_unit_treedef.save() + empty_tectonic_unit_treedefs = list( + Tectonicunittreedef.objects.filter(discipline__isnull=True) + ) + empty_disciplines = list( + Discipline.objects.filter(tectonicunittreedef__isnull=True) + ) + + for discipline, tectonic_unit_treedef in zip( + empty_disciplines, empty_tectonic_unit_treedefs + ): + tectonic_unit_treedef.discipline = discipline + tectonic_unit_treedef.save() + discipline.tectonicunittreedef = tectonic_unit_treedef + discipline.save() + + for discipline in empty_disciplines[len(empty_tectonic_unit_treedefs):]: + tectonic_unit_treedef = Tectonicunittreedef.objects.create( + name=f'{discipline.name} Tectonic Unit Tree', + discipline=discipline + ) + discipline.tectonicunittreedef = tectonic_unit_treedef + discipline.save() From 6231791f315909272fa28bd2a22ce7e8b22dd2f9 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 14/17] Fix RelativeAgeCitation schema field id --- specifyweb/specify/migration_utils/sp7_schemaconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/specify/migration_utils/sp7_schemaconfig.py b/specifyweb/specify/migration_utils/sp7_schemaconfig.py index d870be61399..81d28bca732 100644 --- a/specifyweb/specify/migration_utils/sp7_schemaconfig.py +++ b/specifyweb/specify/migration_utils/sp7_schemaconfig.py @@ -148,7 +148,7 @@ 'RelativeAge': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'], 'CollectionObject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'], 'AbsoluteAgeCitation': ['collectionMember', 'absoluteAgeCitationId'], - 'RelativeAgeCitation': ['absoluteAgeCitationId', 'collectionMember'], + 'RelativeAgeCitation': ['relativeAgeCitationId', 'collectionMember'], 'TectonicUnit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'], 'TectonicUnitTreeDefItem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'], 'TectonicUnitTreeDef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId'] From 5c839d1d028efacdd80418246e0f4ceffbbb8363 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 15/17] Repair tectonic rank root handling --- .../specify/migration_utils/tectonic_ranks.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/specifyweb/specify/migration_utils/tectonic_ranks.py b/specifyweb/specify/migration_utils/tectonic_ranks.py index c2afe88c6fa..ac3c9558125 100644 --- a/specifyweb/specify/migration_utils/tectonic_ranks.py +++ b/specifyweb/specify/migration_utils/tectonic_ranks.py @@ -6,14 +6,13 @@ def create_default_tectonic_ranks(apps): TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') Discipline = apps.get_model('specify', 'Discipline') - disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True).exclude( - id__in=TectonicTreeDef.objects.values_list('discipline_id', flat=True) - ) + disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) for discipline in disciplines: - tectonic_tree_def = TectonicTreeDef.objects.filter(discipline=discipline).first() - if not tectonic_tree_def: - tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) + tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create( + name="Tectonic Unit", + discipline=discipline, + ) root, _ = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", @@ -99,10 +98,17 @@ def create_root_tectonic_node(apps): ) tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, name="Root").first() - if not tectonic_tree_def_item: + if tectonic_tree_def_item: + tectonic_tree_def_item.rankid = 0 + tectonic_tree_def_item.parent = None + tectonic_tree_def_item.isenforced = True + tectonic_tree_def_item.save() + else: tectonic_tree_def_item, is_created = TectonicUnitTreeDefItem.objects.get_or_create( name="Root", title="Root", + rankid=0, + parent=None, treedef=tectonic_tree_def, isenforced=True ) @@ -135,7 +141,9 @@ def revert_create_root_tectonic_node(apps, schema_editor=None): tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() if tectonic_tree_def: - TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() TectonicUnit.objects.filter( - name="Root" - ).delete() \ No newline at end of file + name="Root", + definition=tectonic_tree_def, + parent__isnull=True, + ).delete() + TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() From 59c88e5551a35e619ac5df840784961517e8cb3e Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:00:52 -0500 Subject: [PATCH 16/17] Make schema config backfills exact and reversible --- .../migration_utils/update_schema_config.py | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index 775299a8d07..d46a1798f4d 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -121,21 +121,20 @@ def update_table_schema_config_with_defaults( issystem=table.system, version=0, ) - if not is_new: - return # If the container already exists, we don't need to update it - # Create a Splocaleitemstr for the table name and description - for k, text in { - "containername": camel_to_spaced_title_case(uncapitilize(table.name)), - "containerdesc": table_config.description, - }.items(): - item_str = { - "text": text, - "language": "en", - "version": 0, - } - item_str[k] = sp_local_container - Splocaleitemstr.objects.get_or_create(**item_str) + if is_new: + # Create a Splocaleitemstr for the table name and description + for k, text in { + "containername": camel_to_spaced_title_case(uncapitilize(table.name)), + "containerdesc": table_config.description, + }.items(): + item_str = { + "text": text, + "language": "en", + "version": 0, + } + item_str[k] = sp_local_container + Splocaleitemstr.objects.get_or_create(**item_str) for field in table.all_fields: update_table_field_schema_config_with_defaults(table_name, discipline_id, field.name, apps) @@ -470,7 +469,7 @@ def update_cog_type_fields(apps): container_items = Splocalecontaineritem.objects.filter( name="collectionObjectType", picklistname=None, - container__name="CollectionObject", + container__name="collectionobject", ) for container_item in container_items: Splocaleitemstr.objects.filter(itemname=container_item).delete() @@ -882,7 +881,7 @@ def update_schema_config_field_desc(apps, schema_editor=None): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -963,7 +962,7 @@ def reverse_update_schema_config_field_desc(apps, schema_editor=None): for field_name, new_name, new_desc in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1027,7 +1026,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1103,7 +1102,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1136,7 +1135,7 @@ def hide_co_component(apps): for field_name, _, _ in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1189,7 +1188,7 @@ def reverse_hide_co_component(apps): for field_name, _, _ in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1229,7 +1228,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1322,7 +1321,7 @@ def update_0034_schema_config_field_desc(apps): for (field_name, new_name, new_desc) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: item.ishidden = True @@ -1354,18 +1353,27 @@ def revert_0034_schema_config_field_desc(apps): """ Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') for table, fields in MIGRATION_0034_UPDATE_FIELDS.items(): containers = Splocalecontainer.objects.filter(name=table.lower()) for container in containers: - for (field_name, _, _) in fields: + for (field_name, original_label, original_description) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: - # If needed, reset ishidden or revert text - pass + item.ishidden = False + item.save() + desc_str = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() + name_str = Splocaleitemstr.objects.filter(itemname_id=item.id).first() + if desc_str is not None: + desc_str.text = original_description + desc_str.save() + if name_str is not None: + name_str.text = original_label + name_str.save() revert_0034_fields(apps) revert_0034_schema_config_field_desc(apps) From 1a57b823cb242197bb7e57c135eca51c4efe02cc Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 20 May 2026 16:01:28 -0500 Subject: [PATCH 17/17] Avoid literal-bound SQL in debug logs --- specifyweb/specify/api/utils.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/specifyweb/specify/api/utils.py b/specifyweb/specify/api/utils.py index d9aeb22c5d3..ad4d0afc8be 100644 --- a/specifyweb/specify/api/utils.py +++ b/specifyweb/specify/api/utils.py @@ -1,5 +1,7 @@ import logging +from django.conf import settings + from specifyweb.specify import models as spmodels from specifyweb.backend.businessrules.exceptions import BusinessRuleException @@ -19,15 +21,17 @@ def get_spmodel_class(model_name: str): raise AttributeError(f"Model '{model_name}' not found in models module.") def log_sqlalchemy_query(query): - # Call this function to debug the raw SQL query generated by SQLAlchemy + if not settings.DEBUG: + return + from sqlalchemy.dialects import mysql - compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) + + compiled_query = query.statement.compile(dialect=mysql.dialect()) raw_sql = str(compiled_query).replace('\n', ' ') + ';' - logger.debug('='.join(['' for _ in range(80)])) - logger.debug(raw_sql) - logger.debug('='.join(['' for _ in range(80)])) - # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 - # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) + logger.debug("%s", "=" * 80) + logger.debug("SQL: %s", raw_sql) + logger.debug("Params: %s", compiled_query.params) + logger.debug("%s", "=" * 80) def create_default_collection_types(apps): Collection = apps.get_model('specify', 'Collection') @@ -82,4 +86,4 @@ def get_picklists(collection: spmodels.Collection, tablename: str, fieldname: st if len(collection_picklists) > 0: picklists = collection_picklists - return picklists, schemaitem \ No newline at end of file + return picklists, schemaitem