From 23a7413315720b76bc064058d919178037476654 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 13 Feb 2026 16:10:46 -0600 Subject: [PATCH 1/3] Fix schema override --- .../commands/run_key_migration_functions.py | 11 + .../migration_utils/update_schema_config.py | 211 ++++++++++++++++-- 2 files changed, 206 insertions(+), 16 deletions(-) diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index fd3d116a75c..ea44bbf02a6 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -24,6 +24,7 @@ from specifyweb.specify.migration_utils.misc_migrations import make_selectseries_false from specifyweb.specify.migration_utils.tectonic_ranks import create_default_tectonic_ranks, create_root_tectonic_node from specifyweb.backend.patches.migration_utils import apply_migrations as apply_patches +from specifyweb.backend.setup_tool.schema_defaults import apply_schema_defaults logger = logging.getLogger(__name__) @@ -48,6 +49,15 @@ def fix_cots(stdout: WriteToStdOut | None = None): log_and_run(funcs, stdout) def fix_schema_config(stdout: WriteToStdOut | None = None): + def apply_schema_overrides_for_all_disciplines(_apps): + Discipline = _apps.get_model('specify', 'Discipline') + for discipline in Discipline.objects.all(): + if stdout is not None: + stdout( + f"Applying schema defaults/overrides for discipline {discipline.id} ({discipline.type})..." + ) + apply_schema_defaults(discipline) + funcs = [ # usc.update_all_table_schema_config_with_defaults, usc.create_geo_table_schema_config_with_defaults, # specify 0002 @@ -80,6 +90,7 @@ def fix_schema_config(stdout: WriteToStdOut | None = None): usc.componets_schema_config_migrations, # specify 0040 usc.create_discipline_type_picklist, # specify 0042 usc.update_discipline_type_splocalecontaineritem, # specify 0042 + apply_schema_overrides_for_all_disciplines, usc.deduplicate_schema_config_orm, ] log_and_run(funcs, stdout) diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index dad859b5722..bf0e271dacf 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -1,11 +1,15 @@ import re +import json from typing import NamedTuple, Tuple import logging from collections import defaultdict +from functools import lru_cache +from pathlib import Path from django.db.models import Q, Count, Window, F +from django.conf import settings from django.apps import apps as global_apps from django.core.exceptions import MultipleObjectsReturned from django.db import connection, transaction @@ -54,6 +58,94 @@ "timestampcreated", "timestampmodified", "version", "createdbyagent", "modifiedbyagent" ] +def _has_explicit_hidden_override(field_config: dict) -> bool: + return any(key.lower() == "ishidden" for key in field_config.keys()) + +@lru_cache(maxsize=None) +def _schema_override_hidden_values_for_discipline( + discipline_type: str, +) -> dict[str, dict[str, bool]]: + """ + Return a mapping of {table_name -> {field_name -> ishidden_value}} for fields + that have an + explicit `ishidden` override in config//schema_overrides.json. + """ + normalized_discipline = (discipline_type or "").lower() + if not normalized_discipline: + return {} + + schema_overrides_path = ( + Path(settings.SPECIFY_CONFIG_DIR) / normalized_discipline / "schema_overrides.json" + ) + if not schema_overrides_path.exists(): + return {} + + try: + with schema_overrides_path.open("r", encoding="utf-8") as schema_overrides_file: + overrides = json.load(schema_overrides_file) + except (OSError, json.JSONDecodeError) as exc: + logger.warning( + "Unable to read schema overrides for discipline '%s' at %s: %s", + normalized_discipline, + schema_overrides_path, + exc, + ) + return {} + + if not isinstance(overrides, dict): + return {} + + hidden_override_values_by_table: dict[str, dict[str, bool]] = {} + for table_name, table_config in overrides.items(): + if not isinstance(table_config, dict): + continue + + explicit_hidden_override_values: dict[str, bool] = {} + items = table_config.get("items", []) + if not isinstance(items, list): + continue + + for item in items: + if not isinstance(item, dict): + continue + + for field_name, field_config in item.items(): + if not isinstance(field_config, dict): + continue + if not _has_explicit_hidden_override(field_config): + continue + for key, value in field_config.items(): + if key.lower() == "ishidden": + explicit_hidden_override_values[field_name.lower()] = bool(value) + break + + if explicit_hidden_override_values: + hidden_override_values_by_table[table_name.lower()] = explicit_hidden_override_values + + return hidden_override_values_by_table + +@lru_cache(maxsize=None) +def _schema_override_hidden_fields_for_discipline(discipline_type: str) -> dict[str, set[str]]: + hidden_override_values = _schema_override_hidden_values_for_discipline(discipline_type) + return { + table_name: set(table_values.keys()) + for table_name, table_values in hidden_override_values.items() + } + +def _fields_without_explicit_hidden_override( + table_name: str, + field_names: list[str], + discipline_type: str, +) -> list[str]: + table_hidden_overrides = _schema_override_hidden_fields_for_discipline( + discipline_type + ).get(table_name.lower(), set()) + return [ + field_name + for field_name in field_names + if field_name.lower() not in table_hidden_overrides + ] + def datamodel_type_to_schematype(datamodel_type: str) -> str: """ Converts a string like `many-to-one` to `ManyToOne` by: @@ -110,30 +202,60 @@ def bulk_create_splocaleitemstr_idempotent(Splocaleitemstr, rows: list[dict]) -> for fk_field, group_rows in groups.items(): fk_ids: set[int] = set() - texts: set[str] = set() languages: set[str] = set() for r in group_rows: fk_ids.add(r[fk_field].pk) - texts.add(r["text"]) languages.add(r["language"]) - filt = { - f"{fk_field}_id__in": fk_ids, - "text__in": texts, - "language__in": languages, - } - existing = set( - Splocaleitemstr.objects.filter(**filt).values_list("language", "text", f"{fk_field}_id") + existing_rows = list( + Splocaleitemstr.objects.filter( + **{ + f"{fk_field}_id__in": fk_ids, + "language__in": languages, + } + ) + .filter( + Q(country__isnull=True) | Q(country=""), + Q(variant__isnull=True) | Q(variant=""), + ) + .order_by("id") ) - to_create = [] + existing_by_key: dict[Tuple[str, int], list] = defaultdict(list) + fk_field_id = f"{fk_field}_id" + for existing_row in existing_rows: + key = (existing_row.language, getattr(existing_row, fk_field_id)) + existing_by_key[key].append(existing_row) + + desired_by_key: dict[Tuple[str, int], dict] = {} for r in group_rows: - key: Tuple[str, str, int] = (r["language"], r["text"], r[fk_field].pk) - if key in existing: + key = (r["language"], r[fk_field].pk) + desired_by_key[key] = r + + rows_to_update = [] + ids_to_delete: set[int] = set() + to_create = [] + for key, desired_row in desired_by_key.items(): + existing_for_key = existing_by_key.get(key, []) + + if not existing_for_key: + to_create.append(Splocaleitemstr(**desired_row)) continue - existing.add(key) - to_create.append(Splocaleitemstr(**r)) + + keeper = existing_for_key[0] + if keeper.text != desired_row["text"]: + keeper.text = desired_row["text"] + rows_to_update.append(keeper) + + for duplicate in existing_for_key[1:]: + ids_to_delete.add(duplicate.id) + + if ids_to_delete: + Splocaleitemstr.objects.filter(id__in=ids_to_delete).delete() + + if rows_to_update: + Splocaleitemstr.objects.bulk_update(rows_to_update, ["text"]) if to_create: Splocaleitemstr.objects.bulk_create(to_create) @@ -1103,17 +1225,62 @@ def update_hidden_prop(apps, schema_editor=None): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + Discipline = apps.get_model('specify', 'Discipline') + discipline_types_by_id = dict(Discipline.objects.values_list("id", "type")) for table, fields in MIGRATION_0023_FIELDS_BIS.items(): + field_names = [field_name.lower() for field_name in fields] + field_name_set = set(field_names) containers = Splocalecontainer.objects.filter( name=table.lower(), schematype=0 ) for container in containers: + discipline_type = discipline_types_by_id.get(container.discipline_id, "") + explicit_hidden_overrides = { + field_name: ishidden + for field_name, ishidden in _schema_override_hidden_values_for_discipline( + discipline_type + ).get(table.lower(), {}).items() + if field_name in field_name_set + } + explicit_fields_to_hide = [ + field_name + for field_name, ishidden in explicit_hidden_overrides.items() + if ishidden + ] + explicit_fields_to_show = [ + field_name + for field_name, ishidden in explicit_hidden_overrides.items() + if not ishidden + ] + + if explicit_fields_to_hide: + Splocalecontaineritem.objects.filter( + container=container, + ishidden=False, + name__in=explicit_fields_to_hide, + ).update(ishidden=True) + + if explicit_fields_to_show: + Splocalecontaineritem.objects.filter( + container=container, + ishidden=True, + name__in=explicit_fields_to_show, + ).update(ishidden=False) + + fields_to_hide = _fields_without_explicit_hidden_override( + table, + field_names, + discipline_type, + ) + if not fields_to_hide: + continue + items_updated = Splocalecontaineritem.objects.filter( container=container, ishidden=False, - name__in=[field_name.lower() for field_name in fields] + name__in=fields_to_hide ).update(ishidden=True) if items_updated > 0: logger.info(f"Hid {items_updated} items for table {table} and container {container.id}") @@ -1137,15 +1304,27 @@ def update_hidden_prop(apps, schema_editor=None): def reverse_update_hidden_prop(apps, schema_editor=None): Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + Discipline = apps.get_model('specify', 'Discipline') + discipline_types_by_id = dict(Discipline.objects.values_list("id", "type")) for table, fields in MIGRATION_0023_FIELDS_BIS.items(): + field_names = [field_name.lower() for field_name in fields] containers = Splocalecontainer.objects.filter( name=table.lower(), ) for container in containers: + discipline_type = discipline_types_by_id.get(container.discipline_id, "") + fields_to_unhide = _fields_without_explicit_hidden_override( + table, + field_names, + discipline_type, + ) + if not fields_to_unhide: + continue + items = Splocalecontaineritem.objects.filter( container=container, - name__in=[field_name.lower() for field_name in fields] + name__in=fields_to_unhide ) logger.info(f"Reverting {items.count()} items for table {table} and container {container.id}") items.update(ishidden=False) From ae3383704158c68b98d39a906a8dd13f230ce2bf Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 13 Feb 2026 22:15:07 +0000 Subject: [PATCH 2/3] Lint code with ESLint and Prettier Triggered by 23a7413315720b76bc064058d919178037476654 on branch refs/heads/issue-7682-2 --- .../lib/components/SetupTool/SetupForm.tsx | 51 ++++++++++------ .../lib/components/TreeView/CreateTree.tsx | 59 +++++++++---------- 2 files changed, 62 insertions(+), 48 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx b/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx index 28c36601db6..7f7fb5dd4ec 100644 --- a/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx @@ -25,7 +25,12 @@ import type { } from '../TreeView/CreateTree'; import { PopulatedTreeList } from '../TreeView/CreateTree'; import type { FieldConfig, ResourceConfig } from './setupResources'; -import { disciplineTypeOptions, FIELD_MAX_LENGTH, resources,stepOrder } from './setupResources'; +import { + disciplineTypeOptions, + FIELD_MAX_LENGTH, + resources, + stepOrder, +} from './setupResources'; import type { ResourceFormData } from './types'; function getFormValue( @@ -142,15 +147,20 @@ export function renderFormFieldFactory({ const verticalSpacing = width !== undefined && width < 2 ? '-mb-2' : 'mb-2'; - const disciplineTypeValue = getFormValue(formData, stepOrder.indexOf('discipline'), 'type'); + const disciplineTypeValue = getFormValue( + formData, + stepOrder.indexOf('discipline'), + 'type' + ); const isDisciplineNameDisabled = resources[currentStep].resourceName === 'discipline' && fieldName === 'name' && (disciplineTypeValue === undefined || disciplineTypeValue === ''); const isRowEnableToggle = name === 'include'; - const isRowEnabled = - Boolean(getFormValue(formData, currentStep, `${parentName}.include`)); + const isRowEnabled = Boolean( + getFormValue(formData, currentStep, `${parentName}.include`) + ); const taxonTreeAvailable = Array.isArray(treeOptions) && @@ -177,9 +187,7 @@ export function renderFormFieldFactory({ getFormValue(formData, currentStep, fieldName) )} disabled={ - inTable - ? !isRowEnabled && !isRowEnableToggle - : false + inTable ? !isRowEnabled && !isRowEnableToggle : false } id={fieldName} name={fieldName} @@ -325,7 +333,7 @@ export function renderFormFieldFactory({ { handleChange(fieldName, resource); handleChange('preload', true); @@ -351,10 +359,14 @@ export function renderFormFieldFactory({ value={getFormValue(formData, currentStep, fieldName) ?? ''} onChange={({ target }) => { // Only allow unique discipline names - if (resources[currentStep].resourceName === 'discipline' && fieldName === 'name') { + if ( + resources[currentStep].resourceName === 'discipline' && + fieldName === 'name' + ) { const value = (target.value ?? '').trim(); - const existingDisciplines = institutionData.children.flatMap(division => - division.children.map(discipline => discipline.name) + const existingDisciplines = institutionData.children.flatMap( + (division) => + division.children.map((discipline) => discipline.name) ); const isUsed = existingDisciplines.includes(value); target.setCustomValidity( @@ -441,7 +453,7 @@ export function updateSetupFormData( name: string, newValue: LocalizedString | TaxonFileDefaultDefinition | boolean, currentStep: number, - institutionData?: InstitutionData, + institutionData?: InstitutionData ): void { setFormData((previous: ResourceFormData) => { const resourceName = resources[currentStep].resourceName; @@ -457,10 +469,15 @@ export function updateSetupFormData( (option) => option.value === newValue ); if (matchingType) { - const existingDisciplines = institutionData ? institutionData.children.flatMap(division => - division.children.map(discipline => discipline.name) - ) : []; - const disciplineName = getUniqueName(matchingType.label, existingDisciplines); + const existingDisciplines = institutionData + ? institutionData.children.flatMap((division) => + division.children.map((discipline) => discipline.name) + ) + : []; + const disciplineName = getUniqueName( + matchingType.label, + existingDisciplines + ); updates.name = matchingType ? disciplineName : ''; } @@ -485,4 +502,4 @@ export function updateSetupFormData( [resourceName]: updates, }; }); -} \ No newline at end of file +} diff --git a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx index 2f5541a5759..370dd5af43d 100644 --- a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx +++ b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx @@ -204,8 +204,8 @@ export function ImportTree({ readonly tableName: SCHEMA['tableName']; readonly treeDefId: number; readonly treeDefinitionItems: RA< - SerializedResource> - >; + SerializedResource> + >; readonly buttonLabel?: LocalizedString; readonly buttonClassName?: string; }): JSX.Element { @@ -217,9 +217,14 @@ export function ImportTree({ string | undefined >(undefined); - const [missingTreeRanks, setMissingTreeRanks] = React.useState | undefined>(undefined); - const [isMissingTreeRanks, setIsMissingTreeRanks] = React.useState(false); - const [selectedPopulatedTree, setSelectedPopulatedTree] = React.useState(undefined); + const [missingTreeRanks, setMissingTreeRanks] = React.useState< + RA | undefined + >(undefined); + const [isMissingTreeRanks, setIsMissingTreeRanks] = + React.useState(false); + const [selectedPopulatedTree, setSelectedPopulatedTree] = React.useState< + TaxonFileDefaultDefinition | undefined + >(undefined); const connectedCollection = getSystemInfo().collection; @@ -235,15 +240,20 @@ export function ImportTree({ method: 'POST', headers: { Accept: 'application/json' }, body: { - mappingUrl: resource.mappingFile - } + mappingUrl: resource.mappingFile, + }, }); if (response.status === Http.OK && response.data) { - const mappingRankNames = response.data.ranks.map((rank: any) => rank.name); - const existingNames = new Set(treeDefinitionItems.map((item) => item.name)); + const mappingRankNames = response.data.ranks.map( + (rank: any) => rank.name + ); + const existingNames = new Set( + treeDefinitionItems.map((item) => item.name) + ); - const missing = mappingRankNames - .filter((rankName: string) => !existingNames.has(rankName)); + const missing = mappingRankNames.filter( + (rankName: string) => !existingNames.has(rankName) + ); if (missing.length > 0) { setSelectedPopulatedTree(resource); @@ -318,9 +328,7 @@ export function ImportTree({ {isActive === 1 ? ( - {commonText.cancel()} - + {commonText.cancel()} } header={commonText.import()} onClose={() => setIsActive(0)} @@ -363,7 +371,7 @@ async function startTreeCreation( treeName: string, treeDefId: number | undefined, createMissingRanks: boolean | undefined, - onSuccess: (taskId: string | undefined) => void, + onSuccess: (taskId: string | undefined) => void ): Promise { return ajax('/trees/create_default_tree/', { method: 'POST', @@ -384,10 +392,7 @@ async function startTreeCreation( console.log(`${treeName} tree created successfully:`, data); } else if (status === Http.ACCEPTED) { // Tree is being created in the background. - console.log( - `${treeName} tree creation started successfully:`, - data - ); + console.log(`${treeName} tree creation started successfully:`, data); onSuccess(data.task_id); } }) @@ -483,24 +488,18 @@ export function MissingTreeRanksDialog({ - - {commonText.close()} - + {commonText.close()} {commonText.no()} - - {queryText.yes()} - + {queryText.yes()} } header={treeText.missingRanks()} onClose={onClose} >
-
- {treeText.missingRanksDescription()} -
+
{treeText.missingRanksDescription()}
    {missingTreeRanks && missingTreeRanks.length > 0 @@ -508,9 +507,7 @@ export function MissingTreeRanksDialog({ : null}
-
- {treeText.createMissingRanks()} -
+
{treeText.createMissingRanks()}
 ); From 85d375a6796aaffe726b46801629ff2f2e1e7c42 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 16 Feb 2026 22:55:56 +0000 Subject: [PATCH 3/3] Lint code with ESLint and Prettier Triggered by c48c95a0a6e3278605ea3672f8a5209c554bfd4b on branch refs/heads/issue-7682-2 --- .../SystemConfigurationTool/Hierarchy.tsx | 2 +- .../js_src/lib/components/TreeView/Tree.tsx | 19 ++++++++++--------- .../frontend/js_src/lib/localization/tree.ts | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx b/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx index 1a2fdb0c855..9d56c3c315c 100644 --- a/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx +++ b/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx @@ -28,6 +28,7 @@ import { tables } from '../DataModel/tables'; import { getSystemInfo } from '../InitialContext/systemInfo'; import { Dialog, LoadingScreen } from '../Molecules/Dialog'; import { ResourceLink } from '../Molecules/ResourceLink'; +import { hasTablePermission } from '../Permissions/helpers'; import { tableLabel } from '../Preferences/UserDefinitions'; import { applyFormDefaults, @@ -40,7 +41,6 @@ import { nestAllResources } from '../SetupTool/utils'; import type { TaxonFileDefaultDefinition } from '../TreeView/CreateTree'; import { CollapsibleSection } from './CollapsibleSection'; import type { InstitutionData } from './Utils'; -import { hasTablePermission } from '../Permissions/helpers'; type HierarchyNodeKind = | 'collection' diff --git a/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx b/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx index abb9de1fdca..1c69441d272 100644 --- a/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx +++ b/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx @@ -22,6 +22,7 @@ import { idFromUrl } from '../DataModel/resource'; import { deserializeResource } from '../DataModel/serializers'; import { softError } from '../Errors/assert'; import { ResourceView } from '../Forms/ResourceView'; +import { userInformation } from '../InitialContext/userInformation'; import { hasTablePermission } from '../Permissions/helpers'; import { useHighContrast } from '../Preferences/Hooks'; import { userPreferences } from '../Preferences/userPreferences'; @@ -31,7 +32,6 @@ import { ImportTree } from './CreateTree'; import type { Conformations, Row, Stats } from './helpers'; import { fetchStats } from './helpers'; import { TreeRow } from './Row'; -import { userInformation } from '../InitialContext/userInformation'; const treeToPref = { Geography: 'geography', @@ -168,10 +168,7 @@ export function Tree< stop(); globalThis.location.reload(); return; - } else if ( - oldTreeCreationProgress === undefined && - !data.active - ) { + } else if (oldTreeCreationProgress === undefined && !data.active) { // Tree was already complete stop(); } @@ -318,7 +315,8 @@ export function Tree< ?.slice(1) as Conformations } focusPath={ - (focusPath[0] === 0 && index === 0) || focusPath[0] === row.nodeId + (focusPath[0] === 0 && index === 0) || + focusPath[0] === row.nodeId ? focusPath.slice(1) : undefined } @@ -342,13 +340,16 @@ export function Tree< setFocusPath([rows[index - 1].nodeId]); else if (action === 'previous' || action === 'parent') setFocusPath([]); - else if (action === 'focusPrevious') focusRef.current?.focus(); - else if (action === 'focusNext') searchBoxRef.current?.focus(); + else if (action === 'focusPrevious') + focusRef.current?.focus(); + else if (action === 'focusNext') + searchBoxRef.current?.focus(); return undefined; }} onChangeConformation={(newConformation): void => setConformation([ - ...(conformation?.filter(([id]) => id !== row.nodeId) ?? []), + ...(conformation?.filter(([id]) => id !== row.nodeId) ?? + []), ...(typeof newConformation === 'object' ? ([[row.nodeId, ...newConformation]] as const) : []), diff --git a/specifyweb/frontend/js_src/lib/localization/tree.ts b/specifyweb/frontend/js_src/lib/localization/tree.ts index e67758bd3f9..3528dd03c4d 100644 --- a/specifyweb/frontend/js_src/lib/localization/tree.ts +++ b/specifyweb/frontend/js_src/lib/localization/tree.ts @@ -719,7 +719,7 @@ export const treeText = createDictionary({ 'Creating tree record {current:number|formatted}/{total:number|formatted}', }, defaultTreeCreationLoadingMessage: { - 'en-us': 'Default Tree Creation is in progress, please wait...' + 'en-us': 'Default Tree Creation is in progress, please wait...', }, missingRanks: { 'en-us': 'Missing Ranks',