From 7774dfe51afab9b123f00e2da2ac927e762a943d Mon Sep 17 00:00:00 2001 From: Andre Goncalves Date: Tue, 27 Jan 2026 15:01:31 -0800 Subject: [PATCH 01/10] AB#30936 include dynamic datagrid columns in worksheet - WIP --- .../IWorksheetsMetadataService.cs | 2 +- .../WorksheetFieldSchemaParser.cs | 280 ++++++++++++++++-- .../WorksheetsMetadataService.cs | 11 +- .../Definitions/DataGridDefinition.cs | 3 + .../WorksheetFieldsProvider.cs | 2 +- 5 files changed, 271 insertions(+), 27 deletions(-) diff --git a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application.Contracts/Reporting/Configuration/IWorksheetsMetadataService.cs b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application.Contracts/Reporting/Configuration/IWorksheetsMetadataService.cs index 17ba6be14..e1d3ec774 100644 --- a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application.Contracts/Reporting/Configuration/IWorksheetsMetadataService.cs +++ b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application.Contracts/Reporting/Configuration/IWorksheetsMetadataService.cs @@ -5,7 +5,7 @@ namespace Unity.Flex.Reporting.Configuration { public interface IWorksheetsMetadataService { - Task GetWorksheetSchemaMetaDataAsync(Guid worksheetId); + Task GetWorksheetSchemaMetaDataAsync(Guid worksheetId, Guid formVersionId); Task GetWorksheetSchemaMetaDataItemAsync(Guid worksheetId, string fieldKey); } } diff --git a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetFieldSchemaParser.cs b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetFieldSchemaParser.cs index 8077c8fe6..61ad42dad 100644 --- a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetFieldSchemaParser.cs +++ b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetFieldSchemaParser.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; @@ -23,8 +24,13 @@ public static partial class WorksheetFieldSchemaParser /// /// The custom field to parse /// The worksheet containing the field (for name context) + /// Optional form schema JSON string for resolving dynamic DataGrid columns + /// Optional submission header mapping (not used in current implementation) /// List of component metadata items - public static List ParseField(CustomField field, Worksheet worksheet) + public static List ParseField(CustomField field, + Worksheet worksheet, + string? formSchema = null, + string? submissionHeaderMapping = null) { if (field == null) return []; @@ -34,7 +40,7 @@ public static List ParseField(CustomField fie switch (field.Type) { case CustomFieldType.DataGrid: - components.AddRange(ParseDataGridField(field, worksheet)); + components.AddRange(ParseDataGridField(field, worksheet, formSchema, submissionHeaderMapping)); break; case CustomFieldType.CheckboxGroup: @@ -58,8 +64,12 @@ public static List ParseField(CustomField fie /// Parses all fields in a worksheet and returns flattened component metadata. /// /// The worksheet to parse + /// Optional form schema JSON string for resolving dynamic DataGrid columns + /// Optional submission header mapping (not used in current implementation) /// List of all component metadata items - public static List ParseWorksheet(Worksheet worksheet) + public static List ParseWorksheet(Worksheet worksheet, + string? formSchema = null, + string? submissionHeaderMapping = null) { if (worksheet?.Sections == null) return []; @@ -67,17 +77,23 @@ public static List ParseWorksheet(Worksheet w return [..worksheet.Sections .Where(section => section.Fields != null) .SelectMany(section => section.Fields) - .SelectMany(field => ParseField(field, worksheet))]; + .SelectMany(field => ParseField(field, worksheet, formSchema, submissionHeaderMapping))]; } /// /// Parses a DataGrid field and returns metadata for each column defined in the DataGrid definition. - /// If dynamic is true, creates a placeholder column for dynamically determined columns. + /// If dynamic is true, attempts to extract columns from the form schema. If form schema is not available + /// or parsing fails, creates a placeholder column for dynamically determined columns. /// /// The DataGrid field to parse /// The worksheet containing the field + /// Optional form schema JSON string for resolving dynamic DataGrid columns + /// Optional submission header mapping (not used in current implementation) /// List of component metadata items for each DataGrid column - private static List ParseDataGridField(CustomField field, Worksheet worksheet) + private static List ParseDataGridField(CustomField field, + Worksheet worksheet, + string? formSchema = null, + string? submissionHeaderMapping = null) { var components = new List(); @@ -98,25 +114,68 @@ private static List ParseDataGridField(Custom var worksheetName = SanitizeName(worksheet.Name); var dataGridName = SanitizeName(field.Key); - // If dynamic is true, create a placeholder for dynamically determined columns + // Track whether we successfully extracted columns from CHEFS schema + bool extractedFromChefs = false; + + // If dynamic is true, try to extract columns from form schema if (dataGridDefinition.Dynamic) - { - var dynamicComponent = new WorksheetComponentMetaDataItemDto + { + var headerMappingKey = MatchHeaderMapping(field.Name + ".DataGrid", submissionHeaderMapping); + List? dynamicColumns = null; + + if (!string.IsNullOrWhiteSpace(headerMappingKey)) { - Id = $"{field.Id}_dynamic", - Key = "dynamic_columns", - Label = "Dynamic Columns", - Type = "Dynamic", - Path = $"{worksheetName}->{sectionName}->{dataGridName}->dynamic_columns", - TypePath = $"worksheet->section->datagrid->Dynamic", - DataPath = $"({worksheetName}){dataGridName}->dynamic_columns" - }; + dynamicColumns = ExtractDynamicDataGridColumns(headerMappingKey, formSchema); + } - components.Add(dynamicComponent); + if (dynamicColumns != null && dynamicColumns.Count > 0) + { + // We found columns in the form schema, use them + // CHEFS schema includes ALL columns (both static and dynamic), so we mark this as extracted + extractedFromChefs = true; + + foreach (var column in dynamicColumns) + { + // Use the key for the component Key (becomes PropertyName), sanitize for ID + var columnKey = !string.IsNullOrEmpty(column.Key) ? column.Key : column.Name; + var sanitizedKey = SanitizeName(columnKey); + + var component = new WorksheetComponentMetaDataItemDto + { + Id = $"{field.Id}_{sanitizedKey}", + Key = columnKey, + Label = column.Name, + Type = MapDataGridColumnType(column.Type), + Path = $"{worksheetName}->{sectionName}->{dataGridName}->{columnKey}", + TypePath = $"worksheet->section->datagrid->{MapDataGridColumnType(column.Type)}", + DataPath = $"({worksheetName}){dataGridName}->{columnKey}" + }; + + components.Add(component); + } + } + else + { + // Form schema not available or no columns found, create dynamic placeholder + var dynamicComponent = new WorksheetComponentMetaDataItemDto + { + Id = $"{field.Id}_dynamic", + Key = "dynamic_columns", + Label = "Dynamic Columns", + Type = "Dynamic", + Path = $"{worksheetName}->{sectionName}->{dataGridName}->dynamic_columns", + TypePath = $"worksheet->section->datagrid->Dynamic", + DataPath = $"({worksheetName}){dataGridName}->dynamic_columns" + }; + + components.Add(dynamicComponent); + } } - // Process additional defined columns (if any) - if (dataGridDefinition.Columns != null && dataGridDefinition.Columns.Count > 0) + // Process additional defined columns only if we haven't already extracted them from CHEFS + // When dynamic is true and CHEFS extraction succeeded, the CHEFS schema already includes + // all columns (both static and dynamic), so we skip this to avoid duplicates + if (!extractedFromChefs && dataGridDefinition.Columns != null && dataGridDefinition.Columns.Count > 0) { // Create a component for each column in the DataGrid foreach (var column in dataGridDefinition.Columns) @@ -137,7 +196,7 @@ private static List ParseDataGridField(Custom components.Add(component); } } - else if (!dataGridDefinition.Dynamic) + else if (!dataGridDefinition.Dynamic && (dataGridDefinition.Columns == null || dataGridDefinition.Columns.Count == 0)) { // If no columns defined and not dynamic, return the DataGrid itself as a component return [CreateSimpleComponent(field, worksheet)]; @@ -153,6 +212,30 @@ private static List ParseDataGridField(Custom } } + private static string? MatchHeaderMapping(string keyToSearch, string? submissionHeaderMapping) + { + if (string.IsNullOrWhiteSpace(submissionHeaderMapping)) + return null; + + try + { + using var document = JsonDocument.Parse(submissionHeaderMapping); + var root = document.RootElement; + + if (root.TryGetProperty(keyToSearch, out var valueElement)) + { + return valueElement.GetString(); + } + } + catch (JsonException) + { + // Failed to parse submission header mapping + return null; + } + + return null; + } + /// /// Parses a CheckboxGroup field and returns metadata for each checkbox option defined in the CheckboxGroup definition. /// @@ -298,6 +381,161 @@ private static string SanitizeName(string name) return SantizedNameExpression().Replace(name.Trim().Replace(" ", "_"), ""); } + /// + /// Extracts DataGrid column definitions from a form schema JSON string. + /// Searches for a DataGrid component with the specified key in the form schema and returns its columns. + /// + /// The key of the DataGrid field to find + /// The form schema JSON string to search + /// List of DataGrid column definitions, or null if not found or schema is invalid + private static List? ExtractDynamicDataGridColumns(string dataGridKey, string? formSchema) + { + if (string.IsNullOrWhiteSpace(formSchema)) + return null; + + try + { + using var document = JsonDocument.Parse(formSchema); + var root = document.RootElement; + + // CHEFS form schemas have a "components" array at the root + if (root.TryGetProperty("components", out var componentsElement)) + { + return FindDataGridInComponents(componentsElement, dataGridKey); + } + } + catch (JsonException) + { + // Failed to parse form schema + return null; + } + + return null; + } + + /// + /// Recursively searches for a DataGrid component with the specified key in a components array. + /// + /// The JSON element representing a components array + /// The key of the DataGrid to find + /// List of DataGrid column definitions, or null if not found + private static List? FindDataGridInComponents(JsonElement componentsElement, string dataGridKey) + { + if (componentsElement.ValueKind != JsonValueKind.Array) + return null; + + foreach (var component in componentsElement.EnumerateArray()) + { + // Check if this component matches the DataGrid key + if (component.TryGetProperty("key", out var keyElement) && + keyElement.GetString() == dataGridKey) + { + // Check if it's a DataGrid type + if (component.TryGetProperty("type", out var typeElement)) + { + var type = typeElement.GetString(); + if (type == "datagrid" || type == "dataGrid") + { + // Extract columns from the DataGrid + return ExtractColumnsFromDataGrid(component); + } + } + } + + // Recursively search in nested components (for panels, columns, etc.) + if (component.TryGetProperty("components", out var nestedComponents)) + { + var result = FindDataGridInComponents(nestedComponents, dataGridKey); + if (result != null) + return result; + } + + // Also check columns property (for layout components) + if (component.TryGetProperty("columns", out var columnsElement) && + columnsElement.ValueKind == JsonValueKind.Array) + { + foreach (var column in columnsElement.EnumerateArray()) + { + if (column.TryGetProperty("components", out var columnComponents)) + { + var result = FindDataGridInComponents(columnComponents, dataGridKey); + if (result != null) + return result; + } + } + } + } + + return null; + } + + /// + /// Extracts column definitions from a DataGrid component in the form schema. + /// + /// The JSON element representing a DataGrid component + /// List of DataGrid column definitions + private static List? ExtractColumnsFromDataGrid(JsonElement dataGridComponent) + { + var columns = new List(); + + // CHEFS DataGrid components have a "components" property that contains the column definitions + if (dataGridComponent.TryGetProperty("components", out var componentsElement) && + componentsElement.ValueKind == JsonValueKind.Array) + { + foreach (var columnComponent in componentsElement.EnumerateArray()) + { + // Get the key (property name) from the column component + var columnKey = columnComponent.TryGetProperty("key", out var keyElement) + ? keyElement.GetString() + : null; + + // Get the label (display name) from the column component + var columnLabel = columnComponent.TryGetProperty("label", out var labelElement) + ? labelElement.GetString() + : columnKey; + + var columnType = columnComponent.TryGetProperty("type", out var typeElement) + ? MapChefsTypeToDataGridType(typeElement.GetString()) + : "text"; + + if (!string.IsNullOrEmpty(columnKey)) + { + columns.Add(new DataGridDefinitionColumn + { + Key = columnKey, + Name = columnLabel ?? columnKey, + Type = columnType + }); + } + } + } + + return columns.Count > 0 ? columns : null; + } + + /// + /// Maps CHEFS form component types to DataGrid column types. + /// + /// The CHEFS component type + /// Mapped DataGrid column type + private static string MapChefsTypeToDataGridType(string? chefsType) + { + return chefsType?.ToLowerInvariant() switch + { + "textfield" => "text", + "textarea" => "text", + "number" => "numeric", + "currency" => "currency", + "checkbox" => "checkbox", + "day" => "date", + "datetime" => "datetime", + "email" => "text", + "phoneNumber" => "text", + "url" => "text", + _ => "text" // Default to text for unknown types + }; + } + [GeneratedRegex(@"[^a-zA-Z0-9_\-]")] private static partial Regex SantizedNameExpression(); } diff --git a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetsMetadataService.cs b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetsMetadataService.cs index d51a1f863..13c9b2efd 100644 --- a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetsMetadataService.cs +++ b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Application/Reporting/Configuration/WorksheetsMetadataService.cs @@ -2,19 +2,22 @@ using System.Linq; using System.Threading.Tasks; using Unity.Flex.Domain.Worksheets; +using Unity.GrantManager.ApplicationForms; using Volo.Abp.DependencyInjection; namespace Unity.Flex.Reporting.Configuration { - public class WorksheetsMetadataService(IWorksheetRepository worksheetRepository) + public class WorksheetsMetadataService(IWorksheetRepository worksheetRepository, + IApplicationFormVersionAppService formVersionAppService) : IWorksheetsMetadataService, ITransientDependency { - public async Task GetWorksheetSchemaMetaDataAsync(Guid worksheetId) + public async Task GetWorksheetSchemaMetaDataAsync(Guid worksheetId, Guid formVersionId) { var worksheet = await worksheetRepository.GetAsync(worksheetId); - + var version = await formVersionAppService.GetAsync(formVersionId); + // Use the utility class to parse all fields in the worksheet - var components = WorksheetFieldSchemaParser.ParseWorksheet(worksheet); + var components = WorksheetFieldSchemaParser.ParseWorksheet(worksheet, version.FormSchema, version.SubmissionHeaderMapping); return new WorksheetComponentMetaDataDto() { diff --git a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Shared/Worksheets/Definitions/DataGridDefinition.cs b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Shared/Worksheets/Definitions/DataGridDefinition.cs index 368a06274..bcb81619d 100644 --- a/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Shared/Worksheets/Definitions/DataGridDefinition.cs +++ b/applications/Unity.GrantManager/modules/Unity.Flex/src/Unity.Flex.Shared/Worksheets/Definitions/DataGridDefinition.cs @@ -27,6 +27,9 @@ public class DataGridDefinitionColumn [JsonPropertyName("type")] public string Type { get; set; } = string.Empty; + + [JsonPropertyName("key")] + public string Key { get; set; } = string.Empty; } public enum DataGridDefinitionSummaryOption diff --git a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs index 135bb2ae8..a3ebe73d1 100644 --- a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs +++ b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs @@ -37,7 +37,7 @@ public async Task GetFieldsMetadataAsync(Guid correlationId foreach (var link in links) { - var metadata = await worksheetsMetadataService.GetWorksheetSchemaMetaDataAsync(link.WorksheetId); + var metadata = await worksheetsMetadataService.GetWorksheetSchemaMetaDataAsync(link.WorksheetId, correlationId); worksheetMetadata.Add(metadata); // Add worksheet information to the metadata map From b2470e81e7ea6ad2116da39972b6293732d2f448 Mon Sep 17 00:00:00 2001 From: Andre Goncalves Date: Mon, 2 Mar 2026 14:50:25 -0800 Subject: [PATCH 02/10] AB#30936 fix map and add unmapped dt warning --- .../ScoresheetFieldsProvider.cs | 27 ++++----- .../WorksheetFieldsProvider.cs | 3 +- .../ReportColumnsMapRepository.cs | 8 ++- .../ReportingConfiguration/Default.cshtml | 11 ++++ .../ReportingConfiguration/Default.css | 59 +++++++++++++++++++ .../ReportingConfiguration/Default.js | 32 ++++++++++ 6 files changed, 121 insertions(+), 19 deletions(-) diff --git a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/ScoresheetFieldsProvider.cs b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/ScoresheetFieldsProvider.cs index 265b964e3..2c3c7bee4 100644 --- a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/ScoresheetFieldsProvider.cs +++ b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/ScoresheetFieldsProvider.cs @@ -5,11 +5,10 @@ using System.Threading.Tasks; using Unity.Flex.Reporting.Configuration; using Unity.GrantManager.ApplicationForms; -using Unity.Reporting.Configuration.FieldsProviders; using Unity.Reporting.Domain.Configuration; using Volo.Abp.DependencyInjection; -namespace Unity.Reporting.Configuration.FieldProviders +namespace Unity.Reporting.Configuration.FieldsProviders { /// /// Fields provider implementation for Unity.Flex scoresheets that extracts field metadata from scoresheet configurations. @@ -146,7 +145,7 @@ private static Dictionary GetStoredScoresheetInfo(ReportColumnsM try { // Parse the mapping JSON to extract metadata - var mapping = JsonSerializer.Deserialize(reportColumnsMap.Mapping); + var mapping = JsonSerializer.Deserialize(reportColumnsMap.Mapping); var info = mapping?.Metadata?.Info; if (info != null) { @@ -222,19 +221,19 @@ private static string ExtractScoresheetId(string infoString) return infoString; } - } - /// - /// Simplified internal mapping class structure for JSON deserialization compatibility. - /// Used specifically for parsing stored mapping metadata to extract scoresheet information - /// during change detection operations without requiring the full mapping object structure. - /// - internal class Mapping - { /// - /// Gets or sets the metadata information associated with the mapping. - /// Contains contextual information about scoresheets and other correlation-specific details. + /// Simplified private mapping class structure for JSON deserialization compatibility. + /// Used specifically for parsing stored mapping metadata to extract scoresheet information + /// during change detection operations without requiring the full mapping object structure. /// - public MapMetadataDto? Metadata { get; set; } + private class ScoresheetMapping + { + /// + /// Gets or sets the metadata information associated with the mapping. + /// Contains contextual information about scoresheets and other correlation-specific details. + /// + public MapMetadataDto? Metadata { get; set; } + } } } diff --git a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs index a3ebe73d1..b609850c8 100644 --- a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs +++ b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/Configuration/FieldsProviders/WorksheetFieldsProvider.cs @@ -4,12 +4,11 @@ using System.Threading.Tasks; using Unity.Flex.Reporting.Configuration; using Unity.Flex.WorksheetLinks; -using Unity.Reporting.Configuration.FieldsProviders; using Unity.Reporting.Domain.Configuration; using Volo.Abp.DependencyInjection; using System.Text.Json; -namespace Unity.Reporting.Configuration.FieldProviders +namespace Unity.Reporting.Configuration.FieldsProviders { /// /// Fields provider implementation for Unity.Flex worksheets that extracts field metadata from linked worksheets. diff --git a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/EntityFrameworkCore/Repositories/ReportColumnsMapRepository.cs b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/EntityFrameworkCore/Repositories/ReportColumnsMapRepository.cs index d8350599a..2e559ef4c 100644 --- a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/EntityFrameworkCore/Repositories/ReportColumnsMapRepository.cs +++ b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Application/EntityFrameworkCore/Repositories/ReportColumnsMapRepository.cs @@ -118,13 +118,15 @@ public async Task GetViewPreviewDataAsync(string viewName, ViewD ColumnNames = await GetViewColumnNamesAsync(normalizedViewName) }; - // Build the preview query using the LIMIT 1 subquery pattern + // Build the preview query - select the most recently created application var previewQuery = $@" SELECT * FROM ""Reporting"".""{normalizedViewName}"" WHERE ""application_id"" = ( - SELECT ""application_id"" - FROM ""Reporting"".""{normalizedViewName}"" + SELECT v.""application_id"" + FROM ""Reporting"".""{normalizedViewName}"" v + INNER JOIN ""Applications"" a ON v.""application_id"" = a.""Id"" + ORDER BY a.""CreationTime"" DESC LIMIT 1 )"; diff --git a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Web/Views/Shared/Components/ReportingConfiguration/Default.cshtml b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Web/Views/Shared/Components/ReportingConfiguration/Default.cshtml index 03e3c82fe..871fa26a8 100644 --- a/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Web/Views/Shared/Components/ReportingConfiguration/Default.cshtml +++ b/applications/Unity.GrantManager/modules/Unity.Reporting/src/Unity.Reporting.Web/Views/Shared/Components/ReportingConfiguration/Default.cshtml @@ -102,6 +102,17 @@ +
+ + + Unmapped DataGrid + +
+ @if (await PermissionChecker.IsGrantedAsync(ReportingPermissions.Configuration.Delete)) {