From bcdbf736c61d4a13f4066c66ea5029401a5a1a16 Mon Sep 17 00:00:00 2001 From: Mike Alhayek Date: Mon, 27 Jul 2026 18:57:20 +0300 Subject: [PATCH] Require data source field mapping and stop leaking documents as titles - Require key, title, and content field mappings in the MVC and Blazor data source create/edit forms. - Fall back to the document key instead of the serialized source document when no title is mapped. - Sanitize data source citation titles so serialized documents are never rendered as reference titles. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...icsearchSourceDocumentMappingBenchmarks.cs | 14 +++-- .../docs/changelog/v1.0.0.md | 1 + .../docs/data-sources/index.md | 12 ++++ .../AzureAISearchAIDataSourceSourceHandler.cs | 12 ++-- .../ElasticsearchAIDataSourceSourceHandler.cs | 4 +- .../PostgreSQLAIDataSourceSourceHandler.cs | 13 ++--- .../DataSourcePreemptiveRagHandler.cs | 21 ++++++- .../Tools/DataSourceSearchTool.cs | 30 +++++++++- .../DataSourceAzureAISearchDocumentReader.cs | 20 +++---- .../DataSourceElasticsearchDocumentReader.cs | 5 +- .../ElasticsearchSourceDocumentMapper.cs | 9 +-- .../DataSourcePostgreSQLDocumentReader.cs | 13 ++--- .../DataSources/AIDataSources/Create.razor | 18 ++++-- .../DataSources/AIDataSources/Edit.razor | 18 ++++-- .../Controllers/AIDataSourceController.cs | 10 ++++ .../Views/AIDataSource/Create.cshtml | 12 ++-- .../Views/AIDataSource/Edit.cshtml | 12 ++-- .../DocumentTitleResolver.cs | 49 ++++++++++++++++ ...earchProviderSourceDocumentMappingTests.cs | 56 +++++++++++++++++++ 19 files changed, 263 insertions(+), 66 deletions(-) create mode 100644 src/Utilities/CrestApps.Core.Support/DocumentTitleResolver.cs diff --git a/benchmarks/CrestApps.Core.Benchmarks/ElasticsearchSourceDocumentMappingBenchmarks.cs b/benchmarks/CrestApps.Core.Benchmarks/ElasticsearchSourceDocumentMappingBenchmarks.cs index 0e180a67..f1d7bcf3 100644 --- a/benchmarks/CrestApps.Core.Benchmarks/ElasticsearchSourceDocumentMappingBenchmarks.cs +++ b/benchmarks/CrestApps.Core.Benchmarks/ElasticsearchSourceDocumentMappingBenchmarks.cs @@ -55,9 +55,10 @@ public void Setup() { var legacyKey = LegacyResolveKey(_sources[index]); var currentKey = CurrentResolveKey(_sources[index]); - var legacyDocument = LegacyExtractDocument(_sources[index], TitleFieldName, ContentFieldName); + var legacyDocument = LegacyExtractDocument(_sources[index], legacyKey, TitleFieldName, ContentFieldName); var currentDocument = ElasticsearchSourceDocumentMapper.ExtractDocument( _sources[index], + currentKey, _titleFieldPath, _contentFieldPath, treatWhitespaceAsEmpty: false); @@ -78,7 +79,7 @@ public int Legacy() foreach (var source in _sources) { var key = LegacyResolveKey(source); - var document = LegacyExtractDocument(source, TitleFieldName, ContentFieldName); + var document = LegacyExtractDocument(source, key, TitleFieldName, ContentFieldName); checksum += key.Length + document.Title.Length + document.Content.Length + document.Fields.Count; } @@ -99,6 +100,7 @@ public int Current() var key = CurrentResolveKey(source); var document = ElasticsearchSourceDocumentMapper.ExtractDocument( source, + key, _titleFieldPath, _contentFieldPath, treatWhitespaceAsEmpty: false); @@ -120,11 +122,13 @@ private string CurrentResolveKey(JsonObject source) private static SourceDocument LegacyExtractDocument( JsonObject source, + string documentKey, string titleFieldName, string contentFieldName) { string title = null; string content = null; + var contentIsSerializedDocument = false; if (!string.IsNullOrEmpty(titleFieldName)) { @@ -141,12 +145,10 @@ private static SourceDocument LegacyExtractDocument( if (string.IsNullOrEmpty(content)) { content = source.ToJsonString(); + contentIsSerializedDocument = true; } - if (string.IsNullOrEmpty(title) && !string.IsNullOrEmpty(content)) - { - title = content.ExtractTitleFromContent(); - } + title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey); var fields = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var property in source) diff --git a/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md b/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md index 8d91fcc0..f3633ef3 100644 --- a/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md +++ b/src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md @@ -116,3 +116,4 @@ description: Initial standalone release notes for the CrestApps.Core repository. - promotes the parameterized AI tool instances into a first-class opt-in feature registered on the AI suite builder with `AddToolInstances(toolInstances => toolInstances.AddSource(...))` (with an `AddHttpApiRequestSource()` convenience for the built-in source), instead of being wired into the core AI services automatically; persistence is registered on the tool-instances builder via `AddYesSqlStores()`/`AddEntityCoreStores()` rather than on the AI suite; drops the redundant per-instance `DisplayText` in favor of the unique `Name`, localizes the source `DisplayName`/`Description`/`Category`, simplifies `IAIToolInstanceSource.CreateTool(AIToolInstance instance)` to take the instance directly, generalizes the completion-context handler to honor tool instances on any resource so both AI profiles and chat interactions can select them, renames the built-in HTTP source's basic/OAuth credentials to `Username`/`Password` and adds the OAuth 2.0 resource-owner password grant, hardens model-provided paths so they cannot redirect a request off the configured host, and updates the MVC and Blazor sample hosts to let users attach instances to both AI profiles and chat interactions - namespaces every tool-instance function name with the `AIToolInstanceExtensions.FunctionNamePrefix` (`tool_instance_`) prefix so a user-chosen instance name can never collide with a tool registered in code via `AddCoreAITool` (which the model sees under its bare registered name); both kinds of tool now coexist safely in the single function namespace exposed to OpenAI, Azure OpenAI, and every other client - adds a source dropdown to the AI tool instance create form in the MVC and Blazor sample hosts that reveals only the selected source's fields (the source is fixed and shown read-only on edit), and relocates the "AI Tool Instances" admin menu item next to "AI Profiles" in both samples +- requires the key, title, and content field mappings when creating or editing an AI data source in the MVC and Blazor sample hosts, and makes every built-in source reader fall back to the document key instead of the serialized source document when no title is mapped, so chat citations never render a full JSON document as a reference title diff --git a/src/CrestApps.Core.Docs/docs/data-sources/index.md b/src/CrestApps.Core.Docs/docs/data-sources/index.md index fa02dbfa..f89beb46 100644 --- a/src/CrestApps.Core.Docs/docs/data-sources/index.md +++ b/src/CrestApps.Core.Docs/docs/data-sources/index.md @@ -43,6 +43,18 @@ The built-in source types are: `SearchIndexProfile` remains the default source type and the simplest option when your content is already indexed through CrestApps.Core. +## Field Mapping + +Every `AIDataSource` maps three source fields: + +| Property | Purpose | +| --- | --- | +| `KeyFieldName` | The source field that identifies the document. It becomes the citation reference id. | +| `TitleFieldName` | The source field used as the document title in citations and chat references. | +| `ContentFieldName` | The source field holding the text body that is chunked and embedded. | + +The sample MVC and Blazor hosts require all three fields when creating or editing a data source. Map them explicitly: when a title is not mapped, the framework falls back to the document key and never uses the serialized source document as a title, so citations stay readable and never leak the full document payload. + ## What is Vector Search? If you are new to AI-powered search, here is a brief primer on the concepts that data sources rely on. diff --git a/src/Primitives/CrestApps.Core.AI.Azure.AISearch/Services/AzureAISearchAIDataSourceSourceHandler.cs b/src/Primitives/CrestApps.Core.AI.Azure.AISearch/Services/AzureAISearchAIDataSourceSourceHandler.cs index c0f06ca8..2ccd0ee4 100644 --- a/src/Primitives/CrestApps.Core.AI.Azure.AISearch/Services/AzureAISearchAIDataSourceSourceHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.Azure.AISearch/Services/AzureAISearchAIDataSourceSourceHandler.cs @@ -180,7 +180,7 @@ private static KeyValuePair CreateDocumentPair(AIDataSou key = configuredKeyValue?.ToString() ?? key; } - return new KeyValuePair(key, ExtractDocument(document, dataSource.TitleFieldName, dataSource.ContentFieldName)); + return new KeyValuePair(key, ExtractDocument(document, key, dataSource.TitleFieldName, dataSource.ContentFieldName)); } private (global::Azure.Search.Documents.SearchClient SearchClient, AzureAISearchSourceMetadata Metadata) Resolve(AIDataSource dataSource) @@ -208,7 +208,7 @@ private static KeyValuePair CreateDocumentPair(AIDataSou return (client, metadata); } - private static SourceDocument ExtractDocument(SearchDocument document, string titleFieldName, string contentFieldName) + private static SourceDocument ExtractDocument(SearchDocument document, string documentKey, string titleFieldName, string contentFieldName) { string title = null; string content = null; @@ -223,15 +223,15 @@ private static SourceDocument ExtractDocument(SearchDocument document, string ti content = contentValue?.ToString(); } + var contentIsSerializedDocument = false; + if (string.IsNullOrWhiteSpace(content)) { content = System.Text.Json.JsonSerializer.Serialize(document); + contentIsSerializedDocument = true; } - if (string.IsNullOrWhiteSpace(title) && !string.IsNullOrWhiteSpace(content)) - { - title = content.ExtractTitleFromContent(); - } + title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey); var fields = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var kvp in document) diff --git a/src/Primitives/CrestApps.Core.AI.Elasticsearch/Services/ElasticsearchAIDataSourceSourceHandler.cs b/src/Primitives/CrestApps.Core.AI.Elasticsearch/Services/ElasticsearchAIDataSourceSourceHandler.cs index 0f7963ad..f2072bb2 100644 --- a/src/Primitives/CrestApps.Core.AI.Elasticsearch/Services/ElasticsearchAIDataSourceSourceHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.Elasticsearch/Services/ElasticsearchAIDataSourceSourceHandler.cs @@ -252,6 +252,7 @@ private static KeyValuePair CreateDocumentPair( key, ElasticsearchSourceDocumentMapper.ExtractDocument( source, + key, titleFieldPath, contentFieldPath, treatWhitespaceAsEmpty: true)); @@ -300,10 +301,11 @@ private static KeyValuePair CreateDocumentPair( return (client, metadata); } - private static SourceDocument ExtractDocument(System.Text.Json.Nodes.JsonObject source, string titleFieldName, string contentFieldName) + private static SourceDocument ExtractDocument(System.Text.Json.Nodes.JsonObject source, string documentKey, string titleFieldName, string contentFieldName) { return ElasticsearchSourceDocumentMapper.ExtractDocument( source, + documentKey, ElasticsearchSourceDocumentMapper.CreateFieldPath(titleFieldName), ElasticsearchSourceDocumentMapper.CreateFieldPath(contentFieldName), treatWhitespaceAsEmpty: true); diff --git a/src/Primitives/CrestApps.Core.AI.PostgreSQL/Services/PostgreSQLAIDataSourceSourceHandler.cs b/src/Primitives/CrestApps.Core.AI.PostgreSQL/Services/PostgreSQLAIDataSourceSourceHandler.cs index a047080c..70c9a2fb 100644 --- a/src/Primitives/CrestApps.Core.AI.PostgreSQL/Services/PostgreSQLAIDataSourceSourceHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.PostgreSQL/Services/PostgreSQLAIDataSourceSourceHandler.cs @@ -109,7 +109,7 @@ public async IAsyncEnumerable> ReadAsync(AI var row = ReadRow(reader); var key = ResolveKey(row, dataSource.KeyFieldName); - yield return new KeyValuePair(key, ExtractDocument(row, dataSource.TitleFieldName, dataSource.ContentFieldName)); + yield return new KeyValuePair(key, ExtractDocument(row, key, dataSource.TitleFieldName, dataSource.ContentFieldName)); } if (rowCount < 1000) @@ -157,7 +157,7 @@ public async IAsyncEnumerable> ReadByIdsAsy var row = ReadRow(reader); var key = ResolveKey(row, dataSource.KeyFieldName); - yield return new KeyValuePair(key, ExtractDocument(row, dataSource.TitleFieldName, dataSource.ContentFieldName)); + yield return new KeyValuePair(key, ExtractDocument(row, key, dataSource.TitleFieldName, dataSource.ContentFieldName)); } } @@ -213,10 +213,11 @@ private static string ResolveKey(Dictionary row, string keyField return null; } - private static SourceDocument ExtractDocument(Dictionary row, string titleFieldName, string contentFieldName) + private static SourceDocument ExtractDocument(Dictionary row, string documentKey, string titleFieldName, string contentFieldName) { string title = null; string content = null; + var contentIsSerializedDocument = false; if (!string.IsNullOrWhiteSpace(titleFieldName) && row.TryGetValue(titleFieldName, out var titleValue) && titleValue != null) { @@ -237,12 +238,10 @@ private static SourceDocument ExtractDocument(Dictionary row, st } content = json.ToJsonString(); + contentIsSerializedDocument = true; } - if (string.IsNullOrWhiteSpace(title) && !string.IsNullOrWhiteSpace(content)) - { - title = content.ExtractTitleFromContent(); - } + title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey); var fields = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var kvp in row) diff --git a/src/Primitives/CrestApps.Core.AI/Handlers/DataSourcePreemptiveRagHandler.cs b/src/Primitives/CrestApps.Core.AI/Handlers/DataSourcePreemptiveRagHandler.cs index 77c1dbc2..79d57523 100644 --- a/src/Primitives/CrestApps.Core.AI/Handlers/DataSourcePreemptiveRagHandler.cs +++ b/src/Primitives/CrestApps.Core.AI/Handlers/DataSourcePreemptiveRagHandler.cs @@ -8,6 +8,7 @@ using CrestApps.Core.Infrastructure.Indexing; using CrestApps.Core.Infrastructure.Indexing.DataSources; using CrestApps.Core.Infrastructure.Indexing.Models; +using CrestApps.Core.Support; using CrestApps.Core.Templates.Services; using Cysharp.Text; using Microsoft.Extensions.AI; @@ -295,7 +296,7 @@ private async Task SearchAndInjectContextAsync( { seenReferences[result.ReferenceId] = ( invocationContext?.NextReferenceIndex() ?? seenReferences.Count + 1, - _textNormalizer.NormalizeTitle(result.Title), + ResolveReferenceTitle(result.Title, result.ReferenceId), result.ReferenceType); } @@ -351,6 +352,24 @@ private async Task SearchAndInjectContextAsync( orchestrationContext.SystemMessageBuilder.Append(stringBuilder); } + /// + /// Resolves a citation title that never exposes a serialized source document. + /// + /// The indexed document title. + /// The document reference identifier used as the fallback title. + /// The resolved citation title. + private string ResolveReferenceTitle(string title, string referenceId) + { + var normalizedTitle = _textNormalizer.NormalizeTitle(title); + + if (string.IsNullOrWhiteSpace(normalizedTitle) || DocumentTitleResolver.LooksLikeSerializedDocument(normalizedTitle)) + { + return referenceId; + } + + return normalizedTitle; + } + private static AIDataSourceRagMetadata GetRagMetadata(object resource) { if (resource is AIProfile profile && diff --git a/src/Primitives/CrestApps.Core.AI/Tools/DataSourceSearchTool.cs b/src/Primitives/CrestApps.Core.AI/Tools/DataSourceSearchTool.cs index 2013e4ab..3cdb416b 100644 --- a/src/Primitives/CrestApps.Core.AI/Tools/DataSourceSearchTool.cs +++ b/src/Primitives/CrestApps.Core.AI/Tools/DataSourceSearchTool.cs @@ -9,6 +9,7 @@ using CrestApps.Core.AI.Tooling; using CrestApps.Core.Infrastructure.Indexing; using CrestApps.Core.Infrastructure.Indexing.DataSources; +using CrestApps.Core.Support; using Cysharp.Text; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; @@ -241,20 +242,24 @@ protected override async ValueTask InvokeCoreAsync(AIFunctionArguments a if (!string.IsNullOrEmpty(result.ReferenceId) && !seenReferences.ContainsKey(result.ReferenceId)) { - seenReferences[result.ReferenceId] = (invocationContext.NextReferenceIndex(), textNormalizer.NormalizeTitle(result.Title), result.ReferenceType); + seenReferences[result.ReferenceId] = (invocationContext.NextReferenceIndex(), ResolveReferenceTitle(textNormalizer, result.Title, result.ReferenceId), result.ReferenceType); } var refLabel = !string.IsNullOrEmpty(result.ReferenceId) && seenReferences.TryGetValue(result.ReferenceId, out var entry) ? $"[doc:{entry.Index}]" : $"[doc:{invocationContext.NextReferenceIndex()}]"; + var displayTitle = !string.IsNullOrEmpty(result.ReferenceId) && seenReferences.TryGetValue(result.ReferenceId, out var titleEntry) + ? titleEntry.Title + : ResolveReferenceTitle(textNormalizer, result.Title, result.ReferenceId); + builder.AppendLine("---"); - if (!string.IsNullOrWhiteSpace(result.Title)) + if (!string.IsNullOrWhiteSpace(displayTitle)) { builder.Append(refLabel); builder.Append(" Title: "); - builder.AppendLine(result.Title); + builder.AppendLine(displayTitle); } builder.Append(refLabel); @@ -300,6 +305,25 @@ protected override async ValueTask InvokeCoreAsync(AIFunctionArguments a } } + /// + /// Resolves a citation title that never exposes a serialized source document. + /// + /// The text normalizer. + /// The indexed document title. + /// The document reference identifier used as the fallback title. + /// The resolved citation title. + private static string ResolveReferenceTitle(IAITextNormalizer textNormalizer, string title, string referenceId) + { + var normalizedTitle = textNormalizer.NormalizeTitle(title); + + if (string.IsNullOrWhiteSpace(normalizedTitle) || DocumentTitleResolver.LooksLikeSerializedDocument(normalizedTitle)) + { + return referenceId; + } + + return normalizedTitle; + } + private static AIDataSourceRagMetadata GetRagMetadata(AIToolExecutionContext executionContext) { if (executionContext.Resource is AIProfile profile && profile.TryGet(out var ragMetadata)) diff --git a/src/Primitives/CrestApps.Core.Azure.AISearch/Services/DataSourceAzureAISearchDocumentReader.cs b/src/Primitives/CrestApps.Core.Azure.AISearch/Services/DataSourceAzureAISearchDocumentReader.cs index dec784a5..4a3a653d 100644 --- a/src/Primitives/CrestApps.Core.Azure.AISearch/Services/DataSourceAzureAISearchDocumentReader.cs +++ b/src/Primitives/CrestApps.Core.Azure.AISearch/Services/DataSourceAzureAISearchDocumentReader.cs @@ -91,7 +91,7 @@ public async IAsyncEnumerable> ReadAsync( if (!string.IsNullOrEmpty(documentKey)) { yield return new KeyValuePair( - documentKey, ExtractDocument(doc, titleFieldName, contentFieldName)); + documentKey, ExtractDocument(doc, documentKey, titleFieldName, contentFieldName)); } } } @@ -165,7 +165,7 @@ public async IAsyncEnumerable> ReadByIdsAsy if (!string.IsNullOrEmpty(documentKey)) { yield return new KeyValuePair( - documentKey, ExtractDocument(doc, titleFieldName, contentFieldName)); + documentKey, ExtractDocument(doc, documentKey, titleFieldName, contentFieldName)); } } } @@ -200,16 +200,17 @@ public async IAsyncEnumerable> ReadByIdsAsy : id; yield return new KeyValuePair( - documentKey, ExtractDocument(document, titleFieldName, contentFieldName)); + documentKey, ExtractDocument(document, documentKey, titleFieldName, contentFieldName)); } } } } - private static SourceDocument ExtractDocument(SearchDocument doc, string titleFieldName, string contentFieldName) + private static SourceDocument ExtractDocument(SearchDocument doc, string documentKey, string titleFieldName, string contentFieldName) { string title = null; - string content; + string content = null; + var contentIsSerializedDocument = false; if (!string.IsNullOrEmpty(titleFieldName) && doc.TryGetValue(titleFieldName, out var titleValue)) { @@ -220,16 +221,15 @@ private static SourceDocument ExtractDocument(SearchDocument doc, string titleFi { content = contentValue?.ToString(); } - else + + if (string.IsNullOrEmpty(content)) { // Fallback: serialize the full document as content. content = JsonSerializer.Serialize(doc); + contentIsSerializedDocument = true; } - if (string.IsNullOrEmpty(title) && !string.IsNullOrEmpty(content)) - { - title = content.ExtractTitleFromContent(); - } + title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey); // Populate all source fields for filter field propagation. var fields = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Primitives/CrestApps.Core.Elasticsearch/Services/DataSourceElasticsearchDocumentReader.cs b/src/Primitives/CrestApps.Core.Elasticsearch/Services/DataSourceElasticsearchDocumentReader.cs index 495fe7c8..5985a2cb 100644 --- a/src/Primitives/CrestApps.Core.Elasticsearch/Services/DataSourceElasticsearchDocumentReader.cs +++ b/src/Primitives/CrestApps.Core.Elasticsearch/Services/DataSourceElasticsearchDocumentReader.cs @@ -94,6 +94,7 @@ public async IAsyncEnumerable> ReadAsync( key, ElasticsearchSourceDocumentMapper.ExtractDocument( hit.Source, + key, titleFieldPath, contentFieldPath, treatWhitespaceAsEmpty: false)); @@ -182,16 +183,18 @@ public async IAsyncEnumerable> ReadByIdsAsy key, ElasticsearchSourceDocumentMapper.ExtractDocument( hit.Source, + key, titleFieldPath, contentFieldPath, treatWhitespaceAsEmpty: false)); } } - private static SourceDocument ExtractDocument(JsonObject source, string titleFieldName, string contentFieldName) + private static SourceDocument ExtractDocument(JsonObject source, string documentKey, string titleFieldName, string contentFieldName) { return ElasticsearchSourceDocumentMapper.ExtractDocument( source, + documentKey, ElasticsearchSourceDocumentMapper.CreateFieldPath(titleFieldName), ElasticsearchSourceDocumentMapper.CreateFieldPath(contentFieldName), treatWhitespaceAsEmpty: false); diff --git a/src/Primitives/CrestApps.Core.Elasticsearch/Services/ElasticsearchSourceDocumentMapper.cs b/src/Primitives/CrestApps.Core.Elasticsearch/Services/ElasticsearchSourceDocumentMapper.cs index 49c310c5..575085ba 100644 --- a/src/Primitives/CrestApps.Core.Elasticsearch/Services/ElasticsearchSourceDocumentMapper.cs +++ b/src/Primitives/CrestApps.Core.Elasticsearch/Services/ElasticsearchSourceDocumentMapper.cs @@ -23,18 +23,21 @@ internal static ElasticsearchFieldPath CreateFieldPath(string fieldName) /// Extracts a source document from an Elasticsearch JSON source. /// /// The Elasticsearch JSON source. + /// The resolved document key used as the fallback title. /// The reusable title field path. /// The reusable content field path. /// A value indicating whether whitespace-only values should use fallback content. /// The extracted source document. internal static SourceDocument ExtractDocument( JsonObject source, + string documentKey, ElasticsearchFieldPath titleFieldPath, ElasticsearchFieldPath contentFieldPath, bool treatWhitespaceAsEmpty) { string title = null; string content = null; + var contentIsSerializedDocument = false; if (IsConfigured(titleFieldPath.OriginalName, treatWhitespaceAsEmpty)) { @@ -49,12 +52,10 @@ internal static SourceDocument ExtractDocument( if (IsMissing(content, treatWhitespaceAsEmpty)) { content = source.ToJsonString(); + contentIsSerializedDocument = true; } - if (IsMissing(title, treatWhitespaceAsEmpty) && !IsMissing(content, treatWhitespaceAsEmpty)) - { - title = content.ExtractTitleFromContent(); - } + title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey); var fields = new Dictionary(source.Count, StringComparer.OrdinalIgnoreCase); foreach (var property in source) diff --git a/src/Primitives/CrestApps.Core.PostgreSQL/Services/DataSourcePostgreSQLDocumentReader.cs b/src/Primitives/CrestApps.Core.PostgreSQL/Services/DataSourcePostgreSQLDocumentReader.cs index 4027917c..39c501fa 100644 --- a/src/Primitives/CrestApps.Core.PostgreSQL/Services/DataSourcePostgreSQLDocumentReader.cs +++ b/src/Primitives/CrestApps.Core.PostgreSQL/Services/DataSourcePostgreSQLDocumentReader.cs @@ -70,7 +70,7 @@ public async IAsyncEnumerable> ReadAsync( var key = ResolveKey(row, keyFieldName); yield return new KeyValuePair( - key, ExtractDocument(row, titleFieldName, contentFieldName)); + key, ExtractDocument(row, key, titleFieldName, contentFieldName)); } if (rowCount < BatchSize) @@ -134,7 +134,7 @@ public async IAsyncEnumerable> ReadByIdsAsy var key = ResolveKey(row, keyFieldName); yield return new KeyValuePair( - key, ExtractDocument(row, titleFieldName, contentFieldName)); + key, ExtractDocument(row, key, titleFieldName, contentFieldName)); } } @@ -167,10 +167,11 @@ private static string ResolveKey(Dictionary row, string keyField return null; } - private static SourceDocument ExtractDocument(Dictionary row, string titleFieldName, string contentFieldName) + private static SourceDocument ExtractDocument(Dictionary row, string documentKey, string titleFieldName, string contentFieldName) { string title = null; string content = null; + var contentIsSerializedDocument = false; if (!string.IsNullOrEmpty(titleFieldName) && row.TryGetValue(titleFieldName, out var titleValue) && titleValue != null) { @@ -191,12 +192,10 @@ private static SourceDocument ExtractDocument(Dictionary row, st } content = jsonObj.ToJsonString(); + contentIsSerializedDocument = true; } - if (string.IsNullOrEmpty(title) && !string.IsNullOrEmpty(content)) - { - title = content.ExtractTitleFromContent(); - } + title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey); var fields = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var kvp in row) diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Create.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Create.razor index 42636554..9f42aa7b 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Create.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Create.razor @@ -248,15 +248,15 @@
Field Mapping
- + -
The source index field that maps to the document key. Leave empty to use the native document ID.
+
The source index field that maps to the document key.
- + -
The source index field that maps to the document title.
+
The source index field that maps to the document title. Without it, citations fall back to the document ID.
@@ -320,6 +320,16 @@ _errors.Add("Source type is required."); } + if (string.IsNullOrWhiteSpace(model.KeyFieldName)) + { + _errors.Add("Key field name is required."); + } + + if (string.IsNullOrWhiteSpace(model.TitleFieldName)) + { + _errors.Add("Title field name is required."); + } + if (string.IsNullOrWhiteSpace(model.ContentFieldName)) { _errors.Add("Content field name is required."); diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Edit.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Edit.razor index 0b5e841a..f76898d3 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Edit.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/DataSources/AIDataSources/Edit.razor @@ -257,15 +257,15 @@ else
Field Mapping
- + -
The source index field that maps to the document key. Leave empty to use the native document ID.
+
The source index field that maps to the document key.
- + -
The source index field that maps to the document title.
+
The source index field that maps to the document title. Without it, citations fall back to the document ID.
@@ -348,6 +348,16 @@ else _errors.Add("Source type is required."); } + if (string.IsNullOrWhiteSpace(model.KeyFieldName)) + { + _errors.Add("Key field name is required."); + } + + if (string.IsNullOrWhiteSpace(model.TitleFieldName)) + { + _errors.Add("Title field name is required."); + } + if (string.IsNullOrWhiteSpace(model.ContentFieldName)) { _errors.Add("Content field name is required."); diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Controllers/AIDataSourceController.cs b/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Controllers/AIDataSourceController.cs index 14fbb99c..94e0e586 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Controllers/AIDataSourceController.cs +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Controllers/AIDataSourceController.cs @@ -174,6 +174,16 @@ private async Task ValidateAsync(AIDataSourceViewModel model) ModelState.AddModelError(nameof(model.Source), "Source type is required."); } + if (string.IsNullOrWhiteSpace(model.KeyFieldName)) + { + ModelState.AddModelError(nameof(model.KeyFieldName), "Key field name is required."); + } + + if (string.IsNullOrWhiteSpace(model.TitleFieldName)) + { + ModelState.AddModelError(nameof(model.TitleFieldName), "Title field name is required."); + } + if (string.IsNullOrWhiteSpace(model.ContentFieldName)) { ModelState.AddModelError(nameof(model.ContentFieldName), "Content field name is required."); diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Create.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Create.cshtml index 74b6af7f..01c612ae 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Create.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Create.cshtml @@ -216,17 +216,17 @@
Field Mapping
- - + + -
The source field that maps to the document key. Leave empty to use the native document ID.
+
The source field that maps to the document key.
- - + + -
The source field that maps to the document title.
+
The source field that maps to the document title. Without it, citations fall back to the document ID.
diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Edit.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Edit.cshtml index c03bc700..190c5cf4 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Edit.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/DataSources/Views/AIDataSource/Edit.cshtml @@ -217,17 +217,17 @@
Field Mapping
- - + + -
The source field that maps to the document key. Leave empty to use the native document ID.
+
The source field that maps to the document key.
- - + + -
The source field that maps to the document title.
+
The source field that maps to the document title. Without it, citations fall back to the document ID.
diff --git a/src/Utilities/CrestApps.Core.Support/DocumentTitleResolver.cs b/src/Utilities/CrestApps.Core.Support/DocumentTitleResolver.cs new file mode 100644 index 00000000..b63c6e87 --- /dev/null +++ b/src/Utilities/CrestApps.Core.Support/DocumentTitleResolver.cs @@ -0,0 +1,49 @@ +namespace CrestApps.Core.Support; + +/// +/// Resolves safe document titles when a data source does not map a dedicated title field. +/// +public static class DocumentTitleResolver +{ + /// + /// Resolves a display title for a document without ever exposing a serialized document as the title. + /// + /// The value read from the configured title field, when a title field is mapped. + /// The document content. + /// A value indicating whether is a serialized representation of the whole document rather than a mapped content field. + /// The document key used as the fallback title. + /// The resolved title, or when no safe title is available. + public static string Resolve(string mappedTitle, string content, bool contentIsSerializedDocument, string documentKey) + { + if (!string.IsNullOrWhiteSpace(mappedTitle)) + { + return mappedTitle; + } + + if (!contentIsSerializedDocument && !string.IsNullOrWhiteSpace(content) && !LooksLikeSerializedDocument(content)) + { + return content.ExtractTitleFromContent(); + } + + return string.IsNullOrWhiteSpace(documentKey) + ? null + : documentKey.Trim(); + } + + /// + /// Determines whether a value looks like a serialized JSON document or object graph. + /// + /// The value to inspect. + /// when the value looks like a serialized document; otherwise, . + public static bool LooksLikeSerializedDocument(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var span = value.AsSpan().TrimStart(); + + return span.Length > 0 && (span[0] == '{' || span[0] == '['); + } +} diff --git a/tests/CrestApps.Core.Tests/Core/Services/SearchProviderSourceDocumentMappingTests.cs b/tests/CrestApps.Core.Tests/Core/Services/SearchProviderSourceDocumentMappingTests.cs index e88dbcc7..33c04816 100644 --- a/tests/CrestApps.Core.Tests/Core/Services/SearchProviderSourceDocumentMappingTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Services/SearchProviderSourceDocumentMappingTests.cs @@ -43,6 +43,7 @@ public void PostgreSQLExtractDocument_ShouldPreserveMappedValues(Type mapperType mapperType, "ExtractDocument", row, + "document-1", "title", "body"); var key = Invoke( @@ -73,12 +74,14 @@ public void PostgreSQLExtractDocument_ShouldPreserveWhitespaceSemantics() PostgreSQLReaderType, "ExtractDocument", coreRow, + "document-1", "title", "body"); var aiDocument = Invoke( PostgreSQLHandlerType, "ExtractDocument", aiRow, + "document-1", "title", "body"); @@ -87,6 +90,56 @@ public void PostgreSQLExtractDocument_ShouldPreserveWhitespaceSemantics() Assert.Contains("\"body\":\" \"", aiDocument.Content, StringComparison.Ordinal); } + /// + /// Verifies PostgreSQL mapping falls back to the document key instead of the serialized document when no title is mapped. + /// + /// The provider mapper type. + [Theory] + [MemberData(nameof(PostgreSQLMapperTypes))] + public void PostgreSQLExtractDocument_WhenTitleIsNotMapped_ShouldUseDocumentKey(Type mapperType) + { + var row = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["id"] = "document-1", + ["Body"] = "Mapped content", + }; + + var document = Invoke( + mapperType, + "ExtractDocument", + row, + "document-1", + null, + null); + + Assert.Equal("document-1", document.Title); + } + + /// + /// Verifies Elasticsearch mapping falls back to the document key instead of the serialized document when no title is mapped. + /// + /// The provider mapper type. + [Theory] + [MemberData(nameof(ElasticsearchMapperTypes))] + public void ElasticsearchExtractDocument_WhenTitleIsNotMapped_ShouldUseDocumentKey(Type mapperType) + { + var source = new JsonObject + { + ["id"] = "document-1", + ["body"] = "Mapped content", + }; + + var document = Invoke( + mapperType, + "ExtractDocument", + source, + "document-1", + null, + null); + + Assert.Equal("document-1", document.Title); + } + /// /// Verifies Elasticsearch dotted paths preserve direct-property precedence over nested traversal. /// @@ -135,6 +188,7 @@ public void ElasticsearchExtractDocument_ShouldPreserveMappedValues(Type mapperT mapperType, "ExtractDocument", source, + "document-1", "content.title", "content.body"); @@ -158,12 +212,14 @@ public void ElasticsearchExtractDocument_ShouldPreserveWhitespaceSemantics() ElasticsearchReaderType, "ExtractDocument", coreSource, + "document-1", "title", "body"); var aiDocument = Invoke( ElasticsearchHandlerType, "ExtractDocument", aiSource, + "document-1", "title", "body");