Require data source field mapping and stop leaking documents as titles

benchmarks/CrestApps.Core.Benchmarks/ElasticsearchSourceDocumentMappingBenchmarks.cs

@@ -55,9 +55,10 @@ public void Setup()
         {
             var legacyKey = LegacyResolveKey(_sources[index]);
             var currentKey = CurrentResolveKey(_sources[index]);
-            var legacyDocument = LegacyExtractDocument(_sources[index], TitleFieldName, ContentFieldName);
+            var legacyDocument = LegacyExtractDocument(_sources[index], legacyKey, TitleFieldName, ContentFieldName);
             var currentDocument = ElasticsearchSourceDocumentMapper.ExtractDocument(
                 _sources[index],
+                currentKey,
                 _titleFieldPath,
                 _contentFieldPath,
                 treatWhitespaceAsEmpty: false);
@@ -78,7 +79,7 @@ public int Legacy()
         foreach (var source in _sources)
         {
             var key = LegacyResolveKey(source);
-            var document = LegacyExtractDocument(source, TitleFieldName, ContentFieldName);
+            var document = LegacyExtractDocument(source, key, TitleFieldName, ContentFieldName);
             checksum += key.Length + document.Title.Length + document.Content.Length + document.Fields.Count;
         }
 
@@ -99,6 +100,7 @@ public int Current()
             var key = CurrentResolveKey(source);
             var document = ElasticsearchSourceDocumentMapper.ExtractDocument(
                 source,
+                key,
                 _titleFieldPath,
                 _contentFieldPath,
                 treatWhitespaceAsEmpty: false);
@@ -120,11 +122,13 @@ private string CurrentResolveKey(JsonObject source)
 
     private static SourceDocument LegacyExtractDocument(
         JsonObject source,
+        string documentKey,
         string titleFieldName,
         string contentFieldName)
     {
         string title = null;
         string content = null;
+        var contentIsSerializedDocument = false;
 
         if (!string.IsNullOrEmpty(titleFieldName))
         {
@@ -141,12 +145,10 @@ private static SourceDocument LegacyExtractDocument(
         if (string.IsNullOrEmpty(content))
         {
             content = source.ToJsonString();
+            contentIsSerializedDocument = true;
         }
 
-        if (string.IsNullOrEmpty(title) && !string.IsNullOrEmpty(content))
-        {
-            title = content.ExtractTitleFromContent();
-        }
+        title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey);
 
         var fields = new Dictionary<string, object>(StringComparer.OrdinalIgnoreCase);
         foreach (var property in source)

src/CrestApps.Core.Docs/docs/changelog/v1.0.0.md

@@ -116,4 +116,5 @@ description: Initial standalone release notes for the CrestApps.Core repository.
 - promotes the parameterized AI tool instances into a first-class opt-in feature registered on the AI suite builder with `AddToolInstances(toolInstances => toolInstances.AddSource<TSource>(...))` (with an `AddHttpApiRequestSource()` convenience for the built-in source), instead of being wired into the core AI services automatically; persistence is registered on the tool-instances builder via `AddYesSqlStores()`/`AddEntityCoreStores()` rather than on the AI suite; drops the redundant per-instance `DisplayText` in favor of the unique `Name`, localizes the source `DisplayName`/`Description`/`Category`, simplifies `IAIToolInstanceSource.CreateTool(AIToolInstance instance)` to take the instance directly, generalizes the completion-context handler to honor tool instances on any resource so both AI profiles and chat interactions can select them, renames the built-in HTTP source's basic/OAuth credentials to `Username`/`Password` and adds the OAuth 2.0 resource-owner password grant, hardens model-provided paths so they cannot redirect a request off the configured host, and updates the MVC and Blazor sample hosts to let users attach instances to both AI profiles and chat interactions
 - namespaces every tool-instance function name with the `AIToolInstanceExtensions.FunctionNamePrefix` (`tool_instance_`) prefix so a user-chosen instance name can never collide with a tool registered in code via `AddCoreAITool` (which the model sees under its bare registered name); both kinds of tool now coexist safely in the single function namespace exposed to OpenAI, Azure OpenAI, and every other client
 - adds a source dropdown to the AI tool instance create form in the MVC and Blazor sample hosts that reveals only the selected source's fields (the source is fixed and shown read-only on edit), and relocates the "AI Tool Instances" admin menu item next to "AI Profiles" in both samples
+- requires the key, title, and content field mappings when creating or editing an AI data source in the MVC and Blazor sample hosts, and makes every built-in source reader fall back to the document key instead of the serialized source document when no title is mapped, so chat citations never render a full JSON document as a reference title
 - makes the Copilot CLI acquisition work behind corporate proxies and artifact mirrors, and downloads it only once per machine: `CrestApps.Core.AI.Copilot` now resolves the effective npm registry from `NPM_CONFIG_REGISTRY` or `npm config get registry` before the `GitHub.Copilot.SDK` targets download the CLI tarball (the SDK hardcodes `https://registry.npmjs.org`, and MSBuild's `DownloadFile` task cannot read npm configuration), and redirects the SDK's per-project, per-configuration cache to a shared cache under the NuGet global packages folder so a multi-project solution, a fresh worktree, or a CI agent no longer re-downloads the same large tarball for every project; both behaviors are opt-out through `CopilotResolveNpmRegistry` and `CopilotUseSharedCliCache`, the cache location is configurable through `CopilotCliCacheDir` (point it at a pre-seeded directory to build offline), and an explicitly set `CopilotNpmRegistryUrl`, `CopilotCliBinaryPath`, or `CopilotSkipCliDownload` always takes precedence

src/CrestApps.Core.Docs/docs/data-sources/index.md

@@ -43,6 +43,18 @@ The built-in source types are:
 
 `SearchIndexProfile` remains the default source type and the simplest option when your content is already indexed through CrestApps.Core.
 
+## Field Mapping
+
+Every `AIDataSource` maps three source fields:
+
+| Property | Purpose |
+| --- | --- |
+| `KeyFieldName` | The source field that identifies the document. It becomes the citation reference id. |
+| `TitleFieldName` | The source field used as the document title in citations and chat references. |
+| `ContentFieldName` | The source field holding the text body that is chunked and embedded. |
+
+The sample MVC and Blazor hosts require all three fields when creating or editing a data source. Map them explicitly: when a title is not mapped, the framework falls back to the document key and never uses the serialized source document as a title, so citations stay readable and never leak the full document payload.
+
 ## What is Vector Search?
 
 If you are new to AI-powered search, here is a brief primer on the concepts that data sources rely on.

src/Primitives/CrestApps.Core.AI.Azure.AISearch/Services/AzureAISearchAIDataSourceSourceHandler.cs

@@ -180,7 +180,7 @@ private static KeyValuePair<string, SourceDocument> CreateDocumentPair(AIDataSou
             key = configuredKeyValue?.ToString() ?? key;
         }
 
-        return new KeyValuePair<string, SourceDocument>(key, ExtractDocument(document, dataSource.TitleFieldName, dataSource.ContentFieldName));
+        return new KeyValuePair<string, SourceDocument>(key, ExtractDocument(document, key, dataSource.TitleFieldName, dataSource.ContentFieldName));
     }
 
     private (global::Azure.Search.Documents.SearchClient SearchClient, AzureAISearchSourceMetadata Metadata) Resolve(AIDataSource dataSource)
@@ -208,7 +208,7 @@ private static KeyValuePair<string, SourceDocument> CreateDocumentPair(AIDataSou
         return (client, metadata);
     }
 
-    private static SourceDocument ExtractDocument(SearchDocument document, string titleFieldName, string contentFieldName)
+    private static SourceDocument ExtractDocument(SearchDocument document, string documentKey, string titleFieldName, string contentFieldName)
     {
         string title = null;
         string content = null;
@@ -223,15 +223,15 @@ private static SourceDocument ExtractDocument(SearchDocument document, string ti
             content = contentValue?.ToString();
         }
 
+        var contentIsSerializedDocument = false;
+
         if (string.IsNullOrWhiteSpace(content))
         {
             content = System.Text.Json.JsonSerializer.Serialize(document);
+            contentIsSerializedDocument = true;
         }
 
-        if (string.IsNullOrWhiteSpace(title) && !string.IsNullOrWhiteSpace(content))
-        {
-            title = content.ExtractTitleFromContent();
-        }
+        title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey);
 
         var fields = new Dictionary<string, object>(StringComparer.OrdinalIgnoreCase);
         foreach (var kvp in document)

src/Primitives/CrestApps.Core.AI.Elasticsearch/Services/ElasticsearchAIDataSourceSourceHandler.cs

@@ -252,6 +252,7 @@ private static KeyValuePair<string, SourceDocument> CreateDocumentPair(
             key,
             ElasticsearchSourceDocumentMapper.ExtractDocument(
                 source,
+                key,
                 titleFieldPath,
                 contentFieldPath,
                 treatWhitespaceAsEmpty: true));
@@ -300,10 +301,11 @@ private static KeyValuePair<string, SourceDocument> CreateDocumentPair(
         return (client, metadata);
     }
 
-    private static SourceDocument ExtractDocument(System.Text.Json.Nodes.JsonObject source, string titleFieldName, string contentFieldName)
+    private static SourceDocument ExtractDocument(System.Text.Json.Nodes.JsonObject source, string documentKey, string titleFieldName, string contentFieldName)
     {
         return ElasticsearchSourceDocumentMapper.ExtractDocument(
             source,
+            documentKey,
             ElasticsearchSourceDocumentMapper.CreateFieldPath(titleFieldName),
             ElasticsearchSourceDocumentMapper.CreateFieldPath(contentFieldName),
             treatWhitespaceAsEmpty: true);

src/Primitives/CrestApps.Core.AI.PostgreSQL/Services/PostgreSQLAIDataSourceSourceHandler.cs

@@ -109,7 +109,7 @@ public async IAsyncEnumerable<KeyValuePair<string, SourceDocument>> ReadAsync(AI
                 var row = ReadRow(reader);
                 var key = ResolveKey(row, dataSource.KeyFieldName);
 
-                yield return new KeyValuePair<string, SourceDocument>(key, ExtractDocument(row, dataSource.TitleFieldName, dataSource.ContentFieldName));
+                yield return new KeyValuePair<string, SourceDocument>(key, ExtractDocument(row, key, dataSource.TitleFieldName, dataSource.ContentFieldName));
             }
 
             if (rowCount < 1000)
@@ -157,7 +157,7 @@ public async IAsyncEnumerable<KeyValuePair<string, SourceDocument>> ReadByIdsAsy
             var row = ReadRow(reader);
             var key = ResolveKey(row, dataSource.KeyFieldName);
 
-            yield return new KeyValuePair<string, SourceDocument>(key, ExtractDocument(row, dataSource.TitleFieldName, dataSource.ContentFieldName));
+            yield return new KeyValuePair<string, SourceDocument>(key, ExtractDocument(row, key, dataSource.TitleFieldName, dataSource.ContentFieldName));
         }
     }
 
@@ -213,10 +213,11 @@ private static string ResolveKey(Dictionary<string, object> row, string keyField
         return null;
     }
 
-    private static SourceDocument ExtractDocument(Dictionary<string, object> row, string titleFieldName, string contentFieldName)
+    private static SourceDocument ExtractDocument(Dictionary<string, object> row, string documentKey, string titleFieldName, string contentFieldName)
     {
         string title = null;
         string content = null;
+        var contentIsSerializedDocument = false;
 
         if (!string.IsNullOrWhiteSpace(titleFieldName) && row.TryGetValue(titleFieldName, out var titleValue) && titleValue != null)
         {
@@ -237,12 +238,10 @@ private static SourceDocument ExtractDocument(Dictionary<string, object> row, st
             }
 
             content = json.ToJsonString();
+            contentIsSerializedDocument = true;
         }
 
-        if (string.IsNullOrWhiteSpace(title) && !string.IsNullOrWhiteSpace(content))
-        {
-            title = content.ExtractTitleFromContent();
-        }
+        title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey);
 
         var fields = new Dictionary<string, object>(StringComparer.OrdinalIgnoreCase);
         foreach (var kvp in row)

src/Primitives/CrestApps.Core.AI/Handlers/DataSourcePreemptiveRagHandler.cs

@@ -8,6 +8,7 @@
 using CrestApps.Core.Infrastructure.Indexing;
 using CrestApps.Core.Infrastructure.Indexing.DataSources;
 using CrestApps.Core.Infrastructure.Indexing.Models;
+using CrestApps.Core.Support;
 using CrestApps.Core.Templates.Services;
 using Cysharp.Text;
 using Microsoft.Extensions.AI;
@@ -295,7 +296,7 @@ private async Task SearchAndInjectContextAsync(
             {
                 seenReferences[result.ReferenceId] = (
                     invocationContext?.NextReferenceIndex() ?? seenReferences.Count + 1,
-                    _textNormalizer.NormalizeTitle(result.Title),
+                    ResolveReferenceTitle(result.Title, result.ReferenceId),
                     result.ReferenceType);
             }
 
@@ -351,6 +352,24 @@ private async Task SearchAndInjectContextAsync(
         orchestrationContext.SystemMessageBuilder.Append(stringBuilder);
     }
 
+    /// <summary>
+    /// Resolves a citation title that never exposes a serialized source document.
+    /// </summary>
+    /// <param name="title">The indexed document title.</param>
+    /// <param name="referenceId">The document reference identifier used as the fallback title.</param>
+    /// <returns>The resolved citation title.</returns>
+    private string ResolveReferenceTitle(string title, string referenceId)
+    {
+        var normalizedTitle = _textNormalizer.NormalizeTitle(title);
+
+        if (string.IsNullOrWhiteSpace(normalizedTitle) || DocumentTitleResolver.LooksLikeSerializedDocument(normalizedTitle))
+        {
+            return referenceId;
+        }
+
+        return normalizedTitle;
+    }
+
     private static AIDataSourceRagMetadata GetRagMetadata(object resource)
     {
         if (resource is AIProfile profile &&

src/Primitives/CrestApps.Core.AI/Tools/DataSourceSearchTool.cs

@@ -9,6 +9,7 @@
 using CrestApps.Core.AI.Tooling;
 using CrestApps.Core.Infrastructure.Indexing;
 using CrestApps.Core.Infrastructure.Indexing.DataSources;
+using CrestApps.Core.Support;
 using Cysharp.Text;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.DependencyInjection;
@@ -241,20 +242,24 @@ protected override async ValueTask<object> InvokeCoreAsync(AIFunctionArguments a
 
                 if (!string.IsNullOrEmpty(result.ReferenceId) && !seenReferences.ContainsKey(result.ReferenceId))
                 {
-                    seenReferences[result.ReferenceId] = (invocationContext.NextReferenceIndex(), textNormalizer.NormalizeTitle(result.Title), result.ReferenceType);
+                    seenReferences[result.ReferenceId] = (invocationContext.NextReferenceIndex(), ResolveReferenceTitle(textNormalizer, result.Title, result.ReferenceId), result.ReferenceType);
                 }
 
                 var refLabel = !string.IsNullOrEmpty(result.ReferenceId) && seenReferences.TryGetValue(result.ReferenceId, out var entry)
                 ? $"[doc:{entry.Index}]"
                 : $"[doc:{invocationContext.NextReferenceIndex()}]";
 
+                var displayTitle = !string.IsNullOrEmpty(result.ReferenceId) && seenReferences.TryGetValue(result.ReferenceId, out var titleEntry)
+                    ? titleEntry.Title
+                    : ResolveReferenceTitle(textNormalizer, result.Title, result.ReferenceId);
+
                 builder.AppendLine("---");
 
-                if (!string.IsNullOrWhiteSpace(result.Title))
+                if (!string.IsNullOrWhiteSpace(displayTitle))
                 {
                     builder.Append(refLabel);
                     builder.Append(" Title: ");
-                    builder.AppendLine(result.Title);
+                    builder.AppendLine(displayTitle);
                 }
 
                 builder.Append(refLabel);
@@ -300,6 +305,25 @@ protected override async ValueTask<object> InvokeCoreAsync(AIFunctionArguments a
         }
     }
 
+    /// <summary>
+    /// Resolves a citation title that never exposes a serialized source document.
+    /// </summary>
+    /// <param name="textNormalizer">The text normalizer.</param>
+    /// <param name="title">The indexed document title.</param>
+    /// <param name="referenceId">The document reference identifier used as the fallback title.</param>
+    /// <returns>The resolved citation title.</returns>
+    private static string ResolveReferenceTitle(IAITextNormalizer textNormalizer, string title, string referenceId)
+    {
+        var normalizedTitle = textNormalizer.NormalizeTitle(title);
+
+        if (string.IsNullOrWhiteSpace(normalizedTitle) || DocumentTitleResolver.LooksLikeSerializedDocument(normalizedTitle))
+        {
+            return referenceId;
+        }
+
+        return normalizedTitle;
+    }
+
     private static AIDataSourceRagMetadata GetRagMetadata(AIToolExecutionContext executionContext)
     {
         if (executionContext.Resource is AIProfile profile && profile.TryGet<AIDataSourceRagMetadata>(out var ragMetadata))

src/Primitives/CrestApps.Core.Azure.AISearch/Services/DataSourceAzureAISearchDocumentReader.cs

@@ -91,7 +91,7 @@ public async IAsyncEnumerable<KeyValuePair<string, SourceDocument>> ReadAsync(
             if (!string.IsNullOrEmpty(documentKey))
             {
                 yield return new KeyValuePair<string, SourceDocument>(
-                    documentKey, ExtractDocument(doc, titleFieldName, contentFieldName));
+                    documentKey, ExtractDocument(doc, documentKey, titleFieldName, contentFieldName));
             }
         }
     }
@@ -165,7 +165,7 @@ public async IAsyncEnumerable<KeyValuePair<string, SourceDocument>> ReadByIdsAsy
                 if (!string.IsNullOrEmpty(documentKey))
                 {
                     yield return new KeyValuePair<string, SourceDocument>(
-                        documentKey, ExtractDocument(doc, titleFieldName, contentFieldName));
+                        documentKey, ExtractDocument(doc, documentKey, titleFieldName, contentFieldName));
                 }
             }
         }
@@ -200,16 +200,17 @@ public async IAsyncEnumerable<KeyValuePair<string, SourceDocument>> ReadByIdsAsy
                         : id;
 
                     yield return new KeyValuePair<string, SourceDocument>(
-                        documentKey, ExtractDocument(document, titleFieldName, contentFieldName));
+                        documentKey, ExtractDocument(document, documentKey, titleFieldName, contentFieldName));
                 }
             }
         }
     }
 
-    private static SourceDocument ExtractDocument(SearchDocument doc, string titleFieldName, string contentFieldName)
+    private static SourceDocument ExtractDocument(SearchDocument doc, string documentKey, string titleFieldName, string contentFieldName)
     {
         string title = null;
-        string content;
+        string content = null;
+        var contentIsSerializedDocument = false;
 
         if (!string.IsNullOrEmpty(titleFieldName) && doc.TryGetValue(titleFieldName, out var titleValue))
         {
@@ -220,16 +221,15 @@ private static SourceDocument ExtractDocument(SearchDocument doc, string titleFi
         {
             content = contentValue?.ToString();
         }
-        else
+
+        if (string.IsNullOrEmpty(content))
         {
             // Fallback: serialize the full document as content.
             content = JsonSerializer.Serialize(doc);
+            contentIsSerializedDocument = true;
         }
 
-        if (string.IsNullOrEmpty(title) && !string.IsNullOrEmpty(content))
-        {
-            title = content.ExtractTitleFromContent();
-        }
+        title = DocumentTitleResolver.Resolve(title, content, contentIsSerializedDocument, documentKey);
 
         // Populate all source fields for filter field propagation.
         var fields = new Dictionary<string, object>(StringComparer.OrdinalIgnoreCase);