From 18a29ffe9578f326d3c9659f5f3fd1d848643097 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 13:30:50 -0700 Subject: [PATCH 01/15] AB#32006 Add generic AI retry and output shape validation --- .../AI/OpenAIService.cs | 197 +++++++++++++++++- 1 file changed, 193 insertions(+), 4 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index a1cb4ba783..219c6052a8 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -36,6 +36,7 @@ public class OpenAIService : IAIService, ITransientDependency private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; private const string SummaryFailedRetryMessage = "AI analysis failed - please try again later."; + private const int MaxAiAttempts = 3; private string? ApiKey => _configuration["Azure:OpenAI:ApiKey"]; private string? ApiUrl => _configuration["Azure:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; @@ -118,7 +119,10 @@ public async Task GenerateApplicationAnalysisAsync( data, attachments); await LogPromptInputAsync(ApplicationAnalysisPromptType, promptVersion, systemPrompt, analysisContent); - var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + var raw = await GenerateWithRetryAsync( + () => GenerateSummaryAsync(analysisContent, systemPrompt, 1000), + IsValidApplicationAnalysisJson, + "application analysis"); await LogPromptOutputAsync(ApplicationAnalysisPromptType, promptVersion, raw); SavePromptCapture(capturePromptIo, request.CaptureContextId, ApplicationAnalysisPromptType, promptVersion, "Application Analysis", systemPrompt, analysisContent, raw); return ParseApplicationAnalysisResponse(AddIdsToAnalysisItems(raw)); @@ -233,7 +237,10 @@ public async Task GenerateAttachmentSummaryAsync(Atta var contentToAnalyze = BuildAttachmentUserPrompt(promptVersion, attachment); await LogPromptInputAsync(AttachmentSummaryPromptType, promptVersion, prompt, contentToAnalyze); - var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); + var modelOutput = await GenerateWithRetryAsync( + () => GenerateSummaryAsync(contentToAnalyze, prompt, 150), + IsValidNarrativeText, + "attachment summary"); await LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, modelOutput); SavePromptCapture(capturePromptIo, request.CaptureContextId, AttachmentSummaryPromptType, promptVersion, fileName, prompt, contentToAnalyze, modelOutput); @@ -334,7 +341,6 @@ public async Task GenerateScoresheetSectionAsync(Scor var attachmentSummaries = request.Attachments .Select(a => $"{a.Name}: {a.Summary}") .ToList(); - if (string.IsNullOrEmpty(ApiKey)) { _logger.LogWarning("{Message}", MissingApiKeyMessage); @@ -385,7 +391,10 @@ public async Task GenerateScoresheetSectionAsync(Scor var systemPrompt = BuildScoresheetSectionSystemPrompt(promptVersion); await LogPromptInputAsync(ScoresheetSectionPromptType, promptVersion, systemPrompt, analysisContent); - var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + var modelOutput = await GenerateWithRetryAsync( + () => GenerateSummaryAsync(analysisContent, systemPrompt, 2000), + content => IsValidScoresheetSectionJson(content, sectionJson), + $"scoresheet section {request.SectionName}"); await LogPromptOutputAsync(ScoresheetSectionPromptType, promptVersion, modelOutput); SavePromptCapture(capturePromptIo, request.CaptureContextId, ScoresheetSectionPromptType, promptVersion, request.SectionName, systemPrompt, analysisContent, modelOutput); @@ -398,6 +407,186 @@ public async Task GenerateScoresheetSectionAsync(Scor } } + private async Task GenerateWithRetryAsync( + Func> operation, + Func validator, + string operationName) + { + var lastResponse = string.Empty; + + for (var attempt = 1; attempt <= MaxAiAttempts; attempt++) + { + try + { + lastResponse = await operation(); + } + catch (Exception ex) when (attempt < MaxAiAttempts) + { + _logger.LogWarning(ex, "AI {OperationName} attempt {Attempt}/{MaxAttempts} failed; retrying", operationName, attempt, MaxAiAttempts); + continue; + } + + if (validator(lastResponse)) + { + return lastResponse; + } + + if (attempt < MaxAiAttempts) + { + _logger.LogWarning( + "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid output shape; retrying", + operationName, + attempt, + MaxAiAttempts); + } + } + + _logger.LogWarning("AI {OperationName} exhausted retries; returning last response", operationName); + return lastResponse; + } + + private static bool IsValidNarrativeText(string response) + { + return !string.IsNullOrWhiteSpace(response); + } + + private static bool IsValidApplicationAnalysisJson(string response) + { + if (!TryParseRootObject(response, out var root)) + { + return false; + } + + return root.TryGetProperty(AIJsonKeys.Rating, out var rating) + && rating.ValueKind == JsonValueKind.String + && root.TryGetProperty(AIJsonKeys.Errors, out var errors) + && errors.ValueKind == JsonValueKind.Array + && root.TryGetProperty(AIJsonKeys.Warnings, out var warnings) + && warnings.ValueKind == JsonValueKind.Array + && root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) + && summaries.ValueKind == JsonValueKind.Array + && root.TryGetProperty(AIJsonKeys.NextSteps, out var nextSteps) + && nextSteps.ValueKind == JsonValueKind.Array; + } + + private static bool IsValidScoresheetAnswersJson(string response) + { + if (!TryParseRootObject(response, out var root)) + { + return false; + } + + foreach (var property in root.EnumerateObject()) + { + if (property.Value.ValueKind != JsonValueKind.String && property.Value.ValueKind != JsonValueKind.Number) + { + return false; + } + } + + return true; + } + + private static bool IsValidScoresheetSectionJson(string response, string sectionJson) + { + if (!TryParseRootObject(response, out var root)) + { + return false; + } + + var expectedQuestionIds = ExtractQuestionIds(sectionJson); + if (expectedQuestionIds.Count == 0) + { + return false; + } + + foreach (var questionId in expectedQuestionIds) + { + if (!root.TryGetProperty(questionId, out var answerObject) || answerObject.ValueKind != JsonValueKind.Object) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Answer, out var answerValue) + || answerValue.ValueKind == JsonValueKind.Null + || answerValue.ValueKind == JsonValueKind.Object + || answerValue.ValueKind == JsonValueKind.Array) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Confidence, out var confidenceValue) + || confidenceValue.ValueKind != JsonValueKind.Number + || !confidenceValue.TryGetInt32(out var confidence) + || confidence < 0 + || confidence > 100) + { + return false; + } + } + + return true; + } + + private static HashSet ExtractQuestionIds(string sectionJson) + { + var ids = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + using var jsonDoc = JsonDocument.Parse(sectionJson); + if (jsonDoc.RootElement.ValueKind != JsonValueKind.Array) + { + return ids; + } + + foreach (var item in jsonDoc.RootElement.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Object + && item.TryGetProperty("id", out var idProperty) + && idProperty.ValueKind == JsonValueKind.String) + { + var id = idProperty.GetString(); + if (!string.IsNullOrWhiteSpace(id)) + { + ids.Add(id); + } + } + } + } + catch + { + return ids; + } + + return ids; + } + + private static bool TryParseRootObject(string response, out JsonElement root) + { + root = default; + + if (string.IsNullOrWhiteSpace(response)) + { + return false; + } + + try + { + using var jsonDoc = JsonDocument.Parse(CleanJsonResponse(response)); + if (jsonDoc.RootElement.ValueKind != JsonValueKind.Object) + { + return false; + } + + root = jsonDoc.RootElement.Clone(); + return true; + } + catch + { + return false; + } + } private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) { var response = new ApplicationAnalysisResponse(); From 4a843e019403d1d1e2faa45c801b84d635cdb392 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 18:34:10 -0800 Subject: [PATCH 02/15] AB#32006 Fix scoresheet retry validation for section object payload shape --- .../AI/OpenAIService.cs | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 219c6052a8..0b2d2b5ea4 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -535,23 +535,19 @@ private static HashSet ExtractQuestionIds(string sectionJson) try { using var jsonDoc = JsonDocument.Parse(sectionJson); - if (jsonDoc.RootElement.ValueKind != JsonValueKind.Array) + var root = jsonDoc.RootElement; + + if (root.ValueKind == JsonValueKind.Array) { + AddQuestionIds(root, ids); return ids; } - foreach (var item in jsonDoc.RootElement.EnumerateArray()) + if (root.ValueKind == JsonValueKind.Object && + root.TryGetProperty("questions", out var questionsElement) && + questionsElement.ValueKind == JsonValueKind.Array) { - if (item.ValueKind == JsonValueKind.Object - && item.TryGetProperty("id", out var idProperty) - && idProperty.ValueKind == JsonValueKind.String) - { - var id = idProperty.GetString(); - if (!string.IsNullOrWhiteSpace(id)) - { - ids.Add(id); - } - } + AddQuestionIds(questionsElement, ids); } } catch @@ -562,6 +558,25 @@ private static HashSet ExtractQuestionIds(string sectionJson) return ids; } + private static void AddQuestionIds(JsonElement questionsArray, ISet ids) + { + foreach (var item in questionsArray.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object || + !item.TryGetProperty("id", out var idProperty) || + idProperty.ValueKind != JsonValueKind.String) + { + continue; + } + + var id = idProperty.GetString(); + if (!string.IsNullOrWhiteSpace(id)) + { + ids.Add(id); + } + } + } + private static bool TryParseRootObject(string response, out JsonElement root) { root = default; From 82e3d8306cb5d65e083007feaab1c77d679d8b74 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 19:10:17 -0800 Subject: [PATCH 03/15] AB#32006 Address Sonar loop/value and HashSet parameter suggestions --- .../src/Unity.GrantManager.Application/AI/OpenAIService.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 0b2d2b5ea4..56eea246e4 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -476,9 +476,9 @@ private static bool IsValidScoresheetAnswersJson(string response) return false; } - foreach (var property in root.EnumerateObject()) + foreach (var value in root.EnumerateObject().Select(property => property.Value)) { - if (property.Value.ValueKind != JsonValueKind.String && property.Value.ValueKind != JsonValueKind.Number) + if (value.ValueKind != JsonValueKind.String && value.ValueKind != JsonValueKind.Number) { return false; } @@ -558,7 +558,7 @@ private static HashSet ExtractQuestionIds(string sectionJson) return ids; } - private static void AddQuestionIds(JsonElement questionsArray, ISet ids) + private static void AddQuestionIds(JsonElement questionsArray, HashSet ids) { foreach (var item in questionsArray.EnumerateArray()) { From ee8db04a9926ca9046d222cd894a3cdea9076dce Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 19:21:01 -0800 Subject: [PATCH 04/15] AB#32006 Refactor scoresheet value-kind validation to LINQ predicate --- .../AI/OpenAIService.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 56eea246e4..187b5000eb 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -476,15 +476,9 @@ private static bool IsValidScoresheetAnswersJson(string response) return false; } - foreach (var value in root.EnumerateObject().Select(property => property.Value)) - { - if (value.ValueKind != JsonValueKind.String && value.ValueKind != JsonValueKind.Number) - { - return false; - } - } - - return true; + return root.EnumerateObject() + .Select(value => value.Value.ValueKind) + .All(kind => kind == JsonValueKind.String || kind == JsonValueKind.Number); } private static bool IsValidScoresheetSectionJson(string response, string sectionJson) From 7705947faac86f601caa090a45e98742422d9ecc Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 13:32:47 -0700 Subject: [PATCH 05/15] AB#32006 extract AI response validation helpers --- .../AI/AIResponseJson.cs | 63 +++++ .../AI/AIResponseValidator.cs | 151 ++++++++++++ .../AI/OpenAIService.cs | 223 +----------------- 3 files changed, 220 insertions(+), 217 deletions(-) create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseJson.cs create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseJson.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseJson.cs new file mode 100644 index 0000000000..13c591f0f2 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseJson.cs @@ -0,0 +1,63 @@ +using System; + +namespace Unity.GrantManager.AI +{ + internal static class AIResponseJson + { + public static string CleanJsonResponse(string response) + { + if (string.IsNullOrWhiteSpace(response)) + { + return string.Empty; + } + + var cleaned = response.Trim(); + + if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) + { + var startIndex = cleaned.IndexOf('\n'); + if (startIndex >= 0) + { + cleaned = cleaned[(startIndex + 1)..]; + } + else + { + var jsonStart = FindFirstJsonTokenIndex(cleaned); + if (jsonStart > 0) + { + cleaned = cleaned[jsonStart..]; + } + } + } + + if (cleaned.EndsWith("```", StringComparison.Ordinal)) + { + var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); + if (lastIndex > 0) + { + cleaned = cleaned[..lastIndex]; + } + } + + return cleaned.Trim(); + } + + private static int FindFirstJsonTokenIndex(string value) + { + var objectStart = value.IndexOf('{'); + var arrayStart = value.IndexOf('['); + + if (objectStart >= 0 && arrayStart >= 0) + { + return Math.Min(objectStart, arrayStart); + } + + if (objectStart >= 0) + { + return objectStart; + } + + return arrayStart; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs new file mode 100644 index 0000000000..0ea4155655 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; + +namespace Unity.GrantManager.AI +{ + internal static class AIResponseValidator + { + public static bool IsValidAttachmentSummaryText(string response) + { + return !string.IsNullOrWhiteSpace(response); + } + + public static bool IsValidApplicationAnalysisJson(string response) + { + if (!TryParseRootObject(response, out var root)) + { + return false; + } + + return root.TryGetProperty(AIJsonKeys.Rating, out var rating) + && rating.ValueKind == JsonValueKind.String + && root.TryGetProperty(AIJsonKeys.Errors, out var errors) + && errors.ValueKind == JsonValueKind.Array + && root.TryGetProperty(AIJsonKeys.Warnings, out var warnings) + && warnings.ValueKind == JsonValueKind.Array + && root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) + && summaries.ValueKind == JsonValueKind.Array + && root.TryGetProperty(AIJsonKeys.NextSteps, out var nextSteps) + && nextSteps.ValueKind == JsonValueKind.Array; + } + + public static bool IsValidScoresheetSectionJson(string response, string sectionJson) + { + if (!TryParseRootObject(response, out var root)) + { + return false; + } + + var expectedQuestionIds = ExtractQuestionIds(sectionJson); + if (expectedQuestionIds.Count == 0) + { + return false; + } + + foreach (var questionId in expectedQuestionIds) + { + if (!root.TryGetProperty(questionId, out var answerObject) || answerObject.ValueKind != JsonValueKind.Object) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Answer, out var answerValue) + || answerValue.ValueKind == JsonValueKind.Null + || answerValue.ValueKind == JsonValueKind.Object + || answerValue.ValueKind == JsonValueKind.Array) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Confidence, out var confidenceValue) + || confidenceValue.ValueKind != JsonValueKind.Number + || !confidenceValue.TryGetInt32(out var confidence) + || confidence < 0 + || confidence > 100) + { + return false; + } + } + + return true; + } + + private static HashSet ExtractQuestionIds(string sectionJson) + { + var ids = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + using var jsonDoc = JsonDocument.Parse(sectionJson); + var root = jsonDoc.RootElement; + + if (root.ValueKind == JsonValueKind.Array) + { + AddQuestionIds(root, ids); + return ids; + } + + if (root.ValueKind == JsonValueKind.Object && + root.TryGetProperty("questions", out var questionsElement) && + questionsElement.ValueKind == JsonValueKind.Array) + { + AddQuestionIds(questionsElement, ids); + } + } + catch + { + return ids; + } + + return ids; + } + + private static void AddQuestionIds(JsonElement questionsArray, HashSet ids) + { + foreach (var item in questionsArray.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object || + !item.TryGetProperty("id", out var idProperty) || + idProperty.ValueKind != JsonValueKind.String) + { + continue; + } + + var id = idProperty.GetString(); + if (!string.IsNullOrWhiteSpace(id)) + { + ids.Add(id); + } + } + } + + private static bool TryParseRootObject(string response, out JsonElement root) + { + root = default; + + if (string.IsNullOrWhiteSpace(response)) + { + return false; + } + + try + { + using var jsonDoc = JsonDocument.Parse(AIResponseJson.CleanJsonResponse(response)); + if (jsonDoc.RootElement.ValueKind != JsonValueKind.Object) + { + return false; + } + + root = jsonDoc.RootElement.Clone(); + return true; + } + catch + { + return false; + } + } + + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 187b5000eb..637d7046a8 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -121,7 +121,7 @@ public async Task GenerateApplicationAnalysisAsync( await LogPromptInputAsync(ApplicationAnalysisPromptType, promptVersion, systemPrompt, analysisContent); var raw = await GenerateWithRetryAsync( () => GenerateSummaryAsync(analysisContent, systemPrompt, 1000), - IsValidApplicationAnalysisJson, + AIResponseValidator.IsValidApplicationAnalysisJson, "application analysis"); await LogPromptOutputAsync(ApplicationAnalysisPromptType, promptVersion, raw); SavePromptCapture(capturePromptIo, request.CaptureContextId, ApplicationAnalysisPromptType, promptVersion, "Application Analysis", systemPrompt, analysisContent, raw); @@ -239,7 +239,7 @@ public async Task GenerateAttachmentSummaryAsync(Atta await LogPromptInputAsync(AttachmentSummaryPromptType, promptVersion, prompt, contentToAnalyze); var modelOutput = await GenerateWithRetryAsync( () => GenerateSummaryAsync(contentToAnalyze, prompt, 150), - IsValidNarrativeText, + AIResponseValidator.IsValidAttachmentSummaryText, "attachment summary"); await LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, modelOutput); SavePromptCapture(capturePromptIo, request.CaptureContextId, AttachmentSummaryPromptType, promptVersion, fileName, prompt, contentToAnalyze, modelOutput); @@ -393,7 +393,7 @@ public async Task GenerateScoresheetSectionAsync(Scor await LogPromptInputAsync(ScoresheetSectionPromptType, promptVersion, systemPrompt, analysisContent); var modelOutput = await GenerateWithRetryAsync( () => GenerateSummaryAsync(analysisContent, systemPrompt, 2000), - content => IsValidScoresheetSectionJson(content, sectionJson), + content => AIResponseValidator.IsValidScoresheetSectionJson(content, sectionJson), $"scoresheet section {request.SectionName}"); await LogPromptOutputAsync(ScoresheetSectionPromptType, promptVersion, modelOutput); SavePromptCapture(capturePromptIo, request.CaptureContextId, ScoresheetSectionPromptType, promptVersion, request.SectionName, systemPrompt, analysisContent, modelOutput); @@ -422,7 +422,7 @@ private async Task GenerateWithRetryAsync( } catch (Exception ex) when (attempt < MaxAiAttempts) { - _logger.LogWarning(ex, "AI {OperationName} attempt {Attempt}/{MaxAttempts} failed; retrying", operationName, attempt, MaxAiAttempts); + _logger.LogWarning(ex, "AI {OperationName} attempt {Attempt}/{MaxAttempts} request failed; retrying", operationName, attempt, MaxAiAttempts); continue; } @@ -434,7 +434,7 @@ private async Task GenerateWithRetryAsync( if (attempt < MaxAiAttempts) { _logger.LogWarning( - "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid output shape; retrying", + "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid response shape; retrying", operationName, attempt, MaxAiAttempts); @@ -445,157 +445,6 @@ private async Task GenerateWithRetryAsync( return lastResponse; } - private static bool IsValidNarrativeText(string response) - { - return !string.IsNullOrWhiteSpace(response); - } - - private static bool IsValidApplicationAnalysisJson(string response) - { - if (!TryParseRootObject(response, out var root)) - { - return false; - } - - return root.TryGetProperty(AIJsonKeys.Rating, out var rating) - && rating.ValueKind == JsonValueKind.String - && root.TryGetProperty(AIJsonKeys.Errors, out var errors) - && errors.ValueKind == JsonValueKind.Array - && root.TryGetProperty(AIJsonKeys.Warnings, out var warnings) - && warnings.ValueKind == JsonValueKind.Array - && root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) - && summaries.ValueKind == JsonValueKind.Array - && root.TryGetProperty(AIJsonKeys.NextSteps, out var nextSteps) - && nextSteps.ValueKind == JsonValueKind.Array; - } - - private static bool IsValidScoresheetAnswersJson(string response) - { - if (!TryParseRootObject(response, out var root)) - { - return false; - } - - return root.EnumerateObject() - .Select(value => value.Value.ValueKind) - .All(kind => kind == JsonValueKind.String || kind == JsonValueKind.Number); - } - - private static bool IsValidScoresheetSectionJson(string response, string sectionJson) - { - if (!TryParseRootObject(response, out var root)) - { - return false; - } - - var expectedQuestionIds = ExtractQuestionIds(sectionJson); - if (expectedQuestionIds.Count == 0) - { - return false; - } - - foreach (var questionId in expectedQuestionIds) - { - if (!root.TryGetProperty(questionId, out var answerObject) || answerObject.ValueKind != JsonValueKind.Object) - { - return false; - } - - if (!answerObject.TryGetProperty(AIJsonKeys.Answer, out var answerValue) - || answerValue.ValueKind == JsonValueKind.Null - || answerValue.ValueKind == JsonValueKind.Object - || answerValue.ValueKind == JsonValueKind.Array) - { - return false; - } - - if (!answerObject.TryGetProperty(AIJsonKeys.Confidence, out var confidenceValue) - || confidenceValue.ValueKind != JsonValueKind.Number - || !confidenceValue.TryGetInt32(out var confidence) - || confidence < 0 - || confidence > 100) - { - return false; - } - } - - return true; - } - - private static HashSet ExtractQuestionIds(string sectionJson) - { - var ids = new HashSet(StringComparer.OrdinalIgnoreCase); - - try - { - using var jsonDoc = JsonDocument.Parse(sectionJson); - var root = jsonDoc.RootElement; - - if (root.ValueKind == JsonValueKind.Array) - { - AddQuestionIds(root, ids); - return ids; - } - - if (root.ValueKind == JsonValueKind.Object && - root.TryGetProperty("questions", out var questionsElement) && - questionsElement.ValueKind == JsonValueKind.Array) - { - AddQuestionIds(questionsElement, ids); - } - } - catch - { - return ids; - } - - return ids; - } - - private static void AddQuestionIds(JsonElement questionsArray, HashSet ids) - { - foreach (var item in questionsArray.EnumerateArray()) - { - if (item.ValueKind != JsonValueKind.Object || - !item.TryGetProperty("id", out var idProperty) || - idProperty.ValueKind != JsonValueKind.String) - { - continue; - } - - var id = idProperty.GetString(); - if (!string.IsNullOrWhiteSpace(id)) - { - ids.Add(id); - } - } - } - - private static bool TryParseRootObject(string response, out JsonElement root) - { - root = default; - - if (string.IsNullOrWhiteSpace(response)) - { - return false; - } - - try - { - using var jsonDoc = JsonDocument.Parse(CleanJsonResponse(response)); - if (jsonDoc.RootElement.ValueKind != JsonValueKind.Object) - { - return false; - } - - root = jsonDoc.RootElement.Clone(); - return true; - } - catch - { - return false; - } - } private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) { var response = new ApplicationAnalysisResponse(); @@ -897,7 +746,7 @@ private static string FormatPromptOutputForLog(string output) private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) { objectElement = default; - var cleaned = CleanJsonResponse(response); + var cleaned = AIResponseJson.CleanJsonResponse(response); if (string.IsNullOrWhiteSpace(cleaned)) { return false; @@ -920,66 +769,6 @@ private static bool TryParseJsonObjectFromResponse(string response, out JsonElem } } - private static string CleanJsonResponse(string response) - { - if (string.IsNullOrWhiteSpace(response)) - { - return string.Empty; - } - - var cleaned = response.Trim(); - - if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) - { - var startIndex = cleaned.IndexOf('\n'); - if (startIndex >= 0) - { - // Multi-line fenced code block: remove everything up to and including the first newline. - cleaned = cleaned[(startIndex + 1)..]; - } - else - { - // Single-line fenced JSON, e.g. ```json { ... } ``` or ```{ ... } ```. - // Strip everything before the first likely JSON payload token. - var jsonStart = FindFirstJsonTokenIndex(cleaned); - - if (jsonStart > 0) - { - cleaned = cleaned[jsonStart..]; - } - } - } - - if (cleaned.EndsWith("```", StringComparison.Ordinal)) - { - var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); - if (lastIndex > 0) - { - cleaned = cleaned[..lastIndex]; - } - } - - return cleaned.Trim(); - } - - private static int FindFirstJsonTokenIndex(string value) - { - var objectStart = value.IndexOf('{'); - var arrayStart = value.IndexOf('['); - - if (objectStart >= 0 && arrayStart >= 0) - { - return Math.Min(objectStart, arrayStart); - } - - if (objectStart >= 0) - { - return objectStart; - } - - return arrayStart; - } - private static string ResolvePromptVersion(string? version) { if (!string.IsNullOrWhiteSpace(version) && From 19e1be91e3ff4f48c60fd90be34ed42509ac4065 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Thu, 12 Mar 2026 08:15:59 -0700 Subject: [PATCH 06/15] AB#32006 remove unused validator import --- .../src/Unity.GrantManager.Application/AI/AIResponseValidator.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs index 0ea4155655..7d80341dfb 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIResponseValidator.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Text.Json; namespace Unity.GrantManager.AI From 3a6a75002e8d487cf7a814799b5fbac3458c8438 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 13:55:28 -0700 Subject: [PATCH 07/15] AB#32006 refactor AI retry flow to typed outcomes --- .../AI/AIOperationResult.cs | 21 +++ .../AI/OpenAIService.cs | 165 +++++++++++++----- 2 files changed, 141 insertions(+), 45 deletions(-) create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs new file mode 100644 index 0000000000..96db197787 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs @@ -0,0 +1,21 @@ +namespace Unity.GrantManager.AI +{ + internal enum AIOperationOutcome + { + Success, + TransientFailure, + PermanentFailure, + InvalidOutput + } + + internal sealed record AIOperationResult(AIOperationOutcome Outcome, string Content) + { + public static AIOperationResult Success(string content) => new(AIOperationOutcome.Success, content); + + public static AIOperationResult TransientFailure(string content = "") => new(AIOperationOutcome.TransientFailure, content); + + public static AIOperationResult PermanentFailure(string content = "") => new(AIOperationOutcome.PermanentFailure, content); + + public static AIOperationResult InvalidOutput(string content = "") => new(AIOperationOutcome.InvalidOutput, content); + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 637d7046a8..87a931a90d 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Text; using System.Text.Json; @@ -32,7 +33,6 @@ public class OpenAIService : IAIService, ITransientDependency private const string AttachmentUserTemplateName = "attachment.user"; private const string ScoresheetSystemTemplateName = "scoresheet.system"; private const string ScoresheetUserTemplateName = "scoresheet.user"; - private const string NoSummaryGeneratedMessage = "No summary generated."; private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; private const string SummaryFailedRetryMessage = "AI analysis failed - please try again later."; @@ -87,12 +87,15 @@ public Task IsAvailableAsync() public async Task GenerateCompletionAsync(AICompletionRequest request) { - var content = await GenerateSummaryAsync( + var result = await GenerateWithRetryAsync( + () => GenerateSummaryAsync( request?.UserPrompt ?? string.Empty, null, request?.MaxTokens ?? 150, - request?.Temperature); - return new AICompletionResponse { Content = content }; + request?.Temperature), + AIResponseValidator.IsValidAttachmentSummaryText, + "completion"); + return new AICompletionResponse { Content = ResolveNarrativeContent(result) }; } public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) @@ -119,16 +122,22 @@ public async Task GenerateApplicationAnalysisAsync( data, attachments); await LogPromptInputAsync(ApplicationAnalysisPromptType, promptVersion, systemPrompt, analysisContent); - var raw = await GenerateWithRetryAsync( + var result = await GenerateWithRetryAsync( () => GenerateSummaryAsync(analysisContent, systemPrompt, 1000), AIResponseValidator.IsValidApplicationAnalysisJson, "application analysis"); - await LogPromptOutputAsync(ApplicationAnalysisPromptType, promptVersion, raw); - SavePromptCapture(capturePromptIo, request.CaptureContextId, ApplicationAnalysisPromptType, promptVersion, "Application Analysis", systemPrompt, analysisContent, raw); - return ParseApplicationAnalysisResponse(AddIdsToAnalysisItems(raw)); + await LogPromptOutputAsync(ApplicationAnalysisPromptType, promptVersion, result.Content); + SavePromptCapture(capturePromptIo, request.CaptureContextId, ApplicationAnalysisPromptType, promptVersion, "Application Analysis", systemPrompt, analysisContent, result.Content); + + if (result.Outcome != AIOperationOutcome.Success) + { + return new ApplicationAnalysisResponse(); + } + + return ParseApplicationAnalysisResponse(AddIdsToAnalysisItems(result.Content)); } - private async Task GenerateSummaryAsync( + private async Task GenerateSummaryAsync( string content, string? systemPrompt, int maxTokens = 150, @@ -137,7 +146,7 @@ private async Task GenerateSummaryAsync( if (string.IsNullOrEmpty(ApiKey)) { _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); - return ServiceNotConfiguredMessage; + return AIOperationResult.PermanentFailure(MissingApiKeyMessage); } _logger.LogDebug("Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", content?.Length ?? 0, maxTokens); @@ -177,28 +186,39 @@ private async Task GenerateSummaryAsync( if (!response.IsSuccessStatusCode) { _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); - return ServiceTemporarilyUnavailableMessage; + return MapFailureOutcome(response.StatusCode, responseContent); } if (string.IsNullOrWhiteSpace(responseContent)) { - return NoSummaryGeneratedMessage; + return AIOperationResult.InvalidOutput(); } - using var jsonDoc = JsonDocument.Parse(responseContent); - var choices = jsonDoc.RootElement.GetProperty("choices"); - if (choices.GetArrayLength() > 0) + try { - var message = choices[0].GetProperty("message"); - return message.GetProperty("content").GetString() ?? NoSummaryGeneratedMessage; - } + using var jsonDoc = JsonDocument.Parse(responseContent); + var choices = jsonDoc.RootElement.GetProperty("choices"); + if (choices.GetArrayLength() > 0) + { + var message = choices[0].GetProperty("message"); + var modelOutput = message.GetProperty("content").GetString(); + return string.IsNullOrWhiteSpace(modelOutput) + ? AIOperationResult.InvalidOutput(responseContent) + : AIOperationResult.Success(modelOutput); + } - return NoSummaryGeneratedMessage; + return AIOperationResult.InvalidOutput(responseContent); + } + catch (Exception ex) when (ex is JsonException || ex is KeyNotFoundException || ex is InvalidOperationException) + { + _logger.LogWarning(ex, "AI response payload had an invalid output shape"); + return AIOperationResult.InvalidOutput(responseContent); + } } catch (Exception ex) { _logger.LogError(ex, "Error generating AI summary"); - return SummaryFailedRetryMessage; + return AIOperationResult.TransientFailure(ex.Message); } } @@ -237,16 +257,24 @@ public async Task GenerateAttachmentSummaryAsync(Atta var contentToAnalyze = BuildAttachmentUserPrompt(promptVersion, attachment); await LogPromptInputAsync(AttachmentSummaryPromptType, promptVersion, prompt, contentToAnalyze); - var modelOutput = await GenerateWithRetryAsync( + var result = await GenerateWithRetryAsync( () => GenerateSummaryAsync(contentToAnalyze, prompt, 150), AIResponseValidator.IsValidAttachmentSummaryText, "attachment summary"); - await LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, modelOutput); - SavePromptCapture(capturePromptIo, request.CaptureContextId, AttachmentSummaryPromptType, promptVersion, fileName, prompt, contentToAnalyze, modelOutput); + await LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, result.Content); + SavePromptCapture(capturePromptIo, request.CaptureContextId, AttachmentSummaryPromptType, promptVersion, fileName, prompt, contentToAnalyze, result.Content); + + if (result.Outcome != AIOperationOutcome.Success) + { + return new AttachmentSummaryResponse + { + Summary = $"AI analysis not available for this attachment ({fileName})." + }; + } return new AttachmentSummaryResponse { - Summary = ExtractSummaryFromJson(modelOutput) + Summary = ExtractSummaryFromJson(result.Content) }; } catch (Exception ex) @@ -391,14 +419,19 @@ public async Task GenerateScoresheetSectionAsync(Scor var systemPrompt = BuildScoresheetSectionSystemPrompt(promptVersion); await LogPromptInputAsync(ScoresheetSectionPromptType, promptVersion, systemPrompt, analysisContent); - var modelOutput = await GenerateWithRetryAsync( + var result = await GenerateWithRetryAsync( () => GenerateSummaryAsync(analysisContent, systemPrompt, 2000), content => AIResponseValidator.IsValidScoresheetSectionJson(content, sectionJson), $"scoresheet section {request.SectionName}"); - await LogPromptOutputAsync(ScoresheetSectionPromptType, promptVersion, modelOutput); - SavePromptCapture(capturePromptIo, request.CaptureContextId, ScoresheetSectionPromptType, promptVersion, request.SectionName, systemPrompt, analysisContent, modelOutput); + await LogPromptOutputAsync(ScoresheetSectionPromptType, promptVersion, result.Content); + SavePromptCapture(capturePromptIo, request.CaptureContextId, ScoresheetSectionPromptType, promptVersion, request.SectionName, systemPrompt, analysisContent, result.Content); - return ParseScoresheetSectionResponse(modelOutput); + if (result.Outcome != AIOperationOutcome.Success) + { + return new ScoresheetSectionResponse(); + } + + return ParseScoresheetSectionResponse(result.Content); } catch (Exception ex) { @@ -407,42 +440,84 @@ public async Task GenerateScoresheetSectionAsync(Scor } } - private async Task GenerateWithRetryAsync( - Func> operation, + private async Task GenerateWithRetryAsync( + Func> operation, Func validator, string operationName) { - var lastResponse = string.Empty; + var lastResult = AIOperationResult.InvalidOutput(); for (var attempt = 1; attempt <= MaxAiAttempts; attempt++) { - try + lastResult = await operation(); + + if (lastResult.Outcome == AIOperationOutcome.Success && validator(lastResult.Content)) { - lastResponse = await operation(); + return lastResult; } - catch (Exception ex) when (attempt < MaxAiAttempts) + + if (lastResult.Outcome == AIOperationOutcome.Success) { - _logger.LogWarning(ex, "AI {OperationName} attempt {Attempt}/{MaxAttempts} request failed; retrying", operationName, attempt, MaxAiAttempts); - continue; + lastResult = AIOperationResult.InvalidOutput(lastResult.Content); } - if (validator(lastResponse)) + if (lastResult.Outcome == AIOperationOutcome.PermanentFailure) { - return lastResponse; + return lastResult; } if (attempt < MaxAiAttempts) { - _logger.LogWarning( - "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid response shape; retrying", - operationName, - attempt, - MaxAiAttempts); + if (lastResult.Outcome == AIOperationOutcome.TransientFailure) + { + _logger.LogWarning( + "AI {OperationName} attempt {Attempt}/{MaxAttempts} failed transiently; retrying", + operationName, + attempt, + MaxAiAttempts); + } + else if (lastResult.Outcome == AIOperationOutcome.InvalidOutput) + { + _logger.LogWarning( + "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid response shape; retrying", + operationName, + attempt, + MaxAiAttempts); + } } } - _logger.LogWarning("AI {OperationName} exhausted retries; returning last response", operationName); - return lastResponse; + _logger.LogWarning( + "AI {OperationName} exhausted retries with outcome {Outcome}; returning last result", + operationName, + lastResult.Outcome); + return lastResult; + } + + private static string ResolveNarrativeContent(AIOperationResult result) + { + return result.Outcome switch + { + AIOperationOutcome.Success => result.Content, + AIOperationOutcome.PermanentFailure => ServiceNotConfiguredMessage, + AIOperationOutcome.TransientFailure => ServiceTemporarilyUnavailableMessage, + _ => SummaryFailedRetryMessage + }; + } + + private static AIOperationResult MapFailureOutcome(HttpStatusCode statusCode, string? responseContent) + { + var content = responseContent ?? string.Empty; + var statusCodeValue = (int)statusCode; + + if (statusCode == HttpStatusCode.RequestTimeout + || statusCode == (HttpStatusCode)429 + || statusCodeValue >= 500) + { + return AIOperationResult.TransientFailure(content); + } + + return AIOperationResult.PermanentFailure(content); } private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) From 33c666bfc6c55ff7a8a613619f5de829589927cb Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 14:07:29 -0700 Subject: [PATCH 08/15] AB#32006 make AI token limit parameter configurable --- .../AI/OpenAIService.cs | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 87a931a90d..c7c76c29de 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -37,9 +37,12 @@ public class OpenAIService : IAIService, ITransientDependency private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; private const string SummaryFailedRetryMessage = "AI analysis failed - please try again later."; private const int MaxAiAttempts = 3; + private const string DefaultMaxTokensParameterName = "max_completion_tokens"; + private const string LegacyMaxTokensParameterName = "max_tokens"; private string? ApiKey => _configuration["Azure:OpenAI:ApiKey"]; private string? ApiUrl => _configuration["Azure:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; + private string MaxTokensParameterName => ResolveMaxTokensParameterName(_configuration["Azure:OpenAI:MaxTokensParameter"]); private readonly string MissingApiKeyMessage = "OpenAI API key is not configured"; // Optional local debugging sink for prompt payload logs to a local file. @@ -164,12 +167,17 @@ private async Task GenerateSummaryAsync( { new { role = "system", content = resolvedSystemPrompt }, new { role = "user", content = userPrompt } - }, - max_tokens = maxTokens, - temperature = temperature ?? 0.3 + } }; - var json = JsonSerializer.Serialize(requestBody); + var requestPayload = new Dictionary + { + ["messages"] = requestBody.messages, + [MaxTokensParameterName] = maxTokens, + ["temperature"] = temperature ?? 0.3 + }; + + var json = JsonSerializer.Serialize(requestPayload); var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); _httpClient.DefaultRequestHeaders.Clear(); @@ -520,6 +528,16 @@ private static AIOperationResult MapFailureOutcome(HttpStatusCode statusCode, st return AIOperationResult.PermanentFailure(content); } + private static string ResolveMaxTokensParameterName(string? configuredParameterName) + { + if (string.Equals(configuredParameterName, LegacyMaxTokensParameterName, StringComparison.Ordinal)) + { + return LegacyMaxTokensParameterName; + } + + return DefaultMaxTokensParameterName; + } + private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) { var response = new ApplicationAnalysisResponse(); From 6198c05f7f9c2c9e17bd01db8f0a3c0bb68ec30b Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 15:57:19 -0700 Subject: [PATCH 09/15] AB#32006 capture AI provider responses and operation defaults --- .../AI/AIOperationResult.cs | 22 +- .../AI/AIProviderResponse.cs | 17 + .../AI/AIProviderResponseMetadata.cs | 10 + .../AI/OpenAIService.cs | 319 +++++++++++++++--- .../AI/Prompts/Versions/README.md | 2 +- .../AI/AIPromptToolViewOptionsProvider.cs | 4 +- 6 files changed, 326 insertions(+), 48 deletions(-) create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponse.cs create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponseMetadata.cs diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs index 96db197787..f1ffbda307 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIOperationResult.cs @@ -8,14 +8,26 @@ internal enum AIOperationOutcome InvalidOutput } - internal sealed record AIOperationResult(AIOperationOutcome Outcome, string Content) + internal sealed record AIOperationResult( + AIOperationOutcome Outcome, + AIProviderResponse Response) { - public static AIOperationResult Success(string content) => new(AIOperationOutcome.Success, content); + public string Content => Response.Content; - public static AIOperationResult TransientFailure(string content = "") => new(AIOperationOutcome.TransientFailure, content); + public string CaptureOutput => Response.CaptureOutput; - public static AIOperationResult PermanentFailure(string content = "") => new(AIOperationOutcome.PermanentFailure, content); + public static AIOperationResult Success(AIProviderResponse? response = null) => + new(AIOperationOutcome.Success, response ?? AIProviderResponse.Empty); - public static AIOperationResult InvalidOutput(string content = "") => new(AIOperationOutcome.InvalidOutput, content); + public static AIOperationResult TransientFailure(AIProviderResponse? response = null) => + new(AIOperationOutcome.TransientFailure, response ?? AIProviderResponse.Empty); + + public static AIOperationResult PermanentFailure(AIProviderResponse? response = null) => + new(AIOperationOutcome.PermanentFailure, response ?? AIProviderResponse.Empty); + + public static AIOperationResult InvalidOutput(AIProviderResponse? response = null) => + new(AIOperationOutcome.InvalidOutput, response ?? AIProviderResponse.Empty); + + public AIOperationResult WithOutcome(AIOperationOutcome outcome) => new(outcome, Response); } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponse.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponse.cs new file mode 100644 index 0000000000..973af81507 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponse.cs @@ -0,0 +1,17 @@ +namespace Unity.GrantManager.AI +{ + internal sealed record AIProviderResponse( + string Content, + string RawResponse = "", + string? Model = null, + string? FinishReason = null, + int? PromptTokens = null, + int? CompletionTokens = null, + int? TotalTokens = null, + int? ReasoningTokens = null) + { + public static AIProviderResponse Empty { get; } = new(string.Empty); + + public string CaptureOutput => string.IsNullOrWhiteSpace(RawResponse) ? Content : RawResponse; + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponseMetadata.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponseMetadata.cs new file mode 100644 index 0000000000..34e75b7d0e --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/AIProviderResponseMetadata.cs @@ -0,0 +1,10 @@ +namespace Unity.GrantManager.AI +{ + internal sealed record AIProviderResponseMetadata( + string? Model, + string? FinishReason, + int? PromptTokens, + int? CompletionTokens, + int? TotalTokens, + int? ReasoningTokens); +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index c7c76c29de..f69a0266da 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -39,10 +39,15 @@ public class OpenAIService : IAIService, ITransientDependency private const int MaxAiAttempts = 3; private const string DefaultMaxTokensParameterName = "max_completion_tokens"; private const string LegacyMaxTokensParameterName = "max_tokens"; - - private string? ApiKey => _configuration["Azure:OpenAI:ApiKey"]; - private string? ApiUrl => _configuration["Azure:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; - private string MaxTokensParameterName => ResolveMaxTokensParameterName(_configuration["Azure:OpenAI:MaxTokensParameter"]); + private const string DefaultProviderName = "OpenAI"; + private const int DefaultCompletionTokens = 150; + private const int DefaultAttachmentSummaryCompletionTokens = 500; + private const int DefaultApplicationAnalysisCompletionTokens = 2500; + private const int DefaultScoresheetSectionCompletionTokens = 5000; + + private int AttachmentSummaryCompletionTokens => ResolveCompletionTokens("AttachmentSummary", DefaultAttachmentSummaryCompletionTokens); + private int ApplicationAnalysisCompletionTokens => ResolveCompletionTokens("ApplicationAnalysis", DefaultApplicationAnalysisCompletionTokens); + private int ScoresheetSectionCompletionTokens => ResolveCompletionTokens("ScoresheetSection", DefaultScoresheetSectionCompletionTokens); private readonly string MissingApiKeyMessage = "OpenAI API key is not configured"; // Optional local debugging sink for prompt payload logs to a local file. @@ -61,8 +66,6 @@ public class OpenAIService : IAIService, ITransientDependency }; private static readonly ConcurrentDictionary PromptTemplateCache = new(StringComparer.OrdinalIgnoreCase); - private string SelectedPromptVersion => ResolvePromptVersion(_configuration["Azure:OpenAI:PromptVersion"]); - public OpenAIService( HttpClient httpClient, IConfiguration configuration, @@ -79,7 +82,7 @@ public OpenAIService( public Task IsAvailableAsync() { - if (string.IsNullOrEmpty(ApiKey)) + if (string.IsNullOrEmpty(ResolveApiKey())) { _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); return Task.FromResult(false); @@ -94,7 +97,7 @@ public async Task GenerateCompletionAsync(AICompletionRequ () => GenerateSummaryAsync( request?.UserPrompt ?? string.Empty, null, - request?.MaxTokens ?? 150, + request?.MaxTokens ?? DefaultCompletionTokens, request?.Temperature), AIResponseValidator.IsValidAttachmentSummaryText, "completion"); @@ -104,7 +107,7 @@ public async Task GenerateCompletionAsync(AICompletionRequ public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) { ArgumentNullException.ThrowIfNull(request); - var promptVersion = ResolvePromptVersion(request.PromptVersion ?? SelectedPromptVersion); + var promptVersion = ResolvePromptVersion(request.PromptVersion ?? ResolvePromptVersionSetting(ApplicationAnalysisPromptType)); var capturePromptIo = request.CapturePromptIo; var data = JsonSerializer.Serialize(request.Data, JsonLogOptions); var schema = JsonSerializer.Serialize(request.Schema, JsonLogOptions); @@ -126,11 +129,15 @@ public async Task GenerateApplicationAnalysisAsync( attachments); await LogPromptInputAsync(ApplicationAnalysisPromptType, promptVersion, systemPrompt, analysisContent); var result = await GenerateWithRetryAsync( - () => GenerateSummaryAsync(analysisContent, systemPrompt, 1000), + () => GenerateSummaryAsync( + analysisContent, + systemPrompt, + ApplicationAnalysisCompletionTokens, + operationName: ApplicationAnalysisPromptType), AIResponseValidator.IsValidApplicationAnalysisJson, "application analysis"); - await LogPromptOutputAsync(ApplicationAnalysisPromptType, promptVersion, result.Content); - SavePromptCapture(capturePromptIo, request.CaptureContextId, ApplicationAnalysisPromptType, promptVersion, "Application Analysis", systemPrompt, analysisContent, result.Content); + await LogPromptOutputAsync(ApplicationAnalysisPromptType, promptVersion, result.CaptureOutput); + SavePromptCapture(capturePromptIo, request.CaptureContextId, ApplicationAnalysisPromptType, promptVersion, "Application Analysis", systemPrompt, analysisContent, result.CaptureOutput); if (result.Outcome != AIOperationOutcome.Success) { @@ -144,12 +151,21 @@ private async Task GenerateSummaryAsync( string content, string? systemPrompt, int maxTokens = 150, - double? temperature = null) + double? temperature = null, + string? operationName = null) { - if (string.IsNullOrEmpty(ApiKey)) + var providerName = ResolveProviderName(operationName); + if (!string.Equals(providerName, DefaultProviderName, StringComparison.Ordinal)) + { + _logger.LogWarning("Provider {ProviderName} is not supported by OpenAIService.", providerName); + return AIOperationResult.PermanentFailure(new AIProviderResponse($"Unsupported provider: {providerName}")); + } + + var apiKey = ResolveApiKey(operationName); + if (string.IsNullOrEmpty(apiKey)) { _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); - return AIOperationResult.PermanentFailure(MissingApiKeyMessage); + return AIOperationResult.PermanentFailure(new AIProviderResponse(MissingApiKeyMessage)); } _logger.LogDebug("Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", content?.Length ?? 0, maxTokens); @@ -173,33 +189,41 @@ private async Task GenerateSummaryAsync( var requestPayload = new Dictionary { ["messages"] = requestBody.messages, - [MaxTokensParameterName] = maxTokens, - ["temperature"] = temperature ?? 0.3 + [ResolveMaxTokensParameterNameForOperation(operationName)] = maxTokens }; + var resolvedTemperature = temperature ?? ResolveConfiguredTemperature(operationName); + if (resolvedTemperature.HasValue) + { + requestPayload["temperature"] = resolvedTemperature.Value; + } + var json = JsonSerializer.Serialize(requestPayload); var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); _httpClient.DefaultRequestHeaders.Clear(); - _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); + _httpClient.DefaultRequestHeaders.Add("Authorization", apiKey); - var response = await _httpClient.PostAsync(ApiUrl, httpContent); + var response = await _httpClient.PostAsync(ResolveApiUrl(operationName), httpContent); var responseContent = await response.Content.ReadAsStringAsync(); + var metadata = TryExtractProviderMetadata(responseContent); + var providerResponse = BuildProviderResponseFromMetadata(string.Empty, responseContent, metadata); _logger.LogDebug( "OpenAI chat completions response received. StatusCode: {StatusCode}, ResponseLength: {ResponseLength}", response.StatusCode, responseContent?.Length ?? 0); + LogProviderMetadata(operationName, providerResponse); if (!response.IsSuccessStatusCode) { _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); - return MapFailureOutcome(response.StatusCode, responseContent); + return MapFailureOutcome(response.StatusCode, providerResponse); } if (string.IsNullOrWhiteSpace(responseContent)) { - return AIOperationResult.InvalidOutput(); + return AIOperationResult.InvalidOutput(providerResponse); } try @@ -211,22 +235,22 @@ private async Task GenerateSummaryAsync( var message = choices[0].GetProperty("message"); var modelOutput = message.GetProperty("content").GetString(); return string.IsNullOrWhiteSpace(modelOutput) - ? AIOperationResult.InvalidOutput(responseContent) - : AIOperationResult.Success(modelOutput); + ? AIOperationResult.InvalidOutput(providerResponse) + : AIOperationResult.Success(BuildProviderResponseFromMetadata(modelOutput, responseContent, metadata)); } - return AIOperationResult.InvalidOutput(responseContent); + return AIOperationResult.InvalidOutput(providerResponse); } catch (Exception ex) when (ex is JsonException || ex is KeyNotFoundException || ex is InvalidOperationException) { _logger.LogWarning(ex, "AI response payload had an invalid output shape"); - return AIOperationResult.InvalidOutput(responseContent); + return AIOperationResult.InvalidOutput(providerResponse); } } catch (Exception ex) { _logger.LogError(ex, "Error generating AI summary"); - return AIOperationResult.TransientFailure(ex.Message); + return AIOperationResult.TransientFailure(new AIProviderResponse(ex.Message)); } } @@ -236,7 +260,7 @@ public async Task GenerateAttachmentSummaryAsync(Atta var fileName = request.FileName ?? string.Empty; var fileContent = request.FileContent ?? Array.Empty(); var contentType = request.ContentType ?? "application/octet-stream"; - var promptVersion = ResolvePromptVersion(request.PromptVersion ?? SelectedPromptVersion); + var promptVersion = ResolvePromptVersion(request.PromptVersion ?? ResolvePromptVersionSetting(AttachmentSummaryPromptType)); var capturePromptIo = request.CapturePromptIo; try @@ -266,11 +290,15 @@ public async Task GenerateAttachmentSummaryAsync(Atta await LogPromptInputAsync(AttachmentSummaryPromptType, promptVersion, prompt, contentToAnalyze); var result = await GenerateWithRetryAsync( - () => GenerateSummaryAsync(contentToAnalyze, prompt, 150), + () => GenerateSummaryAsync( + contentToAnalyze, + prompt, + AttachmentSummaryCompletionTokens, + operationName: AttachmentSummaryPromptType), AIResponseValidator.IsValidAttachmentSummaryText, "attachment summary"); - await LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, result.Content); - SavePromptCapture(capturePromptIo, request.CaptureContextId, AttachmentSummaryPromptType, promptVersion, fileName, prompt, contentToAnalyze, result.Content); + await LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, result.CaptureOutput); + SavePromptCapture(capturePromptIo, request.CaptureContextId, AttachmentSummaryPromptType, promptVersion, fileName, prompt, contentToAnalyze, result.CaptureOutput); if (result.Outcome != AIOperationOutcome.Success) { @@ -369,7 +397,7 @@ private string AddIdsToAnalysisItems(string analysisJson) public async Task GenerateScoresheetSectionAsync(ScoresheetSectionRequest request) { ArgumentNullException.ThrowIfNull(request); - var promptVersion = ResolvePromptVersion(request.PromptVersion ?? SelectedPromptVersion); + var promptVersion = ResolvePromptVersion(request.PromptVersion ?? ResolvePromptVersionSetting(ScoresheetSectionPromptType)); var capturePromptIo = request.CapturePromptIo; var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); var sectionJson = JsonSerializer.Serialize(request.SectionSchema, JsonLogOptions); @@ -377,7 +405,7 @@ public async Task GenerateScoresheetSectionAsync(Scor var attachmentSummaries = request.Attachments .Select(a => $"{a.Name}: {a.Summary}") .ToList(); - if (string.IsNullOrEmpty(ApiKey)) + if (string.IsNullOrEmpty(ResolveApiKey(ScoresheetSectionPromptType))) { _logger.LogWarning("{Message}", MissingApiKeyMessage); return new ScoresheetSectionResponse(); @@ -428,11 +456,15 @@ public async Task GenerateScoresheetSectionAsync(Scor await LogPromptInputAsync(ScoresheetSectionPromptType, promptVersion, systemPrompt, analysisContent); var result = await GenerateWithRetryAsync( - () => GenerateSummaryAsync(analysisContent, systemPrompt, 2000), + () => GenerateSummaryAsync( + analysisContent, + systemPrompt, + ScoresheetSectionCompletionTokens, + operationName: ScoresheetSectionPromptType), content => AIResponseValidator.IsValidScoresheetSectionJson(content, sectionJson), $"scoresheet section {request.SectionName}"); - await LogPromptOutputAsync(ScoresheetSectionPromptType, promptVersion, result.Content); - SavePromptCapture(capturePromptIo, request.CaptureContextId, ScoresheetSectionPromptType, promptVersion, request.SectionName, systemPrompt, analysisContent, result.Content); + await LogPromptOutputAsync(ScoresheetSectionPromptType, promptVersion, result.CaptureOutput); + SavePromptCapture(capturePromptIo, request.CaptureContextId, ScoresheetSectionPromptType, promptVersion, request.SectionName, systemPrompt, analysisContent, result.CaptureOutput); if (result.Outcome != AIOperationOutcome.Success) { @@ -466,7 +498,7 @@ private async Task GenerateWithRetryAsync( if (lastResult.Outcome == AIOperationOutcome.Success) { - lastResult = AIOperationResult.InvalidOutput(lastResult.Content); + lastResult = lastResult.WithOutcome(AIOperationOutcome.InvalidOutput); } if (lastResult.Outcome == AIOperationOutcome.PermanentFailure) @@ -513,19 +545,115 @@ private static string ResolveNarrativeContent(AIOperationResult result) }; } - private static AIOperationResult MapFailureOutcome(HttpStatusCode statusCode, string? responseContent) + private static AIOperationResult MapFailureOutcome(HttpStatusCode statusCode, AIProviderResponse response) { - var content = responseContent ?? string.Empty; var statusCodeValue = (int)statusCode; if (statusCode == HttpStatusCode.RequestTimeout || statusCode == (HttpStatusCode)429 || statusCodeValue >= 500) { - return AIOperationResult.TransientFailure(content); + return AIOperationResult.TransientFailure(response); } - return AIOperationResult.PermanentFailure(content); + return AIOperationResult.PermanentFailure(response); + } + + private static AIProviderResponse BuildProviderResponseFromMetadata(string content, string? rawResponse, AIProviderResponseMetadata? metadata) + { + return new AIProviderResponse( + content, + rawResponse ?? string.Empty, + metadata?.Model, + metadata?.FinishReason, + metadata?.PromptTokens, + metadata?.CompletionTokens, + metadata?.TotalTokens, + metadata?.ReasoningTokens); + } + + private static AIProviderResponseMetadata? TryExtractProviderMetadata(string? responseContent) + { + if (string.IsNullOrWhiteSpace(responseContent)) + { + return null; + } + + try + { + using var jsonDoc = JsonDocument.Parse(responseContent); + var root = jsonDoc.RootElement; + var model = root.TryGetProperty("model", out var modelProp) && modelProp.ValueKind == JsonValueKind.String + ? modelProp.GetString() + : null; + + string? finishReason = null; + if (root.TryGetProperty("choices", out var choices) + && choices.ValueKind == JsonValueKind.Array + && choices.GetArrayLength() > 0) + { + var firstChoice = choices[0]; + if (firstChoice.TryGetProperty("finish_reason", out var finishReasonProp) && finishReasonProp.ValueKind == JsonValueKind.String) + { + finishReason = finishReasonProp.GetString(); + } + } + + int? promptTokens = null; + int? completionTokens = null; + int? totalTokens = null; + int? reasoningTokens = null; + if (root.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object) + { + promptTokens = TryGetInt32(usage, "prompt_tokens"); + completionTokens = TryGetInt32(usage, "completion_tokens"); + totalTokens = TryGetInt32(usage, "total_tokens"); + + if (usage.TryGetProperty("completion_tokens_details", out var completionTokenDetails) + && completionTokenDetails.ValueKind == JsonValueKind.Object) + { + reasoningTokens = TryGetInt32(completionTokenDetails, "reasoning_tokens"); + } + } + + return new AIProviderResponseMetadata(model, finishReason, promptTokens, completionTokens, totalTokens, reasoningTokens); + } + catch (JsonException) + { + return null; + } + } + + private void LogProviderMetadata(string? operationName, AIProviderResponse response) + { + if (string.IsNullOrWhiteSpace(response.Model) + && string.IsNullOrWhiteSpace(response.FinishReason) + && response.PromptTokens == null + && response.CompletionTokens == null + && response.TotalTokens == null + && response.ReasoningTokens == null) + { + return; + } + + _logger.LogDebug( + "AI provider response metadata for {OperationName}: Model={Model}, FinishReason={FinishReason}, PromptTokens={PromptTokens}, CompletionTokens={CompletionTokens}, TotalTokens={TotalTokens}, ReasoningTokens={ReasoningTokens}", + operationName ?? "completion", + response.Model, + response.FinishReason, + response.PromptTokens, + response.CompletionTokens, + response.TotalTokens, + response.ReasoningTokens); + } + + private static int? TryGetInt32(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var property) + && property.ValueKind == JsonValueKind.Number + && property.TryGetInt32(out var value) + ? value + : null; } private static string ResolveMaxTokensParameterName(string? configuredParameterName) @@ -538,6 +666,117 @@ private static string ResolveMaxTokensParameterName(string? configuredParameterN return DefaultMaxTokensParameterName; } + private int ResolveCompletionTokens(string operationName, int defaultValue) + { + var configuredValue = _configuration.GetValue($"Azure:Operations:{operationName}:MaxCompletionTokens"); + if (configuredValue is > 0) + { + return configuredValue.Value; + } + + var defaultConfiguredValue = _configuration.GetValue("Azure:Operations:Defaults:MaxCompletionTokens"); + return defaultConfiguredValue is > 0 ? defaultConfiguredValue.Value : defaultValue; + } + + private string? ResolvePromptVersionSetting(string operationName) + { + var operationPromptVersion = _configuration[$"Azure:Operations:{operationName}:PromptVersion"]; + if (!string.IsNullOrWhiteSpace(operationPromptVersion)) + { + return operationPromptVersion; + } + + return _configuration["Azure:Operations:Defaults:PromptVersion"]; + } + + private string ResolveProviderName(string? operationName = null) + { + if (!string.IsNullOrWhiteSpace(operationName)) + { + var configuredProvider = _configuration[$"Azure:Operations:{operationName}:Provider"]; + if (!string.IsNullOrWhiteSpace(configuredProvider)) + { + return configuredProvider.Trim(); + } + } + + var defaultProvider = _configuration["Azure:Operations:Defaults:Provider"]; + return string.IsNullOrWhiteSpace(defaultProvider) ? DefaultProviderName : defaultProvider.Trim(); + } + + private string? ResolveApiKey(string? operationName = null) + { + var providerName = ResolveProviderName(operationName); + return _configuration[$"Azure:{providerName}:ApiKey"]; + } + + private string ResolveMaxTokensParameterNameForOperation(string? operationName = null) + { + var providerName = ResolveProviderName(operationName); + var profileName = ResolveProfileName(operationName, providerName); + var profileParameterName = ResolveProfileSetting(providerName, profileName, "MaxTokensParameter"); + return ResolveMaxTokensParameterName(profileParameterName); + } + + private double? ResolveConfiguredTemperature(string? operationName = null) + { + var providerName = ResolveProviderName(operationName); + var profileName = ResolveProfileName(operationName, providerName); + var profileTemperature = ResolveProfileSetting(providerName, profileName, "Temperature"); + if (profileTemperature != null + && double.TryParse(profileTemperature, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var parsedTemperature)) + { + return parsedTemperature; + } + + return null; + } + + private string ResolveApiUrl(string? operationName) + { + if (!string.IsNullOrWhiteSpace(operationName)) + { + var operationApiUrl = _configuration[$"Azure:Operations:{operationName}:ApiUrl"]; + if (!string.IsNullOrWhiteSpace(operationApiUrl)) + { + return operationApiUrl; + } + } + + var providerName = ResolveProviderName(operationName); + var profileName = ResolveProfileName(operationName, providerName); + var profileApiUrl = ResolveProfileSetting(providerName, profileName, "ApiUrl"); + return profileApiUrl + ?? _configuration[$"Azure:{providerName}:ApiUrl"] + ?? "https://api.openai.com/v1/chat/completions"; + } + + private string? ResolveProfileName(string? operationName, string providerName) + { + if (!string.IsNullOrWhiteSpace(operationName)) + { + var operationProfile = _configuration[$"Azure:Operations:{operationName}:Profile"]; + if (!string.IsNullOrWhiteSpace(operationProfile)) + { + return operationProfile.Trim(); + } + } + + var defaultProfile = _configuration["Azure:Operations:Defaults:Profile"]; + return string.IsNullOrWhiteSpace(defaultProfile) ? null : defaultProfile.Trim(); + } + + private string? ResolveProfileSetting(string providerName, string? profileName, string settingName) + { + if (string.IsNullOrWhiteSpace(profileName)) + { + return null; + } + + var profileSetting = _configuration[$"Azure:{providerName}:Profiles:{profileName}:{settingName}"]; + return string.IsNullOrWhiteSpace(profileSetting) ? null : profileSetting; + } + private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) { var response = new ApplicationAnalysisResponse(); diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md index cc8d06ef5f..76f1ca12d0 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md @@ -40,7 +40,7 @@ Placeholders: Version selection: -- `Azure:OpenAI:PromptVersion = v0|v1` +- `Azure:Operations:Defaults:PromptVersion = v0|v1, with optional overrides under Azure:Operations::PromptVersion` - Unknown or missing version defaults to `v1`. Template loading is strict: diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs index 650eee29ee..2a4b3c470a 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs @@ -13,8 +13,8 @@ public class AIPromptToolViewOptionsProvider( string.Equals(webHostEnvironment.EnvironmentName, "Development", StringComparison.OrdinalIgnoreCase); public string DefaultPromptVersion => - string.IsNullOrWhiteSpace(configuration["Azure:OpenAI:PromptVersion"]) + string.IsNullOrWhiteSpace(configuration["Azure:Operations:Defaults:PromptVersion"]) ? "v1" - : configuration["Azure:OpenAI:PromptVersion"]!.Trim().ToLowerInvariant(); + : configuration["Azure:Operations:Defaults:PromptVersion"]!.Trim().ToLowerInvariant(); } } From fa997fcb4a91c83b4e1b931c83ff95f4e118c114 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 15:57:37 -0700 Subject: [PATCH 10/15] AB#32006 standardize reviewer-facing AI prompt wording --- .../AI/Prompts/Versions/v1/analysis.rules.txt | 3 ++- .../AI/Prompts/Versions/v1/attachment.rules.txt | 3 ++- .../AI/Prompts/Versions/v1/scoresheet.rules.txt | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt index ee9c1cade7..09c04ecc0f 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt @@ -7,6 +7,8 @@ - Prefer direct evidence from DATA over derivative statements in ATTACHMENTS when both address the same point. - If ATTACHMENTS evidence is used, cite the attachment by name in detail. - Each detail must cite concrete evidence from DATA or ATTACHMENTS. +- Write reviewer-facing natural language. Do not refer to prompt section names, internal field keys, or schema labels such as DATA, ATTACHMENTS, ProjectSummary, CustomField1, or OrganizationType. +- Refer to evidence by its plain-language meaning, quoted text, or attachment name rather than internal key names. - Only include warnings when the evidence shows a specific, concrete risk, inconsistency, or meaningful uncertainty; a stated risk label alone is not enough. - Do not state that one amount exceeds, matches, or conflicts with another unless the comparison is directly supported by the provided values. - Do not treat ordinary lack of detailed supporting explanation as a material gap unless the provided evidence creates real uncertainty about feasibility, eligibility, or budget credibility. @@ -29,4 +31,3 @@ - Use HOLD only when provided evidence shows a material eligibility, feasibility, budget, or readiness concern that would reasonably block scoring or decision-making. - recommendation.rationale must explain the high-level recommendation in 1-2 complete sentences using provided evidence. - recommendation.rationale should name the 1-3 strongest evidence-based reasons for the recommendation. - diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.rules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.rules.txt index 0cebe3aa94..2230e39228 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.rules.txt +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.rules.txt @@ -6,7 +6,8 @@ - Begin with what the attachment contains or provides, not the file name or file type, unless that metadata is necessary to describe the evidence. - Do not invent missing details. - Do not calculate or restate totals, sums, or aggregates unless they are explicitly present in ATTACHMENT.text. +- Write reviewer-facing natural language. Do not refer to prompt section names, internal field keys, or schema labels such as ATTACHMENT or ATTACHMENT.text. +- Refer to evidence by its plain-language meaning, quoted text, or file name rather than internal key names. - Write 1-2 complete sentences. - Summary must be grounded in concrete ATTACHMENT evidence. - Return exactly one object with only the key: summary. - diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.rules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.rules.txt index 7c25c7f6bd..81a4132069 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.rules.txt +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.rules.txt @@ -19,7 +19,9 @@ - Never omit "answer", "rationale", or "confidence" for any question type. - The "answer" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. - The "rationale" field must be 1-2 complete sentences grounded in concrete DATA/ATTACHMENTS evidence. -- In rationale, cite concrete source evidence from the provided input content rather than prompt section headers. +- In rationale, cite concrete source evidence from the provided input content in plain language rather than prompt section headers or internal field names. +- Write reviewer-facing natural language. Do not refer to prompt section names, internal field keys, or schema labels such as DATA, ATTACHMENTS, ProjectSummary, CustomField1, or OrganizationType. +- Refer to evidence by its plain-language meaning, quoted text, or attachment name rather than internal key names. - For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. - The "confidence" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. - Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. From ad3176d6f1657cb83f9328b5c45e0783b0d8bac3 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 17:14:27 -0700 Subject: [PATCH 11/15] AB#32006 simplify prompt capture output and legacy config fallback --- .../AI/Responses/AIPromptCaptureResponse.cs | 7 +- .../AI/OpenAIService.cs | 113 +++++++++++++++++- .../AI/Prompts/Versions/README.md | 3 +- .../AI/AIPromptToolViewOptionsProvider.cs | 19 ++- .../Pages/GrantApplications/Details.js | 9 +- 5 files changed, 133 insertions(+), 18 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/Responses/AIPromptCaptureResponse.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/Responses/AIPromptCaptureResponse.cs index fc1fac75f3..5c60ea2ae0 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/Responses/AIPromptCaptureResponse.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/Responses/AIPromptCaptureResponse.cs @@ -23,11 +23,8 @@ public class AIPromptCaptureResponse [JsonPropertyName("userPrompt")] public string UserPrompt { get; set; } = string.Empty; - [JsonPropertyName("rawOutput")] - public string RawOutput { get; set; } = string.Empty; - - [JsonPropertyName("formattedOutput")] - public string FormattedOutput { get; set; } = string.Empty; + [JsonPropertyName("output")] + public string Output { get; set; } = string.Empty; [JsonPropertyName("capturedAt")] public DateTime CapturedAt { get; set; } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index f69a0266da..1495d6ec75 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -686,7 +686,13 @@ private int ResolveCompletionTokens(string operationName, int defaultValue) return operationPromptVersion; } - return _configuration["Azure:Operations:Defaults:PromptVersion"]; + var defaultPromptVersion = _configuration["Azure:Operations:Defaults:PromptVersion"]; + if (!string.IsNullOrWhiteSpace(defaultPromptVersion)) + { + return defaultPromptVersion; + } + + return _configuration["Azure:OpenAI:PromptVersion"]; } private string ResolveProviderName(string? operationName = null) @@ -1047,8 +1053,7 @@ private void SavePromptCapture(bool capturePromptIo, string? contextId, string p CaptureLabel = captureLabel?.Trim() ?? string.Empty, SystemPrompt = systemPrompt?.Trim() ?? string.Empty, UserPrompt = userPrompt?.Trim() ?? string.Empty, - RawOutput = rawOutput?.Trim() ?? string.Empty, - FormattedOutput = FormatPromptOutputForLog(rawOutput ?? string.Empty), + Output = FormatPromptOutputForLog(rawOutput ?? string.Empty), CapturedAt = DateTime.UtcNow }); } @@ -1067,6 +1072,11 @@ private static string FormatPromptOutputForLog(string output) return string.Empty; } + if (TryFormatProviderOutput(output, out var formattedProviderOutput)) + { + return formattedProviderOutput; + } + if (TryParseJsonObjectFromResponse(output, out var jsonObject)) { return JsonSerializer.Serialize(jsonObject, JsonLogOptions); @@ -1075,6 +1085,101 @@ private static string FormatPromptOutputForLog(string output) return output.Trim(); } + private static bool TryFormatProviderOutput(string output, out string formattedOutput) + { + formattedOutput = string.Empty; + + try + { + using var doc = JsonDocument.Parse(output); + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object + || !root.TryGetProperty("choices", out var choices) + || choices.ValueKind != JsonValueKind.Array + || choices.GetArrayLength() == 0) + { + return false; + } + + var firstChoice = choices[0]; + var content = TryGetChoiceContent(firstChoice); + if (string.IsNullOrWhiteSpace(content)) + { + return false; + } + + var lines = new List(); + + if (root.TryGetProperty("usage", out var usage) && usage.ValueKind == JsonValueKind.Object) + { + var promptTokens = TryGetInt32(usage, "prompt_tokens"); + var completionTokens = TryGetInt32(usage, "completion_tokens"); + int? reasoningTokens = null; + + if (usage.TryGetProperty("completion_tokens_details", out var completionTokenDetails) + && completionTokenDetails.ValueKind == JsonValueKind.Object) + { + reasoningTokens = TryGetInt32(completionTokenDetails, "reasoning_tokens"); + } + + if (promptTokens.HasValue) + { + lines.Add($"PROMPT TOKENS: {promptTokens.Value}"); + } + + if (completionTokens.HasValue) + { + lines.Add($"COMPLETION TOKENS: {completionTokens.Value}"); + } + + if (reasoningTokens.HasValue) + { + lines.Add($"REASONING TOKENS: {reasoningTokens.Value}"); + } + } + + var normalizedContent = FormatPromptOutputContent(content); + if (lines.Count > 0) + { + lines.Add(string.Empty); + } + + lines.Add("CONTENT"); + lines.Add(normalizedContent); + formattedOutput = string.Join(Environment.NewLine, lines); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static string? TryGetChoiceContent(JsonElement firstChoice) + { + if (!firstChoice.TryGetProperty("message", out var message) || message.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!message.TryGetProperty("content", out var contentProp) || contentProp.ValueKind != JsonValueKind.String) + { + return null; + } + + return contentProp.GetString(); + } + + private static string FormatPromptOutputContent(string content) + { + if (TryParseJsonObjectFromResponse(content, out var contentObject)) + { + return JsonSerializer.Serialize(contentObject, JsonLogOptions); + } + + return content.Trim(); + } + private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) { objectElement = default; @@ -1358,3 +1463,5 @@ private static string ExtractSummaryFromJson(string output) } } } + + diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md index 76f1ca12d0..0a2ae41b7b 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md @@ -40,7 +40,8 @@ Placeholders: Version selection: -- `Azure:Operations:Defaults:PromptVersion = v0|v1, with optional overrides under Azure:Operations::PromptVersion` +- Preferred: `Azure:Operations:Defaults:PromptVersion = v0|v1`, with optional overrides under `Azure:Operations::PromptVersion` +- Legacy fallback: `Azure:OpenAI:PromptVersion = v0|v1` - Unknown or missing version defaults to `v1`. Template loading is strict: diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs index 2a4b3c470a..7e9f1a5620 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/AI/AIPromptToolViewOptionsProvider.cs @@ -12,9 +12,20 @@ public class AIPromptToolViewOptionsProvider( public bool IsDevPromptControlsEnabled => string.Equals(webHostEnvironment.EnvironmentName, "Development", StringComparison.OrdinalIgnoreCase); - public string DefaultPromptVersion => - string.IsNullOrWhiteSpace(configuration["Azure:Operations:Defaults:PromptVersion"]) - ? "v1" - : configuration["Azure:Operations:Defaults:PromptVersion"]!.Trim().ToLowerInvariant(); + public string DefaultPromptVersion + { + get + { + var configuredPromptVersion = configuration["Azure:Operations:Defaults:PromptVersion"]; + if (string.IsNullOrWhiteSpace(configuredPromptVersion)) + { + configuredPromptVersion = configuration["Azure:OpenAI:PromptVersion"]; + } + + return string.IsNullOrWhiteSpace(configuredPromptVersion) + ? "v1" + : configuredPromptVersion.Trim().ToLowerInvariant(); + } + } } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Pages/GrantApplications/Details.js b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Pages/GrantApplications/Details.js index 79ec9a5e85..8b3542b988 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Pages/GrantApplications/Details.js +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Pages/GrantApplications/Details.js @@ -885,6 +885,7 @@ $(function () { }); function formatAIPromptCaptureBlock(capture) { + const output = capture.output || ''; const parts = [ `PROMPT TYPE: ${capture.promptType || ''}`, `PROMPT VERSION: ${capture.promptVersion || ''}` @@ -906,11 +907,8 @@ function formatAIPromptCaptureBlock(capture) { 'USER PROMPT', capture.userPrompt || '', '', - 'RAW OUTPUT', - capture.rawOutput || '', - '', - 'FORMATTED OUTPUT', - capture.formattedOutput || '' + 'OUTPUT', + output ); return parts.join('\n'); @@ -1378,3 +1376,4 @@ function clearCurrencyError(input) { document.getElementById(errorSpan).textContent = ''; input.attr('aria-invalid', 'false'); } + From b96eab7ba99a41d5b8791b1c2bdf2665a20357d3 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 17:28:08 -0700 Subject: [PATCH 12/15] AB#32006 tighten prompt capture output formatting --- .../src/Unity.GrantManager.Application/AI/OpenAIService.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 1495d6ec75..94907e1d13 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -1139,12 +1139,7 @@ private static bool TryFormatProviderOutput(string output, out string formattedO } var normalizedContent = FormatPromptOutputContent(content); - if (lines.Count > 0) - { - lines.Add(string.Empty); - } - - lines.Add("CONTENT"); + lines.Add("CONTENT:"); lines.Add(normalizedContent); formattedOutput = string.Join(Environment.NewLine, lines); return true; From c40f7270eb4064d72bfdf05f8d262e04e7e82cb1 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 17:34:47 -0700 Subject: [PATCH 13/15] AB#32006 update development AI config template --- .../appsettings.Development.json | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/appsettings.Development.json b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/appsettings.Development.json index bb92c5bf84..e7487ba79d 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/appsettings.Development.json +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/appsettings.Development.json @@ -135,15 +135,40 @@ "ReportingAI": { "JWTSecret": "" }, + "Azure": { + "Operations": { + "Defaults": { + "Provider": "OpenAI", + "Profile": "Gpt4oMini", + "PromptVersion": "v1" + }, + "AttachmentSummary": { + "MaxCompletionTokens": 1500 + }, + "ApplicationAnalysis": { + "MaxCompletionTokens": 2500 + }, + "ScoresheetSection": { + "MaxCompletionTokens": 5000 + } + }, "OpenAI": { "ApiKey": "", - "ApiUrl": "", - "Model": "" + "Profiles": { + "Gpt4oMini": { + "ApiUrl": "", + "MaxTokensParameter": "max_tokens", + "Temperature": 0.3 + }, + "Gpt5Mini": { + "ApiUrl": "", + "MaxTokensParameter": "max_completion_tokens" + } + } }, - "AgenticAPI": { - "Url": "http://localhost:5000/v1/completions" + "Logging": { + "EnablePromptFileLog": true } } - -} \ No newline at end of file +} From a1155021ed0b069d1315bcf0104460e92e523948 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 17:56:50 -0700 Subject: [PATCH 14/15] AB#32006 clean up AI profile and URL resolution --- .../AI/OpenAIService.cs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 94907e1d13..1ffa2cdee2 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -719,7 +719,7 @@ private string ResolveProviderName(string? operationName = null) private string ResolveMaxTokensParameterNameForOperation(string? operationName = null) { var providerName = ResolveProviderName(operationName); - var profileName = ResolveProfileName(operationName, providerName); + var profileName = ResolveProfileName(operationName); var profileParameterName = ResolveProfileSetting(providerName, profileName, "MaxTokensParameter"); return ResolveMaxTokensParameterName(profileParameterName); } @@ -727,7 +727,7 @@ private string ResolveMaxTokensParameterNameForOperation(string? operationName = private double? ResolveConfiguredTemperature(string? operationName = null) { var providerName = ResolveProviderName(operationName); - var profileName = ResolveProfileName(operationName, providerName); + var profileName = ResolveProfileName(operationName); var profileTemperature = ResolveProfileSetting(providerName, profileName, "Temperature"); if (profileTemperature != null && double.TryParse(profileTemperature, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var parsedTemperature)) @@ -740,24 +740,25 @@ private string ResolveMaxTokensParameterNameForOperation(string? operationName = private string ResolveApiUrl(string? operationName) { - if (!string.IsNullOrWhiteSpace(operationName)) + var providerName = ResolveProviderName(operationName); + var profileName = ResolveProfileName(operationName); + var profileApiUrl = ResolveProfileSetting(providerName, profileName, "ApiUrl"); + var legacyOpenAiApiUrl = _configuration["Azure:OpenAI:ApiUrl"]; + + if (!string.IsNullOrWhiteSpace(profileApiUrl)) { - var operationApiUrl = _configuration[$"Azure:Operations:{operationName}:ApiUrl"]; - if (!string.IsNullOrWhiteSpace(operationApiUrl)) - { - return operationApiUrl; - } + return profileApiUrl; } - var providerName = ResolveProviderName(operationName); - var profileName = ResolveProfileName(operationName, providerName); - var profileApiUrl = ResolveProfileSetting(providerName, profileName, "ApiUrl"); - return profileApiUrl - ?? _configuration[$"Azure:{providerName}:ApiUrl"] - ?? "https://api.openai.com/v1/chat/completions"; + if (!string.IsNullOrWhiteSpace(legacyOpenAiApiUrl)) + { + return legacyOpenAiApiUrl; + } + + throw new InvalidOperationException($"AI API URL is not configured for provider '{providerName}'."); } - private string? ResolveProfileName(string? operationName, string providerName) + private string? ResolveProfileName(string? operationName) { if (!string.IsNullOrWhiteSpace(operationName)) { From d0e30a4858ce37bd855c4ad193fbf7dcf9437633 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Mon, 16 Mar 2026 18:03:30 -0700 Subject: [PATCH 15/15] AB#32006 finalize AI config and reviewer prompt rules --- .../AI/Prompts/Versions/v1/analysis.rules.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt index 09c04ecc0f..a250310372 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt @@ -25,8 +25,10 @@ - If no findings exist, return empty arrays. - Rating must be HIGH, MEDIUM, or LOW. - Use summaries for overall application quality/readiness synthesis. -- Use nextSteps for reviewer-facing follow-up actions or considerations before scoring or decision-making. -- Only include nextSteps when there is a specific evidence gap, inconsistency, or verification need; otherwise return an empty array. +- Use nextSteps for concrete reviewer-facing next actions based on the provided evidence. +- nextSteps may include proceeding with the normal review process when the application appears ready for that step. +- When evidence shows a meaningful gap, inconsistency, or uncertainty, use nextSteps for specific follow-up or verification actions. +- Return an empty array only when no concrete next action would help the reviewer. - recommendation.decision must be PROCEED or HOLD. - Use HOLD only when provided evidence shows a material eligibility, feasibility, budget, or readiness concern that would reasonably block scoring or decision-making. - recommendation.rationale must explain the high-level recommendation in 1-2 complete sentences using provided evidence.