From 9667a1058e97091b6320845a94ecd0542715247a Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 16:11:12 -0800 Subject: [PATCH 01/10] AB#32009 Add prompt baseline snapshots for v0 and v1 comparison --- .../Baselines/v0/AnalysisPrompts.v1.txt | 58 + .../Baselines/v0/AttachmentPrompts.v1.txt | 29 + .../Baselines/v0/OpenAIService.v0.cs.txt | 406 ++++++ .../Baselines/v0/OpenAIService.v1.cs.txt | 1134 +++++++++++++++++ .../Baselines/v0/PromptCoreRules.v1.txt | 13 + .../Prompts/Baselines/v0/PromptHeader.v1.txt | 14 + .../Baselines/v0/ScoresheetPrompts.v1.txt | 85 ++ 7 files changed, 1739 insertions(+) create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.v1.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.v1.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v0.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.v1.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.v1.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.v1.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.v1.txt new file mode 100644 index 000000000..d267a1216 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.v1.txt @@ -0,0 +1,58 @@ +namespace Unity.GrantManager.AI +{ + internal static class AnalysisPrompts + { + public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. +COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. +FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. +RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. +QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; + + public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. +MEDIUM: Application has some gaps or weaknesses that require reviewer attention. +LOW: Application has significant gaps or risks across key rubric areas."; + + public const string OutputTemplate = @"{ + ""rating"": """", + ""errors"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""warnings"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""summaries"": [ + { + ""title"": """", + ""detail"": """" + } + ] +}"; + + public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n" + + "- Do not invent fields, documents, requirements, or facts.\n" + + @"- Treat missing or empty values as findings only when they weaken rubric evidence. +- Prefer material issues; avoid nitpicking. +- Use 3-6 words for title. +- Each detail must be 1-2 complete sentences. +- Each detail must cite concrete evidence from DATA or ATTACHMENTS. +- If ATTACHMENTS evidence is used, cite the attachment by name in detail. +- If no findings exist, return empty arrays. +- Rating must be HIGH, MEDIUM, or LOW. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static readonly string SystemPrompt = PromptHeader.Build( + "You are an expert grant analyst assistant for human reviewers.", + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.v1.txt new file mode 100644 index 000000000..a61cc5084 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.v1.txt @@ -0,0 +1,29 @@ +namespace Unity.GrantManager.AI +{ + internal static class AttachmentPrompts + { + public static readonly string SystemPrompt = PromptHeader.Build( + "You are a professional grant analyst for the BC Government.", + "Produce a concise reviewer-facing summary of the provided attachment context."); + + public const string OutputSection = @"OUTPUT +{ + ""summary"": """" +}"; + + public const string RulesSection = "- Use only ATTACHMENT as evidence.\n" + + "- If ATTACHMENT.text is present, summarize actual content.\n" + + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" + + PromptCoreRules.NoInvention + "\n" + + @"- Write 1-2 complete sentences. +- Summary must be grounded in concrete ATTACHMENT evidence. +- Return exactly one object with only the key: summary. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + } +} + diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v0.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v0.cs.txt new file mode 100644 index 000000000..239db1062 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v0.cs.txt @@ -0,0 +1,406 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Volo.Abp.DependencyInjection; + +namespace Unity.GrantManager.AI +{ + public class OpenAIService : IAIService, ITransientDependency + { + private readonly HttpClient _httpClient; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly ITextExtractionService _textExtractionService; + + private string? ApiKey => _configuration["AI:OpenAI:ApiKey"]; + private string? ApiUrl => _configuration["AI:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; + private readonly string NoKeyError = "OpenAI API key is not configured"; + + public OpenAIService(HttpClient httpClient, IConfiguration configuration, ILogger logger, ITextExtractionService textExtractionService) + { + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + _textExtractionService = textExtractionService; + } + + public Task IsAvailableAsync() + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", NoKeyError); + return Task.FromResult(false); + } + + return Task.FromResult(true); + } + + public async Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", NoKeyError); + return "AI analysis not available - service not configured."; + } + + _logger.LogDebug("Calling OpenAI with prompt: {Prompt}", content); + + try + { + var systemPrompt = prompt ?? "You are a professional grant analyst for the BC Government."; + + var requestBody = new + { + messages = new[] + { + new { role = "system", content = systemPrompt }, + new { role = "user", content = content } + }, + max_tokens = maxTokens, + temperature = 0.3 + }; + + var json = JsonSerializer.Serialize(requestBody); + var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); + + _httpClient.DefaultRequestHeaders.Clear(); + _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); + + var response = await _httpClient.PostAsync(ApiUrl, httpContent); + var responseContent = await response.Content.ReadAsStringAsync(); + + _logger.LogDebug("Response: {Response}", responseContent); + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); + return "AI analysis failed - service temporarily unavailable."; + } + + using var jsonDoc = JsonDocument.Parse(responseContent); + var choices = jsonDoc.RootElement.GetProperty("choices"); + if (choices.GetArrayLength() > 0) + { + var message = choices[0].GetProperty("message"); + return message.GetProperty("content").GetString() ?? "No summary generated."; + } + + return "No summary generated."; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating AI summary"); + return "AI analysis failed - please try again later."; + } + } + + public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] fileContent, string contentType) + { + try + { + var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); + + string contentToAnalyze; + string prompt; + + if (!string.IsNullOrWhiteSpace(extractedText)) + { + _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText.Length, fileName); + + contentToAnalyze = $"Document: {fileName}\nType: {contentType}\nContent:\n{extractedText}"; + prompt = "Please analyze this document and provide a concise summary of its content, purpose, and key information, for use by your fellow grant analysts. It should be 1-2 sentences long and about 46 tokens."; + } + else + { + _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", fileName); + + contentToAnalyze = $"File: {fileName}, Type: {contentType}, Size: {fileContent.Length} bytes"; + prompt = "Please analyze this document and provide a concise summary of its content, purpose, and key information, for use by your fellow grant analysts. It should be 1-2 sentences long and about 46 tokens."; + } + + return await GenerateSummaryAsync(contentToAnalyze, prompt, 150); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); + return $"AI analysis not available for this attachment ({fileName})."; + } + } + + public async Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", NoKeyError); + return "AI analysis not available - service not configured."; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var fieldConfigurationSection = !string.IsNullOrEmpty(formFieldConfiguration) + ? $@" +{formFieldConfiguration}" + : string.Empty; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} +{fieldConfigurationSection} + +EVALUATION RUBRIC: +{rubric} + +Analyze this grant application comprehensively across all five rubric categories (Eligibility, Completeness, Financial Review, Risk Assessment, and Quality Indicators). Identify issues, concerns, and areas for improvement. Return your findings in the following JSON format: +{{ + ""overall_score"": ""HIGH/MEDIUM/LOW"", + ""warnings"": [ + {{ + ""category"": ""Brief summary of the warning"", + ""message"": ""Detailed warning message with full context and explanation"", + ""severity"": ""WARNING"" + }} + ], + ""errors"": [ + {{ + ""category"": ""Brief summary of the error"", + ""message"": ""Detailed error message with full context and explanation"", + ""severity"": ""ERROR"" + }} + ], + ""recommendations"": [ + {{ + ""category"": ""Brief summary of the recommendation"", + ""message"": ""Detailed recommendation with specific actionable guidance"" + }} + ] +}} + +Important: The 'category' field should be a concise summary (3-6 words) that captures the essence of the issue, while the 'message' field should contain the detailed explanation."; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. + +Conduct a thorough, comprehensive analysis across all rubric categories. Identify substantive issues, concerns, and opportunities for improvement. + +Classify findings based on their impact on the application's evaluation and fundability: +- ERRORS: Important missing information, significant gaps in required content, compliance issues, or major concerns affecting eligibility +- WARNINGS: Areas needing clarification, moderate issues, or concerns that should be addressed + +Evaluate the quality, clarity, and appropriateness of all application content. Be thorough but fair - identify real issues while avoiding nitpicking. + +Respond only with valid JSON in the exact format requested."; + + var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + + // Post-process the AI response to add unique IDs to errors and warnings + return AddIdsToAnalysisItems(rawAnalysis); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error analyzing application"); + return "AI analysis failed - please try again later."; + } + } + + private string AddIdsToAnalysisItems(string analysisJson) + { + try + { + using var jsonDoc = JsonDocument.Parse(analysisJson); + using var memoryStream = new System.IO.MemoryStream(); + using (var writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions { Indented = true })) + { + writer.WriteStartObject(); + + foreach (var property in jsonDoc.RootElement.EnumerateObject()) + { + if (property.Name == "errors" || property.Name == "warnings") + { + writer.WritePropertyName(property.Name); + writer.WriteStartArray(); + + foreach (var item in property.Value.EnumerateArray()) + { + writer.WriteStartObject(); + + // Add unique ID first + writer.WriteString("id", Guid.NewGuid().ToString()); + + // Copy existing properties + foreach (var itemProperty in item.EnumerateObject()) + { + itemProperty.WriteTo(writer); + } + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + else + { + property.WriteTo(writer); + } + } + + // Add dismissed_items array if not present + if (!jsonDoc.RootElement.TryGetProperty("dismissed_items", out _)) + { + writer.WritePropertyName("dismissed_items"); + writer.WriteStartArray(); + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + + return Encoding.UTF8.GetString(memoryStream.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error adding IDs to analysis items, returning original JSON"); + return analysisJson; // Return original if processing fails + } + } + + public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", NoKeyError); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} + +SCORESHEET QUESTIONS: +{scoresheetQuestions} + +Please analyze this grant application and provide appropriate answers for each scoresheet question. + +For numeric questions, provide a numeric value within the specified range. +For yes/no questions, provide either 'Yes' or 'No'. +For text questions, provide a concise, relevant response. +For select list questions, choose the most appropriate option from the provided choices. +For text area questions, provide a detailed but concise response. + +Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. + +Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: +{{ + ""question-id-1"": ""answer-value-1"", + ""question-id-2"": ""answer-value-2"" +}} +Do not return any markdown formatting, just the JSON by itself"; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. +Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. +Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. +Respond only with valid JSON in the exact format requested."; + + return await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet answers"); + return "{}"; + } + } + + public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", NoKeyError); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} + +SCORESHEET SECTION: {sectionName} +{sectionJson} + +Please analyze this grant application and provide appropriate answers for each question in the ""{sectionName}"" section only. + +For each question, provide: +1. Your answer based on the application content +2. A brief cited description (1-2 sentences) explaining your reasoning with specific references to the application content +3. A confidence score from 0-100 indicating how confident you are in your answer based on available information + +Guidelines for answers: +- For numeric questions, provide a numeric value within the specified range +- For yes/no questions, provide either 'Yes' or 'No' +- For text questions, provide a concise, relevant response +- For select list questions, respond with ONLY the number from the 'number' field (1, 2, 3, etc.) of your chosen option. NEVER return 0 - the lowest valid answer is 1. For example: if you want '(0 pts) No outcomes provided', choose the option where number=1, not 0. +- For text area questions, provide a detailed but concise response +- Base your confidence score on how clearly the application content supports your answer + +Return your response as a JSON object where each key is the question ID and the value contains the answer, citation, and confidence: +{{ + ""question-id-1"": {{ + ""answer"": ""your-answer-here"", + ""citation"": ""Brief explanation with specific reference to application content"", + ""confidence"": 85 + }}, + ""question-id-2"": {{ + ""answer"": ""3"", + ""citation"": ""Based on the project budget of $50,000 mentioned in the application, this falls into the medium budget category"", + ""confidence"": 90 + }} +}} + +IMPORTANT FOR SELECT LIST QUESTIONS: If a question has availableOptions like: +[{{""number"":1,""value"":""Low (Under $25K)""}}, {{""number"":2,""value"":""Medium ($25K-$75K)""}}, {{""number"":3,""value"":""High (Over $75K)""}}] +Then respond with ONLY the number (e.g., ""3"" for ""High (Over $75K)""), not the text value. + +Do not return any markdown formatting, just the JSON by itself"; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. +Analyze the provided application and generate appropriate answers for the scoresheet section questions based on the application content. +Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. +Always provide citations that reference specific parts of the application content to support your answers. +Be honest about your confidence level - if information is missing or unclear, reflect this in a lower confidence score. +Respond only with valid JSON in the exact format requested."; + + return await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", sectionName); + return "{}"; + } + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt new file mode 100644 index 000000000..e421036eb --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt @@ -0,0 +1,1134 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Volo.Abp.DependencyInjection; + +namespace Unity.GrantManager.AI +{ + public class OpenAIService : IAIService, ITransientDependency + { + private readonly HttpClient _httpClient; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly ITextExtractionService _textExtractionService; + private readonly JsonSerializerOptions _prettyJsonOptions = new() { WriteIndented = true }; + + private string? ApiKey => _configuration["AI:OpenAI:ApiKey"]; + private bool HasApiKey => !string.IsNullOrWhiteSpace(ApiKey); + private string? ApiUrl => _configuration["AI:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; + private bool LogPayloads => _configuration.GetValue("AI:Logging:LogPayloads") ?? false; + private readonly string NoApiKeyMessage = "OpenAI API key is not configured"; + private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; + private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; + private const string GenericFailureMessage = "AI analysis failed - please try again later."; + private const string NoSummaryGeneratedMessage = "No summary generated."; + private const string EmptyJsonObject = "{}"; + private const int ScoresheetAllMaxTokens = 2000; + private const int ScoresheetSectionMaxTokens = 3200; + private const int AnalysisMaxTokens = 1600; + private const string DefaultContentType = "application/octet-stream"; + private const int AttachmentSummaryMaxTokens = 240; + private const string AttachmentSummaryUnavailableMessage = "AI analysis not available for this attachment"; + private const string ScoreHigh = "HIGH"; + private const string ScoreMedium = "MEDIUM"; + private const string ScoreLow = "LOW"; + private const string AiPromptLogRelativePath = "logs/ai-prompts.log"; + private static int _aiPromptLogInitialized; + + public OpenAIService(HttpClient httpClient, IConfiguration configuration, ILogger logger, ITextExtractionService textExtractionService) + { + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + _textExtractionService = textExtractionService; + } + + public Task IsAvailableAsync() + { + if (!HasApiKey) + { + _logger.LogWarning("Error: {Message}", NoApiKeyMessage); + return Task.FromResult(false); + } + + return Task.FromResult(true); + } + + public async Task GenerateCompletionAsync(AICompletionRequest request) + { + if (request == null) + { + _logger.LogWarning("AI completion request was null."); + return GenericFailureMessage; + } + + var userPrompt = request.UserPrompt ?? string.Empty; + var systemPrompt = request.SystemPrompt; + var maxTokens = request.MaxTokens <= 0 ? 150 : request.MaxTokens; + var temperature = request.Temperature ?? 0.3; + + return await ExecuteChatCompletionAsync(userPrompt, systemPrompt, maxTokens, temperature); + } + + private async Task ExecuteChatCompletionAsync( + string userPrompt, + string? systemPrompt = null, + int maxTokens = 150, + double temperature = 0.3) + { + if (!HasApiKey) + { + _logger.LogWarning("Error: {Message}", NoApiKeyMessage); + return ServiceNotConfiguredMessage; + } + + _logger.LogDebug( + "Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", + userPrompt?.Length ?? 0, + maxTokens); + + try + { + string resolvedSystemPrompt = systemPrompt ?? "You are a professional grant analyst for the BC Government."; + + var requestBody = new Dictionary + { + ["messages"] = new[] + { + new { role = "system", content = resolvedSystemPrompt }, + new { role = "user", content = userPrompt ?? string.Empty } + }, + ["max_tokens"] = maxTokens, + ["temperature"] = temperature + }; + + var json = JsonSerializer.Serialize(requestBody); + var authValue = ApiKey!.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase) + ? ApiKey.Substring("Bearer ".Length).Trim() + : ApiKey; + + using var request = new HttpRequestMessage(HttpMethod.Post, ApiUrl); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authValue); + request.Content = new StringContent(json, Encoding.UTF8, "application/json"); + + var response = await _httpClient.SendAsync(request); + var responseContent = await response.Content.ReadAsStringAsync() ?? string.Empty; + + _logger.LogDebug( + "OpenAI chat completions response received. StatusCode: {StatusCode}, ResponseLength: {ResponseLength}", + response.StatusCode, + responseContent?.Length ?? 0); + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); + return ServiceTemporarilyUnavailableMessage; + } + + using var jsonDoc = JsonDocument.Parse(responseContent ?? string.Empty); + var choices = jsonDoc.RootElement.GetProperty("choices"); + if (choices.GetArrayLength() > 0) + { + var message = choices[0].GetProperty("message"); + return message.GetProperty("content").GetString() ?? NoSummaryGeneratedMessage; + } + + return NoSummaryGeneratedMessage; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating AI summary"); + return GenericFailureMessage; + } + } + + // Canonical attachment summary prompt contract is defined by: + // AttachmentPrompts.SystemPrompt, AttachmentPrompts.OutputSection, and AttachmentPrompts.RulesSection. + public async Task GenerateAttachmentSummaryAsync(AttachmentSummaryRequest request) + { + try + { + if (request == null) + { + _logger.LogWarning("Attachment summary request was null."); + return $"{AttachmentSummaryUnavailableMessage} (unknown)."; + } + + var normalizedFileName = string.IsNullOrWhiteSpace(request.FileName) ? "unknown" : request.FileName.Trim(); + var normalizedContentType = string.IsNullOrWhiteSpace(request.ContentType) ? DefaultContentType : request.ContentType.Trim(); + var normalizedFileContent = request.FileContent ?? Array.Empty(); + var extractedText = await _textExtractionService.ExtractTextAsync(normalizedFileName, normalizedFileContent, normalizedContentType); + + var hasExtractedText = !string.IsNullOrWhiteSpace(extractedText); + if (hasExtractedText) + { + _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText!.Length, normalizedFileName); + } + else + { + _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", normalizedFileName); + } + + var attachmentInput = new AttachmentPromptInput + { + Name = normalizedFileName, + Text = hasExtractedText ? extractedText : null + }; + + var contentToAnalyze = BuildAttachmentSummaryPrompt(attachmentInput); + return await ExecutePromptWithRetryAsync( + promptType: "AttachmentSummary", + systemPrompt: AttachmentPrompts.SystemPrompt, + userPrompt: contentToAnalyze, + maxTokens: AttachmentSummaryMaxTokens, + normalizeResponse: NormalizeAttachmentSummaryResponse, + isValidNormalizedResponse: normalized => !string.IsNullOrWhiteSpace(normalized), + fallbackResponse: string.Empty); + } + catch (Exception ex) + { + var fileName = request?.FileName ?? "unknown"; + _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); + return $"{AttachmentSummaryUnavailableMessage} ({fileName})."; + } + } + + private string BuildAttachmentSummaryPrompt(AttachmentPromptInput attachmentInput) + { + return $@"ATTACHMENT +{JsonSerializer.Serialize(attachmentInput, _prettyJsonOptions)} + +{AttachmentPrompts.OutputSection} + +RULES +{AttachmentPrompts.RulesSection}"; + } + + private string NormalizeAttachmentSummaryResponse(string response) + { + if (!TryParseJsonObjectFromResponse(response, out var responseObject)) + { + return string.Empty; + } + + if (responseObject.TryGetProperty(AIJsonKeys.Summary, out var summaryProp) && + summaryProp.ValueKind == JsonValueKind.String) + { + return summaryProp.GetString()?.Trim() ?? string.Empty; + } + + return responseObject.ToString().Trim(); + } + + // Canonical analysis prompt contract is defined by: + // AnalysisPrompts.DefaultRubric, AnalysisPrompts.ScoreRules, AnalysisPrompts.OutputTemplate, + // AnalysisPrompts.Rules, and AnalysisPrompts.SystemPrompt. + public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) + { + if (!HasApiKey) + { + _logger.LogWarning("{Message}", NoApiKeyMessage); + return ServiceNotConfiguredMessage; + } + + try + { + if (request == null) + { + _logger.LogWarning("Application analysis request was null."); + return BuildEmptyAnalysisResponseJson(); + } + + var emptyObject = CreateEmptyJsonObject(); + var schemaPayload = request.Schema.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Schema; + var dataPayload = request.Data.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Data; + + if (schemaPayload.ValueKind != JsonValueKind.Object || dataPayload.ValueKind != JsonValueKind.Object) + { + _logger.LogWarning( + "Invalid application analysis request payload shape. Schema kind: {SchemaKind}, Data kind: {DataKind}.", + schemaPayload.ValueKind, + dataPayload.ValueKind); + return BuildEmptyAnalysisResponseJson(); + } + + var attachmentsPayload = request.Attachments? + .Where(a => a != null && !string.IsNullOrWhiteSpace(a.Summary)) + .Select(a => new ApplicationAnalysisAttachment + { + Name = string.IsNullOrWhiteSpace(a.Name) ? "attachment" : a.Name.Trim(), + Summary = a.Summary.Trim() + }) + .ToList() ?? new List(); + + var rubricText = !string.IsNullOrWhiteSpace(request.Rubric) ? request.Rubric : AnalysisPrompts.DefaultRubric; + var analysisPrompt = BuildAnalysisPrompt(schemaPayload, dataPayload, attachmentsPayload, rubricText); + return await ExecutePromptWithRetryAsync( + promptType: "ApplicationAnalysis", + systemPrompt: AnalysisPrompts.SystemPrompt, + userPrompt: analysisPrompt, + maxTokens: AnalysisMaxTokens, + normalizeResponse: NormalizeAnalysisResponse, + isValidNormalizedResponse: IsValidAnalysisNormalizedResponse, + fallbackResponse: BuildEmptyAnalysisResponseJson()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error analyzing application"); + return BuildEmptyAnalysisResponseJson(); + } + } + + private string NormalizeAnalysisResponse(string analysisJson) + { + try + { + if (!TryParseJsonObjectFromResponse(analysisJson, out var analysisObject)) + { + _logger.LogError("Invalid analysis JSON response."); + return BuildEmptyAnalysisResponseJson(); + } + + var parseOptions = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }; + + var model = JsonSerializer.Deserialize(analysisObject.GetRawText(), parseOptions); + if (model == null) + { + return BuildEmptyAnalysisResponseJson(); + } + + model.Errors ??= new List(); + model.Warnings ??= new List(); + model.Summaries ??= new List(); + model.Dismissed ??= new List(); + + model.Rating = NormalizeRating(model.Rating); + + foreach (var error in model.Errors) + { + error.Id = string.IsNullOrWhiteSpace(error.Id) ? Guid.NewGuid().ToString() : error.Id; + } + + foreach (var warning in model.Warnings) + { + warning.Id = string.IsNullOrWhiteSpace(warning.Id) ? Guid.NewGuid().ToString() : warning.Id; + } + + model.Dismissed = model.Dismissed + .Where(id => !string.IsNullOrWhiteSpace(id)) + .Distinct(StringComparer.Ordinal) + .ToList(); + + var normalizedOutput = new ApplicationAnalysisResponse + { + Rating = model.Rating, + Errors = model.Errors, + Warnings = model.Warnings, + Summaries = model.Summaries, + Dismissed = model.Dismissed + }; + + return JsonSerializer.Serialize(normalizedOutput, _prettyJsonOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error normalizing analysis response."); + return BuildEmptyAnalysisResponseJson(); + } + } + + private static string NormalizeRating(string? rating) + { + var normalized = rating?.Trim().ToUpperInvariant(); + return normalized switch + { + ScoreHigh => ScoreHigh, + ScoreMedium => ScoreMedium, + ScoreLow => ScoreLow, + _ => ScoreMedium + }; + } + + private string BuildAnalysisPrompt( + JsonElement schemaPayload, + JsonElement dataPayload, + List attachmentsPayload, + string rubricText) + { + var analysisAttachments = (attachmentsPayload ?? new List()) + .Select(a => new AnalysisAttachmentPromptItem + { + Name = a.Name, + Summary = a.Summary + }) + .ToList(); + + return $@"SCHEMA +{JsonSerializer.Serialize(schemaPayload, _prettyJsonOptions)} + +DATA +{JsonSerializer.Serialize(dataPayload, _prettyJsonOptions)} + +ATTACHMENTS +{JsonSerializer.Serialize(analysisAttachments, _prettyJsonOptions)} + +RUBRIC +{rubricText ?? AnalysisPrompts.DefaultRubric} + +SCORE +{AnalysisPrompts.ScoreRules} + +OUTPUT +{AnalysisPrompts.OutputTemplate} + +RULES +{AnalysisPrompts.Rules}"; + } + // Canonical scoresheet-all prompt contract is defined by: + // ScoresheetPrompts.AllSystemPrompt, ScoresheetPrompts.AllOutputTemplate, and ScoresheetPrompts.AllRules. + public async Task GenerateScoresheetAllAnswersAsync(ScoresheetAllRequest request) + { + if (!HasApiKey) + { + _logger.LogWarning("{Message}", NoApiKeyMessage); + return EmptyJsonObject; + } + + try + { + if (request == null) + { + _logger.LogWarning("Scoresheet-all request was null."); + return EmptyJsonObject; + } + + if (!IsValidScoresheetQuestionsPayload(request.Questions)) + { + _logger.LogWarning( + "Invalid scoresheet-all questions payload shape. Questions kind: {QuestionsKind}.", + request.Questions.ValueKind); + return EmptyJsonObject; + } + + var scoresheetPrompt = BuildScoresheetAllPrompt(request); + return await ExecutePromptWithRetryAsync( + promptType: "ScoresheetAll", + systemPrompt: ScoresheetPrompts.AllSystemPrompt, + userPrompt: scoresheetPrompt, + maxTokens: ScoresheetAllMaxTokens, + normalizeResponse: NormalizeScoresheetAllResponse, + isValidNormalizedResponse: normalized => !IsEmptyJsonObject(normalized), + fallbackResponse: EmptyJsonObject); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet answers"); + return EmptyJsonObject; + } + } + // Canonical scoresheet-section prompt contract is defined by: + // ScoresheetPrompts.SectionSystemPrompt, ScoresheetPrompts.SectionOutputTemplate, and ScoresheetPrompts.SectionRules. + public async Task GenerateScoresheetSectionAnswersAsync(ScoresheetSectionRequest request) + { + if (!HasApiKey) + { + _logger.LogWarning("{Message}", NoApiKeyMessage); + return EmptyJsonObject; + } + + try + { + if (request == null) + { + _logger.LogWarning("Scoresheet-section request was null."); + return EmptyJsonObject; + } + + if (!IsValidScoresheetSectionSchemaPayload(request.SectionSchema)) + { + _logger.LogWarning( + "Invalid scoresheet-section schema payload shape. SectionSchema kind: {SectionSchemaKind}.", + request.SectionSchema.ValueKind); + return EmptyJsonObject; + } + + var scoresheetSectionPrompt = BuildScoresheetSectionPrompt(request); + return await ExecutePromptWithRetryAsync( + promptType: "ScoresheetSection", + systemPrompt: ScoresheetPrompts.SectionSystemPrompt, + userPrompt: scoresheetSectionPrompt, + maxTokens: ScoresheetSectionMaxTokens, + normalizeResponse: raw => NormalizeScoresheetSectionResponse(raw, request.SectionSchema), + isValidNormalizedResponse: normalized => IsCompleteScoresheetSectionResponse(normalized, request.SectionSchema), + fallbackResponse: EmptyJsonObject); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", request?.SectionName); + return EmptyJsonObject; + } + } + + private string BuildScoresheetAllPrompt(ScoresheetAllRequest request) + { + var emptyObject = CreateEmptyJsonObject(); + var questionsPayload = request.Questions.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Questions; + var dataPayload = request.Data.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Data; + var attachmentsPayload = BuildScoresheetAttachmentPromptItems(request.Attachments); + + return $@"DATA +{JsonSerializer.Serialize(dataPayload, _prettyJsonOptions)} + +ATTACHMENTS +{JsonSerializer.Serialize(attachmentsPayload, _prettyJsonOptions)} + +QUESTIONS +{JsonSerializer.Serialize(questionsPayload, _prettyJsonOptions)} + +OUTPUT +{ScoresheetPrompts.AllOutputTemplate} + +RULES +{ScoresheetPrompts.AllRules}"; + } + + private string BuildScoresheetSectionPrompt(ScoresheetSectionRequest request) + { + var emptyObject = CreateEmptyJsonObject(); + var sectionSchemaPayload = request.SectionSchema.ValueKind == JsonValueKind.Undefined ? emptyObject : request.SectionSchema; + var dataPayload = request.Data.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Data; + var attachmentsPayload = BuildScoresheetAttachmentPromptItems(request.Attachments); + var responseTemplate = BuildScoresheetSectionResponseTemplate(sectionSchemaPayload); + var sectionName = request.SectionName ?? string.Empty; + var section = new + { + name = sectionName, + questions = sectionSchemaPayload + }; + + return $@"DATA +{JsonSerializer.Serialize(dataPayload, _prettyJsonOptions)} + +ATTACHMENTS +{JsonSerializer.Serialize(attachmentsPayload, _prettyJsonOptions)} + +SECTION +{JsonSerializer.Serialize(section, _prettyJsonOptions)} + +RESPONSE +{JsonSerializer.Serialize(responseTemplate, _prettyJsonOptions)} + +OUTPUT +{ScoresheetPrompts.SectionOutputTemplate} + +RULES +{ScoresheetPrompts.SectionRules}"; + } + + private static Dictionary BuildScoresheetSectionResponseTemplate(JsonElement sectionSchemaPayload) + { + var template = new Dictionary(StringComparer.Ordinal); + var questions = EnumerateSectionQuestions(sectionSchemaPayload); + + foreach (var question in questions) + { + if (!TryGetQuestionId(question, out var questionId)) + { + continue; + } + + template[questionId] = new Dictionary + { + [AIJsonKeys.Answer] = string.Empty, + [AIJsonKeys.Rationale] = string.Empty, + [AIJsonKeys.Confidence] = 0 + }; + } + + return template; + } + + private static List BuildScoresheetAttachmentPromptItems(List attachments) + { + return attachments? + .Where(attachment => attachment != null && !string.IsNullOrWhiteSpace(attachment.Summary)) + .Select(attachment => (object)new + { + name = string.IsNullOrWhiteSpace(attachment.Name) ? "attachment" : attachment.Name.Trim(), + summary = attachment.Summary.Trim() + }) + .ToList() ?? new List(); + } + + private static bool IsValidScoresheetQuestionsPayload(JsonElement questions) + { + return questions.ValueKind == JsonValueKind.Object || questions.ValueKind == JsonValueKind.Array; + } + + private static bool IsValidScoresheetSectionSchemaPayload(JsonElement sectionSchema) + { + return sectionSchema.ValueKind == JsonValueKind.Object || sectionSchema.ValueKind == JsonValueKind.Array; + } + + private string BuildEmptyAnalysisResponseJson() + { + var emptyResponse = new ApplicationAnalysisResponse + { + Rating = ScoreMedium, + Errors = new List(), + Warnings = new List(), + Summaries = new List(), + Dismissed = new List() + }; + + return JsonSerializer.Serialize(emptyResponse, _prettyJsonOptions); + } + + private static JsonElement CreateEmptyJsonObject() + { + return JsonSerializer.SerializeToElement(new { }); + } + + private string NormalizeScoresheetAllResponse(string response) + { + if (!TryParseJsonObjectFromResponse(response, out var responseObject)) + { + _logger.LogError("Invalid scoresheet-all JSON response."); + return EmptyJsonObject; + } + + return JsonSerializer.Serialize(responseObject, _prettyJsonOptions); + } + + private string NormalizeScoresheetSectionResponse(string response, JsonElement sectionSchemaPayload) + { + if (!TryParseJsonObjectFromResponse(response, out var responseObject)) + { + _logger.LogError("Invalid scoresheet-section JSON response."); + return EmptyJsonObject; + } + + var questionSpecs = BuildSectionQuestionSpecs(sectionSchemaPayload); + var normalized = new Dictionary(); + IEnumerable questionIds = questionSpecs.Count > 0 + ? questionSpecs.Keys + : responseObject.EnumerateObject().Select(p => p.Name); + + foreach (var questionId in questionIds) + { + responseObject.TryGetProperty(questionId, out var value); + var answer = value.ValueKind == JsonValueKind.Undefined ? string.Empty : value.ToString(); + var rationale = string.Empty; + var confidence = 0; + + if (value.ValueKind == JsonValueKind.Object) + { + if (value.TryGetProperty(AIJsonKeys.Answer, out var answerProp)) + { + answer = answerProp.ToString(); + } + + if (value.TryGetProperty(AIJsonKeys.Rationale, out var rationaleProp)) + { + rationale = rationaleProp.ToString(); + } + + if (value.TryGetProperty(AIJsonKeys.Confidence, out var confidenceProp)) + { + confidence = NormalizeConfidenceIncrement(ParseConfidenceValue(confidenceProp)); + } + } + + questionSpecs.TryGetValue(questionId, out var questionSpec); + var normalizedAnswer = NormalizeAnswerByQuestionType(answer, questionSpec); + var normalizedRationale = rationale?.Trim() ?? string.Empty; + var normalizedConfidence = NormalizeConfidenceIncrement(confidence); + + normalized[questionId] = new Dictionary + { + [AIJsonKeys.Answer] = normalizedAnswer, + [AIJsonKeys.Rationale] = normalizedRationale, + [AIJsonKeys.Confidence] = normalizedConfidence + }; + } + + return JsonSerializer.Serialize(normalized, _prettyJsonOptions); + } + + private static Dictionary BuildSectionQuestionSpecs(JsonElement sectionSchemaPayload) + { + var specs = new Dictionary(StringComparer.Ordinal); + foreach (var question in EnumerateSectionQuestions(sectionSchemaPayload)) + { + if (!TryGetQuestionId(question, out var questionId)) + { + continue; + } + + var spec = new SectionQuestionSpec + { + QuestionType = question.TryGetProperty("type", out var typeProp) + ? typeProp.GetString() ?? string.Empty + : string.Empty + }; + + if (question.TryGetProperty("options", out var options) && options.ValueKind == JsonValueKind.Array) + { + foreach (var option in options.EnumerateArray()) + { + if (!option.TryGetProperty("number", out var numberProp)) + { + continue; + } + + var number = numberProp.ValueKind == JsonValueKind.Number + ? numberProp.GetInt32().ToString() + : numberProp.ToString(); + + if (string.IsNullOrWhiteSpace(number)) + { + continue; + } + + spec.OptionNumbers.Add(number); + var label = option.TryGetProperty("value", out var valueProp) ? valueProp.ToString() : string.Empty; + spec.OptionLabels[number] = label ?? string.Empty; + } + } + + specs[questionId] = spec; + } + + return specs; + } + + private static IEnumerable EnumerateSectionQuestions(JsonElement sectionSchemaPayload) + { + if (sectionSchemaPayload.ValueKind == JsonValueKind.Array) + { + foreach (var question in sectionSchemaPayload.EnumerateArray()) + { + if (question.ValueKind == JsonValueKind.Object) + { + yield return question; + } + } + } + else if (sectionSchemaPayload.ValueKind == JsonValueKind.Object && + sectionSchemaPayload.TryGetProperty("questions", out var questions) && + questions.ValueKind == JsonValueKind.Array) + { + foreach (var question in questions.EnumerateArray()) + { + if (question.ValueKind == JsonValueKind.Object) + { + yield return question; + } + } + } + } + + private static bool TryGetQuestionId(JsonElement question, out string questionId) + { + questionId = string.Empty; + if (!question.TryGetProperty("id", out var idProp) || idProp.ValueKind != JsonValueKind.String) + { + return false; + } + + questionId = idProp.GetString() ?? string.Empty; + return !string.IsNullOrWhiteSpace(questionId); + } + + private static object NormalizeAnswerByQuestionType(string answer, SectionQuestionSpec? questionSpec) + { + var normalizedAnswer = answer?.Trim() ?? string.Empty; + var questionType = questionSpec?.QuestionType ?? string.Empty; + + if (questionType.Equals("YesNo", StringComparison.OrdinalIgnoreCase)) + { + if (normalizedAnswer.Equals("Yes", StringComparison.OrdinalIgnoreCase)) + { + return "Yes"; + } + + if (normalizedAnswer.Equals("No", StringComparison.OrdinalIgnoreCase)) + { + return "No"; + } + + return "No"; + } + + if (questionType.Equals("Number", StringComparison.OrdinalIgnoreCase)) + { + if (decimal.TryParse(normalizedAnswer, out var decimalAnswer)) + { + return decimalAnswer; + } + + return 0; + } + + if (questionType.Equals("SelectList", StringComparison.OrdinalIgnoreCase)) + { + return NormalizeSelectListAnswer(normalizedAnswer, questionSpec); + } + + if (questionType.Equals("Text", StringComparison.OrdinalIgnoreCase) || + questionType.Equals("TextArea", StringComparison.OrdinalIgnoreCase)) + { + return normalizedAnswer; + } + + return normalizedAnswer; + } + + private static string NormalizeSelectListAnswer(string answer, SectionQuestionSpec? questionSpec) + { + var options = questionSpec?.OptionNumbers ?? new List(); + if (options.Count == 0) + { + return answer; + } + + if (options.Contains(answer)) + { + return answer; + } + + if (int.TryParse(answer, out var parsedAnswer) && options.Contains(parsedAnswer.ToString())) + { + return parsedAnswer.ToString(); + } + + return answer; + } + + private sealed class SectionQuestionSpec + { + public string QuestionType { get; set; } = string.Empty; + public List OptionNumbers { get; set; } = new(); + public Dictionary OptionLabels { get; set; } = new(StringComparer.OrdinalIgnoreCase); + } + + private static int NormalizeConfidenceIncrement(int confidence) + { + var rounded = (int)Math.Round(confidence / 5.0, MidpointRounding.AwayFromZero) * 5; + return Math.Clamp(rounded, 0, 100); + } + + private static int ParseConfidenceValue(JsonElement confidenceProp) + { + if (confidenceProp.ValueKind == JsonValueKind.Number) + { + if (confidenceProp.TryGetInt32(out var intValue)) + { + return intValue; + } + + if (confidenceProp.TryGetDouble(out var doubleValue)) + { + return (int)Math.Round(doubleValue, MidpointRounding.AwayFromZero); + } + } + + if (confidenceProp.ValueKind == JsonValueKind.String) + { + var raw = confidenceProp.GetString(); + if (int.TryParse(raw, out var parsedInt)) + { + return parsedInt; + } + + if (double.TryParse(raw, out var parsedDouble)) + { + return (int)Math.Round(parsedDouble, MidpointRounding.AwayFromZero); + } + } + + return 0; + } + + private void LogPromptOutput(string promptType, string output) + { + if (!LogPayloads) + { + return; + } + + var formattedOutput = FormatPromptOutputForLog(promptType, output); + _logger.LogDebug( + "AI {PromptType} model output payload: {ModelOutput}", + promptType, + formattedOutput); + WriteAiPromptLog(promptType, "OUTPUT", formattedOutput); + } + + private void LogPromptInput(string promptType, string? systemPrompt, string userPrompt) + { + if (!LogPayloads) + { + return; + } + + var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); + _logger.LogDebug( + "AI {PromptType} input payload: {PromptInput}", + promptType, + formattedInput); + WriteAiPromptLog(promptType, "INPUT", formattedInput); + } + + private void WriteAiPromptLog(string promptType, string payloadType, string payload) + { + if (!LogPayloads) + { + return; + } + + try + { + var now = DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss zzz"); + var logPath = Path.Combine(AppContext.BaseDirectory, AiPromptLogRelativePath); + EnsureAiPromptLogInitialized(logPath); + + var entry = $"{now} [{promptType}] {payloadType}\n{payload}\n\n"; + File.AppendAllText(logPath, entry); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write AI prompt log file."); + } + } + + private static void EnsureAiPromptLogInitialized(string logPath) + { + var directory = Path.GetDirectoryName(logPath); + if (!string.IsNullOrWhiteSpace(directory)) + { + Directory.CreateDirectory(directory); + } + + // Reset once per process run so each fresh app run starts with a clean AI prompt log. + if (Interlocked.Exchange(ref _aiPromptLogInitialized, 1) == 0) + { + File.WriteAllText(logPath, string.Empty); + } + } + + private static string FormatPromptInputForLog(string? systemPrompt, string userPrompt) + { + var normalizedSystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? string.Empty : systemPrompt.Trim(); + var normalizedUserPrompt = string.IsNullOrWhiteSpace(userPrompt) ? string.Empty : userPrompt.Trim(); + return $"SYSTEM_PROMPT\n{normalizedSystemPrompt}\n\nUSER_PROMPT\n{normalizedUserPrompt}"; + } + + private string FormatPromptOutputForLog(string promptType, string output) + { + if (string.IsNullOrWhiteSpace(output)) + { + return string.Empty; + } + + // For JSON contracts, log only normalized payload JSON. + if (TryParseJsonObjectFromResponse(output, out var jsonObject)) + { + return JsonSerializer.Serialize(jsonObject, _prettyJsonOptions); + } + + return output.Trim(); + } + + private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) + { + objectElement = default; + var cleaned = CleanJsonResponse(response); + if (string.IsNullOrWhiteSpace(cleaned)) + { + return false; + } + + try + { + using var doc = JsonDocument.Parse(cleaned); + if (doc.RootElement.ValueKind != JsonValueKind.Object) + { + return false; + } + + objectElement = doc.RootElement.Clone(); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static bool IsEmptyJsonObject(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return true; + } + + try + { + using var doc = JsonDocument.Parse(json); + return doc.RootElement.ValueKind == JsonValueKind.Object && + !doc.RootElement.EnumerateObject().Any(); + } + catch (JsonException) + { + return true; + } + } + + private async Task ExecutePromptWithRetryAsync( + string promptType, + string systemPrompt, + string userPrompt, + int maxTokens, + Func normalizeResponse, + Func isValidNormalizedResponse, + string fallbackResponse, + int maxAttempts = 2) + { + LogPromptInput(promptType, systemPrompt, userPrompt); + + for (var attempt = 1; attempt <= maxAttempts; attempt++) + { + var rawResponse = await GenerateCompletionAsync(new AICompletionRequest + { + UserPrompt = userPrompt, + SystemPrompt = systemPrompt, + MaxTokens = maxTokens + }); + + var outputType = attempt == 1 ? promptType : $"{promptType}Retry"; + LogPromptOutput(outputType, rawResponse); + + var normalized = normalizeResponse(rawResponse); + if (isValidNormalizedResponse(normalized)) + { + return normalized; + } + + if (attempt < maxAttempts) + { + _logger.LogWarning( + "{PromptType} response failed output-shape validation on attempt {Attempt}/{MaxAttempts}. Retrying.", + promptType, + attempt, + maxAttempts); + } + } + + return fallbackResponse; + } + + private static bool IsValidAnalysisNormalizedResponse(string normalizedJson) + { + if (!TryParseJsonObjectFromResponse(normalizedJson, out var root)) + { + return false; + } + + return root.TryGetProperty("rating", out _) && + root.TryGetProperty("errors", out _) && + root.TryGetProperty("warnings", out _) && + root.TryGetProperty("summaries", out _); + } + + private static bool IsCompleteScoresheetSectionResponse(string normalizedJson, JsonElement sectionSchemaPayload) + { + if (!TryParseJsonObjectFromResponse(normalizedJson, out var root)) + { + return false; + } + + var expectedQuestionIds = EnumerateSectionQuestions(sectionSchemaPayload) + .Select(q => TryGetQuestionId(q, out var id) ? id : string.Empty) + .Where(id => !string.IsNullOrWhiteSpace(id)) + .ToList(); + + if (expectedQuestionIds.Count == 0) + { + return false; + } + + foreach (var questionId in expectedQuestionIds) + { + if (!root.TryGetProperty(questionId, out var answerObject) || answerObject.ValueKind != JsonValueKind.Object) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Answer, out var answerProp) || + string.IsNullOrWhiteSpace(answerProp.ToString())) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Rationale, out var rationaleProp) || + string.IsNullOrWhiteSpace(rationaleProp.ToString())) + { + return false; + } + + if (!answerObject.TryGetProperty(AIJsonKeys.Confidence, out _)) + { + return false; + } + } + + return true; + } + + private static string CleanJsonResponse(string response) + { + if (string.IsNullOrWhiteSpace(response)) + { + return string.Empty; + } + + var cleaned = response.Trim(); + + if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) + { + var startIndex = cleaned.IndexOf('\n'); + if (startIndex >= 0) + { + cleaned = cleaned.Substring(startIndex + 1); + } + } + + if (cleaned.EndsWith("```", StringComparison.Ordinal)) + { + var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); + if (lastIndex > 0) + { + cleaned = cleaned.Substring(0, lastIndex); + } + } + + return cleaned.Trim(); + } + } +} + + diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.v1.txt new file mode 100644 index 000000000..e11dce3c9 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.v1.txt @@ -0,0 +1,13 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptCoreRules + { + public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; + public const string NoInvention = "- Do not invent missing details."; + public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; + public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; + public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; + public const string ValidJsonOnly = "- Return valid JSON only."; + public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.v1.txt new file mode 100644 index 000000000..701a43e74 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.v1.txt @@ -0,0 +1,14 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptHeader + { + public static string Build(string role, string task) + { + return $@"ROLE +{role} + +TASK +{task}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.v1.txt new file mode 100644 index 000000000..bfe883d64 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.v1.txt @@ -0,0 +1,85 @@ +namespace Unity.GrantManager.AI +{ + internal static class ScoresheetPrompts + { + public static readonly string AllSystemPrompt = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, QUESTIONS, OUTPUT, and RULES, provide answers for all scoresheet questions."); + + public const string AllOutputTemplate = @"{ + """": """" +}"; + + public const string AllRules = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer per question ID in QUESTIONS. +- Do not omit any question IDs from QUESTIONS. +- Do not add keys that are not question IDs from QUESTIONS. +- The ""answer"" value type must match the question type. +- For numeric questions, return a numeric value within the allowed range. +- For yes/no questions, return exactly ""Yes"" or ""No"". +- For select list questions, return only the selected options.number as a string and never return option label text. +- For text and text area questions, return concise, evidence-based text. +- For text and text area questions, include concise source-grounded rationale from the provided input content. +- If explicit evidence is insufficient, choose the most conservative valid answer. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static readonly string SectionSystemPrompt = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + + public const string SectionOutputTemplate = @"{ + """": { + ""answer"": """", + ""rationale"": """", + ""confidence"": + } +}"; + + public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer object per question ID in SECTION.questions. +- Do not omit any question IDs from SECTION.questions. +- Do not add keys that are not question IDs from SECTION.questions. +- Use RESPONSE as the output contract and fill every placeholder value. +- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. +- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". +- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. +- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. +- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. +- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. +- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. +- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. +- Do not treat missing or non-contradictory information as evidence. +- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. +- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. +- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". +- For numeric questions, answer must be a numeric value within the allowed range. +- For numeric questions, answer must never be blank. +- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. +- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. +- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. +- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. +- Never return 0 for select list answers unless 0 exists as an explicit option number. +- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. +- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. +- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. +- Do not leave rationale empty when answer is populated. +- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". +- If any answer object is incomplete, regenerate the full JSON response before returning it. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + } +} + + + From 9ee1136a83d352d957f8c318bdde34baf2268a22 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 16:21:16 -0800 Subject: [PATCH 02/10] AB#32009 Normalize prompt baseline version folders and snapshots --- .../AI/Prompts/Baselines/README.md | 8 + ...ysisPrompts.v1.txt => AnalysisPrompts.txt} | 0 ...ntPrompts.v1.txt => AttachmentPrompts.txt} | 0 ...Service.v0.cs.txt => OpenAIService.cs.txt} | 0 .../Baselines/v0/OpenAIService.v1.cs.txt | 1134 ----------------- ...ptCoreRules.v1.txt => PromptCoreRules.txt} | 0 .../{PromptHeader.v1.txt => PromptHeader.txt} | 0 ...etPrompts.v1.txt => ScoresheetPrompts.txt} | 0 .../Baselines/v1/AnalysisPrompts.cs.txt | 126 ++ .../Baselines/v1/AttachmentPrompts.cs.txt | 27 + .../Prompts/Baselines/v1/OpenAIService.cs.txt | 833 ++++++++++++ .../Baselines/v1/PromptCoreRules.cs.txt | 13 + .../Prompts/Baselines/v1/PromptHeader.cs.txt | 14 + .../Baselines/v1/ScoresheetPrompts.cs.txt | 80 ++ 14 files changed, 1101 insertions(+), 1134 deletions(-) create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{AnalysisPrompts.v1.txt => AnalysisPrompts.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{AttachmentPrompts.v1.txt => AttachmentPrompts.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{OpenAIService.v0.cs.txt => OpenAIService.cs.txt} (100%) delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{PromptCoreRules.v1.txt => PromptCoreRules.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{PromptHeader.v1.txt => PromptHeader.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{ScoresheetPrompts.v1.txt => ScoresheetPrompts.txt} (100%) create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md new file mode 100644 index 000000000..4106b7aa0 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md @@ -0,0 +1,8 @@ +# Prompt Baselines + +- `v0`: legacy prompt/service snapshots used as historical baseline. +- `v1`: current runtime prompt/service snapshots. + +Versioning convention: +- Folder name is the baseline version. +- Filenames inside each folder are normalized and do not include version suffixes. diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.v1.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.v1.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v0.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v0.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt deleted file mode 100644 index e421036eb..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.v1.cs.txt +++ /dev/null @@ -1,1134 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Volo.Abp.DependencyInjection; - -namespace Unity.GrantManager.AI -{ - public class OpenAIService : IAIService, ITransientDependency - { - private readonly HttpClient _httpClient; - private readonly IConfiguration _configuration; - private readonly ILogger _logger; - private readonly ITextExtractionService _textExtractionService; - private readonly JsonSerializerOptions _prettyJsonOptions = new() { WriteIndented = true }; - - private string? ApiKey => _configuration["AI:OpenAI:ApiKey"]; - private bool HasApiKey => !string.IsNullOrWhiteSpace(ApiKey); - private string? ApiUrl => _configuration["AI:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; - private bool LogPayloads => _configuration.GetValue("AI:Logging:LogPayloads") ?? false; - private readonly string NoApiKeyMessage = "OpenAI API key is not configured"; - private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; - private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; - private const string GenericFailureMessage = "AI analysis failed - please try again later."; - private const string NoSummaryGeneratedMessage = "No summary generated."; - private const string EmptyJsonObject = "{}"; - private const int ScoresheetAllMaxTokens = 2000; - private const int ScoresheetSectionMaxTokens = 3200; - private const int AnalysisMaxTokens = 1600; - private const string DefaultContentType = "application/octet-stream"; - private const int AttachmentSummaryMaxTokens = 240; - private const string AttachmentSummaryUnavailableMessage = "AI analysis not available for this attachment"; - private const string ScoreHigh = "HIGH"; - private const string ScoreMedium = "MEDIUM"; - private const string ScoreLow = "LOW"; - private const string AiPromptLogRelativePath = "logs/ai-prompts.log"; - private static int _aiPromptLogInitialized; - - public OpenAIService(HttpClient httpClient, IConfiguration configuration, ILogger logger, ITextExtractionService textExtractionService) - { - _httpClient = httpClient; - _configuration = configuration; - _logger = logger; - _textExtractionService = textExtractionService; - } - - public Task IsAvailableAsync() - { - if (!HasApiKey) - { - _logger.LogWarning("Error: {Message}", NoApiKeyMessage); - return Task.FromResult(false); - } - - return Task.FromResult(true); - } - - public async Task GenerateCompletionAsync(AICompletionRequest request) - { - if (request == null) - { - _logger.LogWarning("AI completion request was null."); - return GenericFailureMessage; - } - - var userPrompt = request.UserPrompt ?? string.Empty; - var systemPrompt = request.SystemPrompt; - var maxTokens = request.MaxTokens <= 0 ? 150 : request.MaxTokens; - var temperature = request.Temperature ?? 0.3; - - return await ExecuteChatCompletionAsync(userPrompt, systemPrompt, maxTokens, temperature); - } - - private async Task ExecuteChatCompletionAsync( - string userPrompt, - string? systemPrompt = null, - int maxTokens = 150, - double temperature = 0.3) - { - if (!HasApiKey) - { - _logger.LogWarning("Error: {Message}", NoApiKeyMessage); - return ServiceNotConfiguredMessage; - } - - _logger.LogDebug( - "Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", - userPrompt?.Length ?? 0, - maxTokens); - - try - { - string resolvedSystemPrompt = systemPrompt ?? "You are a professional grant analyst for the BC Government."; - - var requestBody = new Dictionary - { - ["messages"] = new[] - { - new { role = "system", content = resolvedSystemPrompt }, - new { role = "user", content = userPrompt ?? string.Empty } - }, - ["max_tokens"] = maxTokens, - ["temperature"] = temperature - }; - - var json = JsonSerializer.Serialize(requestBody); - var authValue = ApiKey!.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase) - ? ApiKey.Substring("Bearer ".Length).Trim() - : ApiKey; - - using var request = new HttpRequestMessage(HttpMethod.Post, ApiUrl); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authValue); - request.Content = new StringContent(json, Encoding.UTF8, "application/json"); - - var response = await _httpClient.SendAsync(request); - var responseContent = await response.Content.ReadAsStringAsync() ?? string.Empty; - - _logger.LogDebug( - "OpenAI chat completions response received. StatusCode: {StatusCode}, ResponseLength: {ResponseLength}", - response.StatusCode, - responseContent?.Length ?? 0); - - if (!response.IsSuccessStatusCode) - { - _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); - return ServiceTemporarilyUnavailableMessage; - } - - using var jsonDoc = JsonDocument.Parse(responseContent ?? string.Empty); - var choices = jsonDoc.RootElement.GetProperty("choices"); - if (choices.GetArrayLength() > 0) - { - var message = choices[0].GetProperty("message"); - return message.GetProperty("content").GetString() ?? NoSummaryGeneratedMessage; - } - - return NoSummaryGeneratedMessage; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating AI summary"); - return GenericFailureMessage; - } - } - - // Canonical attachment summary prompt contract is defined by: - // AttachmentPrompts.SystemPrompt, AttachmentPrompts.OutputSection, and AttachmentPrompts.RulesSection. - public async Task GenerateAttachmentSummaryAsync(AttachmentSummaryRequest request) - { - try - { - if (request == null) - { - _logger.LogWarning("Attachment summary request was null."); - return $"{AttachmentSummaryUnavailableMessage} (unknown)."; - } - - var normalizedFileName = string.IsNullOrWhiteSpace(request.FileName) ? "unknown" : request.FileName.Trim(); - var normalizedContentType = string.IsNullOrWhiteSpace(request.ContentType) ? DefaultContentType : request.ContentType.Trim(); - var normalizedFileContent = request.FileContent ?? Array.Empty(); - var extractedText = await _textExtractionService.ExtractTextAsync(normalizedFileName, normalizedFileContent, normalizedContentType); - - var hasExtractedText = !string.IsNullOrWhiteSpace(extractedText); - if (hasExtractedText) - { - _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText!.Length, normalizedFileName); - } - else - { - _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", normalizedFileName); - } - - var attachmentInput = new AttachmentPromptInput - { - Name = normalizedFileName, - Text = hasExtractedText ? extractedText : null - }; - - var contentToAnalyze = BuildAttachmentSummaryPrompt(attachmentInput); - return await ExecutePromptWithRetryAsync( - promptType: "AttachmentSummary", - systemPrompt: AttachmentPrompts.SystemPrompt, - userPrompt: contentToAnalyze, - maxTokens: AttachmentSummaryMaxTokens, - normalizeResponse: NormalizeAttachmentSummaryResponse, - isValidNormalizedResponse: normalized => !string.IsNullOrWhiteSpace(normalized), - fallbackResponse: string.Empty); - } - catch (Exception ex) - { - var fileName = request?.FileName ?? "unknown"; - _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); - return $"{AttachmentSummaryUnavailableMessage} ({fileName})."; - } - } - - private string BuildAttachmentSummaryPrompt(AttachmentPromptInput attachmentInput) - { - return $@"ATTACHMENT -{JsonSerializer.Serialize(attachmentInput, _prettyJsonOptions)} - -{AttachmentPrompts.OutputSection} - -RULES -{AttachmentPrompts.RulesSection}"; - } - - private string NormalizeAttachmentSummaryResponse(string response) - { - if (!TryParseJsonObjectFromResponse(response, out var responseObject)) - { - return string.Empty; - } - - if (responseObject.TryGetProperty(AIJsonKeys.Summary, out var summaryProp) && - summaryProp.ValueKind == JsonValueKind.String) - { - return summaryProp.GetString()?.Trim() ?? string.Empty; - } - - return responseObject.ToString().Trim(); - } - - // Canonical analysis prompt contract is defined by: - // AnalysisPrompts.DefaultRubric, AnalysisPrompts.ScoreRules, AnalysisPrompts.OutputTemplate, - // AnalysisPrompts.Rules, and AnalysisPrompts.SystemPrompt. - public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) - { - if (!HasApiKey) - { - _logger.LogWarning("{Message}", NoApiKeyMessage); - return ServiceNotConfiguredMessage; - } - - try - { - if (request == null) - { - _logger.LogWarning("Application analysis request was null."); - return BuildEmptyAnalysisResponseJson(); - } - - var emptyObject = CreateEmptyJsonObject(); - var schemaPayload = request.Schema.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Schema; - var dataPayload = request.Data.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Data; - - if (schemaPayload.ValueKind != JsonValueKind.Object || dataPayload.ValueKind != JsonValueKind.Object) - { - _logger.LogWarning( - "Invalid application analysis request payload shape. Schema kind: {SchemaKind}, Data kind: {DataKind}.", - schemaPayload.ValueKind, - dataPayload.ValueKind); - return BuildEmptyAnalysisResponseJson(); - } - - var attachmentsPayload = request.Attachments? - .Where(a => a != null && !string.IsNullOrWhiteSpace(a.Summary)) - .Select(a => new ApplicationAnalysisAttachment - { - Name = string.IsNullOrWhiteSpace(a.Name) ? "attachment" : a.Name.Trim(), - Summary = a.Summary.Trim() - }) - .ToList() ?? new List(); - - var rubricText = !string.IsNullOrWhiteSpace(request.Rubric) ? request.Rubric : AnalysisPrompts.DefaultRubric; - var analysisPrompt = BuildAnalysisPrompt(schemaPayload, dataPayload, attachmentsPayload, rubricText); - return await ExecutePromptWithRetryAsync( - promptType: "ApplicationAnalysis", - systemPrompt: AnalysisPrompts.SystemPrompt, - userPrompt: analysisPrompt, - maxTokens: AnalysisMaxTokens, - normalizeResponse: NormalizeAnalysisResponse, - isValidNormalizedResponse: IsValidAnalysisNormalizedResponse, - fallbackResponse: BuildEmptyAnalysisResponseJson()); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error analyzing application"); - return BuildEmptyAnalysisResponseJson(); - } - } - - private string NormalizeAnalysisResponse(string analysisJson) - { - try - { - if (!TryParseJsonObjectFromResponse(analysisJson, out var analysisObject)) - { - _logger.LogError("Invalid analysis JSON response."); - return BuildEmptyAnalysisResponseJson(); - } - - var parseOptions = new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true - }; - - var model = JsonSerializer.Deserialize(analysisObject.GetRawText(), parseOptions); - if (model == null) - { - return BuildEmptyAnalysisResponseJson(); - } - - model.Errors ??= new List(); - model.Warnings ??= new List(); - model.Summaries ??= new List(); - model.Dismissed ??= new List(); - - model.Rating = NormalizeRating(model.Rating); - - foreach (var error in model.Errors) - { - error.Id = string.IsNullOrWhiteSpace(error.Id) ? Guid.NewGuid().ToString() : error.Id; - } - - foreach (var warning in model.Warnings) - { - warning.Id = string.IsNullOrWhiteSpace(warning.Id) ? Guid.NewGuid().ToString() : warning.Id; - } - - model.Dismissed = model.Dismissed - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Distinct(StringComparer.Ordinal) - .ToList(); - - var normalizedOutput = new ApplicationAnalysisResponse - { - Rating = model.Rating, - Errors = model.Errors, - Warnings = model.Warnings, - Summaries = model.Summaries, - Dismissed = model.Dismissed - }; - - return JsonSerializer.Serialize(normalizedOutput, _prettyJsonOptions); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error normalizing analysis response."); - return BuildEmptyAnalysisResponseJson(); - } - } - - private static string NormalizeRating(string? rating) - { - var normalized = rating?.Trim().ToUpperInvariant(); - return normalized switch - { - ScoreHigh => ScoreHigh, - ScoreMedium => ScoreMedium, - ScoreLow => ScoreLow, - _ => ScoreMedium - }; - } - - private string BuildAnalysisPrompt( - JsonElement schemaPayload, - JsonElement dataPayload, - List attachmentsPayload, - string rubricText) - { - var analysisAttachments = (attachmentsPayload ?? new List()) - .Select(a => new AnalysisAttachmentPromptItem - { - Name = a.Name, - Summary = a.Summary - }) - .ToList(); - - return $@"SCHEMA -{JsonSerializer.Serialize(schemaPayload, _prettyJsonOptions)} - -DATA -{JsonSerializer.Serialize(dataPayload, _prettyJsonOptions)} - -ATTACHMENTS -{JsonSerializer.Serialize(analysisAttachments, _prettyJsonOptions)} - -RUBRIC -{rubricText ?? AnalysisPrompts.DefaultRubric} - -SCORE -{AnalysisPrompts.ScoreRules} - -OUTPUT -{AnalysisPrompts.OutputTemplate} - -RULES -{AnalysisPrompts.Rules}"; - } - // Canonical scoresheet-all prompt contract is defined by: - // ScoresheetPrompts.AllSystemPrompt, ScoresheetPrompts.AllOutputTemplate, and ScoresheetPrompts.AllRules. - public async Task GenerateScoresheetAllAnswersAsync(ScoresheetAllRequest request) - { - if (!HasApiKey) - { - _logger.LogWarning("{Message}", NoApiKeyMessage); - return EmptyJsonObject; - } - - try - { - if (request == null) - { - _logger.LogWarning("Scoresheet-all request was null."); - return EmptyJsonObject; - } - - if (!IsValidScoresheetQuestionsPayload(request.Questions)) - { - _logger.LogWarning( - "Invalid scoresheet-all questions payload shape. Questions kind: {QuestionsKind}.", - request.Questions.ValueKind); - return EmptyJsonObject; - } - - var scoresheetPrompt = BuildScoresheetAllPrompt(request); - return await ExecutePromptWithRetryAsync( - promptType: "ScoresheetAll", - systemPrompt: ScoresheetPrompts.AllSystemPrompt, - userPrompt: scoresheetPrompt, - maxTokens: ScoresheetAllMaxTokens, - normalizeResponse: NormalizeScoresheetAllResponse, - isValidNormalizedResponse: normalized => !IsEmptyJsonObject(normalized), - fallbackResponse: EmptyJsonObject); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet answers"); - return EmptyJsonObject; - } - } - // Canonical scoresheet-section prompt contract is defined by: - // ScoresheetPrompts.SectionSystemPrompt, ScoresheetPrompts.SectionOutputTemplate, and ScoresheetPrompts.SectionRules. - public async Task GenerateScoresheetSectionAnswersAsync(ScoresheetSectionRequest request) - { - if (!HasApiKey) - { - _logger.LogWarning("{Message}", NoApiKeyMessage); - return EmptyJsonObject; - } - - try - { - if (request == null) - { - _logger.LogWarning("Scoresheet-section request was null."); - return EmptyJsonObject; - } - - if (!IsValidScoresheetSectionSchemaPayload(request.SectionSchema)) - { - _logger.LogWarning( - "Invalid scoresheet-section schema payload shape. SectionSchema kind: {SectionSchemaKind}.", - request.SectionSchema.ValueKind); - return EmptyJsonObject; - } - - var scoresheetSectionPrompt = BuildScoresheetSectionPrompt(request); - return await ExecutePromptWithRetryAsync( - promptType: "ScoresheetSection", - systemPrompt: ScoresheetPrompts.SectionSystemPrompt, - userPrompt: scoresheetSectionPrompt, - maxTokens: ScoresheetSectionMaxTokens, - normalizeResponse: raw => NormalizeScoresheetSectionResponse(raw, request.SectionSchema), - isValidNormalizedResponse: normalized => IsCompleteScoresheetSectionResponse(normalized, request.SectionSchema), - fallbackResponse: EmptyJsonObject); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", request?.SectionName); - return EmptyJsonObject; - } - } - - private string BuildScoresheetAllPrompt(ScoresheetAllRequest request) - { - var emptyObject = CreateEmptyJsonObject(); - var questionsPayload = request.Questions.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Questions; - var dataPayload = request.Data.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Data; - var attachmentsPayload = BuildScoresheetAttachmentPromptItems(request.Attachments); - - return $@"DATA -{JsonSerializer.Serialize(dataPayload, _prettyJsonOptions)} - -ATTACHMENTS -{JsonSerializer.Serialize(attachmentsPayload, _prettyJsonOptions)} - -QUESTIONS -{JsonSerializer.Serialize(questionsPayload, _prettyJsonOptions)} - -OUTPUT -{ScoresheetPrompts.AllOutputTemplate} - -RULES -{ScoresheetPrompts.AllRules}"; - } - - private string BuildScoresheetSectionPrompt(ScoresheetSectionRequest request) - { - var emptyObject = CreateEmptyJsonObject(); - var sectionSchemaPayload = request.SectionSchema.ValueKind == JsonValueKind.Undefined ? emptyObject : request.SectionSchema; - var dataPayload = request.Data.ValueKind == JsonValueKind.Undefined ? emptyObject : request.Data; - var attachmentsPayload = BuildScoresheetAttachmentPromptItems(request.Attachments); - var responseTemplate = BuildScoresheetSectionResponseTemplate(sectionSchemaPayload); - var sectionName = request.SectionName ?? string.Empty; - var section = new - { - name = sectionName, - questions = sectionSchemaPayload - }; - - return $@"DATA -{JsonSerializer.Serialize(dataPayload, _prettyJsonOptions)} - -ATTACHMENTS -{JsonSerializer.Serialize(attachmentsPayload, _prettyJsonOptions)} - -SECTION -{JsonSerializer.Serialize(section, _prettyJsonOptions)} - -RESPONSE -{JsonSerializer.Serialize(responseTemplate, _prettyJsonOptions)} - -OUTPUT -{ScoresheetPrompts.SectionOutputTemplate} - -RULES -{ScoresheetPrompts.SectionRules}"; - } - - private static Dictionary BuildScoresheetSectionResponseTemplate(JsonElement sectionSchemaPayload) - { - var template = new Dictionary(StringComparer.Ordinal); - var questions = EnumerateSectionQuestions(sectionSchemaPayload); - - foreach (var question in questions) - { - if (!TryGetQuestionId(question, out var questionId)) - { - continue; - } - - template[questionId] = new Dictionary - { - [AIJsonKeys.Answer] = string.Empty, - [AIJsonKeys.Rationale] = string.Empty, - [AIJsonKeys.Confidence] = 0 - }; - } - - return template; - } - - private static List BuildScoresheetAttachmentPromptItems(List attachments) - { - return attachments? - .Where(attachment => attachment != null && !string.IsNullOrWhiteSpace(attachment.Summary)) - .Select(attachment => (object)new - { - name = string.IsNullOrWhiteSpace(attachment.Name) ? "attachment" : attachment.Name.Trim(), - summary = attachment.Summary.Trim() - }) - .ToList() ?? new List(); - } - - private static bool IsValidScoresheetQuestionsPayload(JsonElement questions) - { - return questions.ValueKind == JsonValueKind.Object || questions.ValueKind == JsonValueKind.Array; - } - - private static bool IsValidScoresheetSectionSchemaPayload(JsonElement sectionSchema) - { - return sectionSchema.ValueKind == JsonValueKind.Object || sectionSchema.ValueKind == JsonValueKind.Array; - } - - private string BuildEmptyAnalysisResponseJson() - { - var emptyResponse = new ApplicationAnalysisResponse - { - Rating = ScoreMedium, - Errors = new List(), - Warnings = new List(), - Summaries = new List(), - Dismissed = new List() - }; - - return JsonSerializer.Serialize(emptyResponse, _prettyJsonOptions); - } - - private static JsonElement CreateEmptyJsonObject() - { - return JsonSerializer.SerializeToElement(new { }); - } - - private string NormalizeScoresheetAllResponse(string response) - { - if (!TryParseJsonObjectFromResponse(response, out var responseObject)) - { - _logger.LogError("Invalid scoresheet-all JSON response."); - return EmptyJsonObject; - } - - return JsonSerializer.Serialize(responseObject, _prettyJsonOptions); - } - - private string NormalizeScoresheetSectionResponse(string response, JsonElement sectionSchemaPayload) - { - if (!TryParseJsonObjectFromResponse(response, out var responseObject)) - { - _logger.LogError("Invalid scoresheet-section JSON response."); - return EmptyJsonObject; - } - - var questionSpecs = BuildSectionQuestionSpecs(sectionSchemaPayload); - var normalized = new Dictionary(); - IEnumerable questionIds = questionSpecs.Count > 0 - ? questionSpecs.Keys - : responseObject.EnumerateObject().Select(p => p.Name); - - foreach (var questionId in questionIds) - { - responseObject.TryGetProperty(questionId, out var value); - var answer = value.ValueKind == JsonValueKind.Undefined ? string.Empty : value.ToString(); - var rationale = string.Empty; - var confidence = 0; - - if (value.ValueKind == JsonValueKind.Object) - { - if (value.TryGetProperty(AIJsonKeys.Answer, out var answerProp)) - { - answer = answerProp.ToString(); - } - - if (value.TryGetProperty(AIJsonKeys.Rationale, out var rationaleProp)) - { - rationale = rationaleProp.ToString(); - } - - if (value.TryGetProperty(AIJsonKeys.Confidence, out var confidenceProp)) - { - confidence = NormalizeConfidenceIncrement(ParseConfidenceValue(confidenceProp)); - } - } - - questionSpecs.TryGetValue(questionId, out var questionSpec); - var normalizedAnswer = NormalizeAnswerByQuestionType(answer, questionSpec); - var normalizedRationale = rationale?.Trim() ?? string.Empty; - var normalizedConfidence = NormalizeConfidenceIncrement(confidence); - - normalized[questionId] = new Dictionary - { - [AIJsonKeys.Answer] = normalizedAnswer, - [AIJsonKeys.Rationale] = normalizedRationale, - [AIJsonKeys.Confidence] = normalizedConfidence - }; - } - - return JsonSerializer.Serialize(normalized, _prettyJsonOptions); - } - - private static Dictionary BuildSectionQuestionSpecs(JsonElement sectionSchemaPayload) - { - var specs = new Dictionary(StringComparer.Ordinal); - foreach (var question in EnumerateSectionQuestions(sectionSchemaPayload)) - { - if (!TryGetQuestionId(question, out var questionId)) - { - continue; - } - - var spec = new SectionQuestionSpec - { - QuestionType = question.TryGetProperty("type", out var typeProp) - ? typeProp.GetString() ?? string.Empty - : string.Empty - }; - - if (question.TryGetProperty("options", out var options) && options.ValueKind == JsonValueKind.Array) - { - foreach (var option in options.EnumerateArray()) - { - if (!option.TryGetProperty("number", out var numberProp)) - { - continue; - } - - var number = numberProp.ValueKind == JsonValueKind.Number - ? numberProp.GetInt32().ToString() - : numberProp.ToString(); - - if (string.IsNullOrWhiteSpace(number)) - { - continue; - } - - spec.OptionNumbers.Add(number); - var label = option.TryGetProperty("value", out var valueProp) ? valueProp.ToString() : string.Empty; - spec.OptionLabels[number] = label ?? string.Empty; - } - } - - specs[questionId] = spec; - } - - return specs; - } - - private static IEnumerable EnumerateSectionQuestions(JsonElement sectionSchemaPayload) - { - if (sectionSchemaPayload.ValueKind == JsonValueKind.Array) - { - foreach (var question in sectionSchemaPayload.EnumerateArray()) - { - if (question.ValueKind == JsonValueKind.Object) - { - yield return question; - } - } - } - else if (sectionSchemaPayload.ValueKind == JsonValueKind.Object && - sectionSchemaPayload.TryGetProperty("questions", out var questions) && - questions.ValueKind == JsonValueKind.Array) - { - foreach (var question in questions.EnumerateArray()) - { - if (question.ValueKind == JsonValueKind.Object) - { - yield return question; - } - } - } - } - - private static bool TryGetQuestionId(JsonElement question, out string questionId) - { - questionId = string.Empty; - if (!question.TryGetProperty("id", out var idProp) || idProp.ValueKind != JsonValueKind.String) - { - return false; - } - - questionId = idProp.GetString() ?? string.Empty; - return !string.IsNullOrWhiteSpace(questionId); - } - - private static object NormalizeAnswerByQuestionType(string answer, SectionQuestionSpec? questionSpec) - { - var normalizedAnswer = answer?.Trim() ?? string.Empty; - var questionType = questionSpec?.QuestionType ?? string.Empty; - - if (questionType.Equals("YesNo", StringComparison.OrdinalIgnoreCase)) - { - if (normalizedAnswer.Equals("Yes", StringComparison.OrdinalIgnoreCase)) - { - return "Yes"; - } - - if (normalizedAnswer.Equals("No", StringComparison.OrdinalIgnoreCase)) - { - return "No"; - } - - return "No"; - } - - if (questionType.Equals("Number", StringComparison.OrdinalIgnoreCase)) - { - if (decimal.TryParse(normalizedAnswer, out var decimalAnswer)) - { - return decimalAnswer; - } - - return 0; - } - - if (questionType.Equals("SelectList", StringComparison.OrdinalIgnoreCase)) - { - return NormalizeSelectListAnswer(normalizedAnswer, questionSpec); - } - - if (questionType.Equals("Text", StringComparison.OrdinalIgnoreCase) || - questionType.Equals("TextArea", StringComparison.OrdinalIgnoreCase)) - { - return normalizedAnswer; - } - - return normalizedAnswer; - } - - private static string NormalizeSelectListAnswer(string answer, SectionQuestionSpec? questionSpec) - { - var options = questionSpec?.OptionNumbers ?? new List(); - if (options.Count == 0) - { - return answer; - } - - if (options.Contains(answer)) - { - return answer; - } - - if (int.TryParse(answer, out var parsedAnswer) && options.Contains(parsedAnswer.ToString())) - { - return parsedAnswer.ToString(); - } - - return answer; - } - - private sealed class SectionQuestionSpec - { - public string QuestionType { get; set; } = string.Empty; - public List OptionNumbers { get; set; } = new(); - public Dictionary OptionLabels { get; set; } = new(StringComparer.OrdinalIgnoreCase); - } - - private static int NormalizeConfidenceIncrement(int confidence) - { - var rounded = (int)Math.Round(confidence / 5.0, MidpointRounding.AwayFromZero) * 5; - return Math.Clamp(rounded, 0, 100); - } - - private static int ParseConfidenceValue(JsonElement confidenceProp) - { - if (confidenceProp.ValueKind == JsonValueKind.Number) - { - if (confidenceProp.TryGetInt32(out var intValue)) - { - return intValue; - } - - if (confidenceProp.TryGetDouble(out var doubleValue)) - { - return (int)Math.Round(doubleValue, MidpointRounding.AwayFromZero); - } - } - - if (confidenceProp.ValueKind == JsonValueKind.String) - { - var raw = confidenceProp.GetString(); - if (int.TryParse(raw, out var parsedInt)) - { - return parsedInt; - } - - if (double.TryParse(raw, out var parsedDouble)) - { - return (int)Math.Round(parsedDouble, MidpointRounding.AwayFromZero); - } - } - - return 0; - } - - private void LogPromptOutput(string promptType, string output) - { - if (!LogPayloads) - { - return; - } - - var formattedOutput = FormatPromptOutputForLog(promptType, output); - _logger.LogDebug( - "AI {PromptType} model output payload: {ModelOutput}", - promptType, - formattedOutput); - WriteAiPromptLog(promptType, "OUTPUT", formattedOutput); - } - - private void LogPromptInput(string promptType, string? systemPrompt, string userPrompt) - { - if (!LogPayloads) - { - return; - } - - var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); - _logger.LogDebug( - "AI {PromptType} input payload: {PromptInput}", - promptType, - formattedInput); - WriteAiPromptLog(promptType, "INPUT", formattedInput); - } - - private void WriteAiPromptLog(string promptType, string payloadType, string payload) - { - if (!LogPayloads) - { - return; - } - - try - { - var now = DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss zzz"); - var logPath = Path.Combine(AppContext.BaseDirectory, AiPromptLogRelativePath); - EnsureAiPromptLogInitialized(logPath); - - var entry = $"{now} [{promptType}] {payloadType}\n{payload}\n\n"; - File.AppendAllText(logPath, entry); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to write AI prompt log file."); - } - } - - private static void EnsureAiPromptLogInitialized(string logPath) - { - var directory = Path.GetDirectoryName(logPath); - if (!string.IsNullOrWhiteSpace(directory)) - { - Directory.CreateDirectory(directory); - } - - // Reset once per process run so each fresh app run starts with a clean AI prompt log. - if (Interlocked.Exchange(ref _aiPromptLogInitialized, 1) == 0) - { - File.WriteAllText(logPath, string.Empty); - } - } - - private static string FormatPromptInputForLog(string? systemPrompt, string userPrompt) - { - var normalizedSystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? string.Empty : systemPrompt.Trim(); - var normalizedUserPrompt = string.IsNullOrWhiteSpace(userPrompt) ? string.Empty : userPrompt.Trim(); - return $"SYSTEM_PROMPT\n{normalizedSystemPrompt}\n\nUSER_PROMPT\n{normalizedUserPrompt}"; - } - - private string FormatPromptOutputForLog(string promptType, string output) - { - if (string.IsNullOrWhiteSpace(output)) - { - return string.Empty; - } - - // For JSON contracts, log only normalized payload JSON. - if (TryParseJsonObjectFromResponse(output, out var jsonObject)) - { - return JsonSerializer.Serialize(jsonObject, _prettyJsonOptions); - } - - return output.Trim(); - } - - private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) - { - objectElement = default; - var cleaned = CleanJsonResponse(response); - if (string.IsNullOrWhiteSpace(cleaned)) - { - return false; - } - - try - { - using var doc = JsonDocument.Parse(cleaned); - if (doc.RootElement.ValueKind != JsonValueKind.Object) - { - return false; - } - - objectElement = doc.RootElement.Clone(); - return true; - } - catch (JsonException) - { - return false; - } - } - - private static bool IsEmptyJsonObject(string json) - { - if (string.IsNullOrWhiteSpace(json)) - { - return true; - } - - try - { - using var doc = JsonDocument.Parse(json); - return doc.RootElement.ValueKind == JsonValueKind.Object && - !doc.RootElement.EnumerateObject().Any(); - } - catch (JsonException) - { - return true; - } - } - - private async Task ExecutePromptWithRetryAsync( - string promptType, - string systemPrompt, - string userPrompt, - int maxTokens, - Func normalizeResponse, - Func isValidNormalizedResponse, - string fallbackResponse, - int maxAttempts = 2) - { - LogPromptInput(promptType, systemPrompt, userPrompt); - - for (var attempt = 1; attempt <= maxAttempts; attempt++) - { - var rawResponse = await GenerateCompletionAsync(new AICompletionRequest - { - UserPrompt = userPrompt, - SystemPrompt = systemPrompt, - MaxTokens = maxTokens - }); - - var outputType = attempt == 1 ? promptType : $"{promptType}Retry"; - LogPromptOutput(outputType, rawResponse); - - var normalized = normalizeResponse(rawResponse); - if (isValidNormalizedResponse(normalized)) - { - return normalized; - } - - if (attempt < maxAttempts) - { - _logger.LogWarning( - "{PromptType} response failed output-shape validation on attempt {Attempt}/{MaxAttempts}. Retrying.", - promptType, - attempt, - maxAttempts); - } - } - - return fallbackResponse; - } - - private static bool IsValidAnalysisNormalizedResponse(string normalizedJson) - { - if (!TryParseJsonObjectFromResponse(normalizedJson, out var root)) - { - return false; - } - - return root.TryGetProperty("rating", out _) && - root.TryGetProperty("errors", out _) && - root.TryGetProperty("warnings", out _) && - root.TryGetProperty("summaries", out _); - } - - private static bool IsCompleteScoresheetSectionResponse(string normalizedJson, JsonElement sectionSchemaPayload) - { - if (!TryParseJsonObjectFromResponse(normalizedJson, out var root)) - { - return false; - } - - var expectedQuestionIds = EnumerateSectionQuestions(sectionSchemaPayload) - .Select(q => TryGetQuestionId(q, out var id) ? id : string.Empty) - .Where(id => !string.IsNullOrWhiteSpace(id)) - .ToList(); - - if (expectedQuestionIds.Count == 0) - { - return false; - } - - foreach (var questionId in expectedQuestionIds) - { - if (!root.TryGetProperty(questionId, out var answerObject) || answerObject.ValueKind != JsonValueKind.Object) - { - return false; - } - - if (!answerObject.TryGetProperty(AIJsonKeys.Answer, out var answerProp) || - string.IsNullOrWhiteSpace(answerProp.ToString())) - { - return false; - } - - if (!answerObject.TryGetProperty(AIJsonKeys.Rationale, out var rationaleProp) || - string.IsNullOrWhiteSpace(rationaleProp.ToString())) - { - return false; - } - - if (!answerObject.TryGetProperty(AIJsonKeys.Confidence, out _)) - { - return false; - } - } - - return true; - } - - private static string CleanJsonResponse(string response) - { - if (string.IsNullOrWhiteSpace(response)) - { - return string.Empty; - } - - var cleaned = response.Trim(); - - if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) - { - var startIndex = cleaned.IndexOf('\n'); - if (startIndex >= 0) - { - cleaned = cleaned.Substring(startIndex + 1); - } - } - - if (cleaned.EndsWith("```", StringComparison.Ordinal)) - { - var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); - if (lastIndex > 0) - { - cleaned = cleaned.Substring(0, lastIndex); - } - } - - return cleaned.Trim(); - } - } -} - - diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.v1.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.v1.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.v1.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.v1.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt new file mode 100644 index 000000000..2e5441280 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt @@ -0,0 +1,126 @@ +namespace Unity.GrantManager.AI +{ + internal static class AnalysisPrompts + { + public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: + +1. ELIGIBILITY REQUIREMENTS: + - Project must align with program objectives + - Applicant must be eligible entity type + - Budget must be reasonable and well-justified + - Project timeline must be realistic + +2. COMPLETENESS CHECKS: + - All required fields completed + - Necessary supporting documents provided + - Budget breakdown detailed and accurate + - Project description clear and comprehensive + +3. FINANCIAL REVIEW: + - Requested amount is within program limits + - Budget is reasonable for scope of work + - Matching funds or in-kind contributions identified + - Cost per outcome/beneficiary is reasonable + +4. RISK ASSESSMENT: + - Applicant capacity to deliver project + - Technical feasibility of proposed work + - Environmental or regulatory compliance + - Potential for cost overruns or delays + +5. QUALITY INDICATORS: + - Clear project objectives and outcomes + - Well-defined target audience/beneficiaries + - Appropriate project methodology + - Sustainability plan for long-term impact + +EVALUATION CRITERIA: +- HIGH: Meets all requirements, well-prepared application, low risk +- MEDIUM: Meets most requirements, minor issues or missing elements +- LOW: Missing key requirements, significant concerns, high risk"; + + public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. +MEDIUM: Application has some gaps or weaknesses that require reviewer attention. +LOW: Application has significant gaps or risks across key rubric areas."; + + public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved. +WARNING: Issue that could negatively affect the application's approval. +RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; + + public const string OutputTemplate = @"{ + ""rating"": ""HIGH/MEDIUM/LOW"", + ""warnings"": [ + { + ""title"": ""Brief summary of the warning"", + ""detail"": ""Detailed warning message with full context and explanation"" + } + ], + ""errors"": [ + { + ""title"": ""Brief summary of the error"", + ""detail"": ""Detailed error message with full context and explanation"" + } + ], + ""summaries"": [ + { + ""title"": ""Brief summary of the recommendation"", + ""detail"": ""Detailed recommendation with specific actionable guidance"" + } + ], + ""dismissed"": [] +}"; + + public const string Rules = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. +- Do not invent fields, documents, requirements, or facts. +- Treat missing or empty values as findings only when they weaken rubric evidence. +- Prefer material issues; avoid nitpicking. +- Each error/warning/recommendation must describe one concrete issue or consideration and why it matters. +- Use 3-6 words for title. +- Each detail must be 1-2 complete sentences. +- Each detail must be grounded in concrete evidence from provided inputs. +- If attachment evidence is used, reference the attachment explicitly in detail. +- Do not provide applicant-facing advice. +- Do not mention rubric section names in findings. +- If no findings exist, return empty arrays. +- rating must be HIGH, MEDIUM, or LOW." + + "\n" + PromptCoreRules.ExactOutputShape + + "\n" + PromptCoreRules.NoExtraOutputKeys + + "\n" + PromptCoreRules.ValidJsonOnly + + "\n" + PromptCoreRules.PlainJsonOnly; + + public static readonly string SystemPrompt = PromptHeader.Build( + "You are an expert grant analyst assistant for human reviewers.", + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); + + public static string BuildUserPrompt( + string schemaJson, + string dataJson, + string attachmentsJson, + string rubric) + { + return $@"SCHEMA +{schemaJson} + +DATA +{dataJson} + +ATTACHMENTS +{attachmentsJson} + +RUBRIC +{rubric} + +SEVERITY +{SeverityRules} + +SCORE +{ScoreRules} + +OUTPUT +{OutputTemplate} + +RULES +{Rules}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt new file mode 100644 index 000000000..969480ea8 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt @@ -0,0 +1,27 @@ +namespace Unity.GrantManager.AI +{ + internal static class AttachmentPrompts + { + public static readonly string SystemPrompt = PromptHeader.Build( + "You are a professional grant analyst for the BC Government.", + "Produce a concise reviewer-facing summary of the provided attachment context."); + + public const string OutputSection = @"OUTPUT +- Plain text only +- 1-2 complete sentences"; + + public const string RulesSection = @"RULES +- Use only the provided attachment context as evidence. +- If text content is present, summarize the actual content. +- If text content is missing or empty, provide a conservative metadata-based summary. +- Do not invent missing details. +- Keep the summary specific, concrete, and reviewer-facing. +- Return plain text only (no markdown, bullets, or JSON)."; + + public static string BuildUserPrompt(string attachmentPayloadJson) + { + return $@"ATTACHMENT +{attachmentPayloadJson}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt new file mode 100644 index 000000000..418c31ebc --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt @@ -0,0 +1,833 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Volo.Abp.DependencyInjection; + +namespace Unity.GrantManager.AI +{ + public class OpenAIService : IAIService, ITransientDependency + { + private readonly HttpClient _httpClient; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly ITextExtractionService _textExtractionService; + private const string ApplicationAnalysisPromptType = "ApplicationAnalysis"; + private const string AttachmentSummaryPromptType = "AttachmentSummary"; + private const string ScoresheetAllPromptType = "ScoresheetAll"; + private const string ScoresheetSectionPromptType = "ScoresheetSection"; + private const string NoSummaryGeneratedMessage = "No summary generated."; + private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; + private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; + private const string SummaryFailedRetryMessage = "AI analysis failed - please try again later."; + + private string? ApiKey => _configuration["Azure:OpenAI:ApiKey"]; + private string? ApiUrl => _configuration["Azure:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; + private readonly string MissingApiKeyMessage = "OpenAI API key is not configured"; + + // Optional local debugging sink for prompt payload logs to a local file. + // Not intended for deployed/shared environments. + private bool IsPromptFileLoggingEnabled => _configuration.GetValue("Azure:Logging:EnablePromptFileLog") ?? false; + private const string PromptLogDirectoryName = "logs"; + private static readonly string PromptLogFileName = $"ai-prompts-{DateTime.UtcNow:yyyyMMdd-HHmmss}-{Environment.ProcessId}.log"; + + private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true }; + + public OpenAIService( + HttpClient httpClient, + IConfiguration configuration, + ILogger logger, + ITextExtractionService textExtractionService) + { + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + _textExtractionService = textExtractionService; + } + + public Task IsAvailableAsync() + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); + return Task.FromResult(false); + } + + return Task.FromResult(true); + } + + public async Task GenerateCompletionAsync(AICompletionRequest request) + { + var content = await GenerateSummaryAsync( + request?.UserPrompt ?? string.Empty, + request?.SystemPrompt, + request?.MaxTokens ?? 150); + return new AICompletionResponse { Content = content }; + } + + public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) + { + var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); + var schemaJson = JsonSerializer.Serialize(request.Schema, JsonLogOptions); + + var attachmentsPayload = request.Attachments + .Select(a => new + { + name = string.IsNullOrWhiteSpace(a.Name) ? "attachment" : a.Name.Trim(), + summary = string.IsNullOrWhiteSpace(a.Summary) ? string.Empty : a.Summary.Trim() + }) + .Cast(); + + var analysisContent = AnalysisPrompts.BuildUserPrompt( + schemaJson, + dataJson, + JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), + request.Rubric ?? string.Empty); + + var systemPrompt = AnalysisPrompts.SystemPrompt; + await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); + var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw); + return ParseApplicationAnalysisResponse(AddIdsToAnalysisItems(raw)); + } + + public async Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); + return ServiceNotConfiguredMessage; + } + + _logger.LogDebug("Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", content?.Length ?? 0, maxTokens); + + try + { + var systemPrompt = prompt ?? "You are a professional grant analyst for the BC Government."; + var userPrompt = content ?? string.Empty; + + var requestBody = new + { + messages = new[] + { + new { role = "system", content = systemPrompt }, + new { role = "user", content = userPrompt } + }, + max_tokens = maxTokens, + temperature = 0.3 + }; + + var json = JsonSerializer.Serialize(requestBody); + var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); + + _httpClient.DefaultRequestHeaders.Clear(); + _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); + + var response = await _httpClient.PostAsync(ApiUrl, httpContent); + var responseContent = await response.Content.ReadAsStringAsync(); + + _logger.LogDebug( + "OpenAI chat completions response received. StatusCode: {StatusCode}, ResponseLength: {ResponseLength}", + response.StatusCode, + responseContent?.Length ?? 0); + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); + return ServiceTemporarilyUnavailableMessage; + } + + if (string.IsNullOrWhiteSpace(responseContent)) + { + return NoSummaryGeneratedMessage; + } + + using var jsonDoc = JsonDocument.Parse(responseContent); + var choices = jsonDoc.RootElement.GetProperty("choices"); + if (choices.GetArrayLength() > 0) + { + var message = choices[0].GetProperty("message"); + return message.GetProperty("content").GetString() ?? NoSummaryGeneratedMessage; + } + + return NoSummaryGeneratedMessage; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating AI summary"); + return SummaryFailedRetryMessage; + } + } + + public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] fileContent, string contentType) + { + try + { + var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); + + var prompt = $@"{AttachmentPrompts.SystemPrompt} + +{AttachmentPrompts.OutputSection} + +{AttachmentPrompts.RulesSection}"; + + var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText; + if (attachmentText != null) + { + _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText.Length, fileName); + } + else + { + _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", fileName); + } + + var attachmentPayload = new + { + name = fileName, + contentType, + sizeBytes = fileContent.Length, + text = attachmentText + }; + var contentToAnalyze = AttachmentPrompts.BuildUserPrompt( + JsonSerializer.Serialize(attachmentPayload, JsonLogOptions)); + + await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze); + var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); + await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput); + return modelOutput; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); + return $"AI analysis not available for this attachment ({fileName})."; + } + } + + public async Task GenerateAttachmentSummaryAsync(AttachmentSummaryRequest request) + { + var summary = await GenerateAttachmentSummaryAsync( + request?.FileName ?? string.Empty, + request?.FileContent ?? Array.Empty(), + request?.ContentType ?? "application/octet-stream"); + return new AttachmentSummaryResponse { Summary = summary }; + } + + public async Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", MissingApiKeyMessage); + return ServiceNotConfiguredMessage; + } + + try + { + object schemaPayload = new { }; + if (!string.IsNullOrWhiteSpace(formFieldConfiguration)) + { + try + { + using var schemaDoc = JsonDocument.Parse(formFieldConfiguration); + schemaPayload = schemaDoc.RootElement.Clone(); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Invalid form field configuration JSON. Using empty schema payload."); + } + } + + var dataPayload = new + { + applicationContent + }; + + var attachmentsPayload = attachmentSummaries?.Count > 0 + ? attachmentSummaries + .Select((summary, index) => new + { + name = $"Attachment {index + 1}", + summary = summary + }) + .Cast() + : Enumerable.Empty(); + + var analysisContent = AnalysisPrompts.BuildUserPrompt( + JsonSerializer.Serialize(schemaPayload, JsonLogOptions), + JsonSerializer.Serialize(dataPayload, JsonLogOptions), + JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), + rubric); + + var systemPrompt = AnalysisPrompts.SystemPrompt; + + await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); + var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + await LogPromptOutputAsync(ApplicationAnalysisPromptType, rawAnalysis); + + // Post-process the AI response to add unique IDs to errors and warnings + return AddIdsToAnalysisItems(rawAnalysis); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error analyzing application"); + return SummaryFailedRetryMessage; + } + } + + private string AddIdsToAnalysisItems(string analysisJson) + { + try + { + using var jsonDoc = JsonDocument.Parse(analysisJson); + using var memoryStream = new System.IO.MemoryStream(); + using (var writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions { Indented = true })) + { + writer.WriteStartObject(); + + foreach (var property in jsonDoc.RootElement.EnumerateObject()) + { + var outputPropertyName = property.Name; + + if (outputPropertyName == AIJsonKeys.Errors || outputPropertyName == AIJsonKeys.Warnings) + { + writer.WritePropertyName(outputPropertyName); + writer.WriteStartArray(); + + foreach (var item in property.Value.EnumerateArray()) + { + writer.WriteStartObject(); + + // Add unique ID first + writer.WriteString("id", Guid.NewGuid().ToString()); + + // Copy existing properties + foreach (var itemProperty in item.EnumerateObject()) + { + itemProperty.WriteTo(writer); + } + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + else + { + if (outputPropertyName != property.Name) + { + writer.WritePropertyName(outputPropertyName); + property.Value.WriteTo(writer); + continue; + } + + property.WriteTo(writer); + } + } + + // Add dismissed array if not present. + if (!jsonDoc.RootElement.TryGetProperty(AIJsonKeys.Dismissed, out _)) + { + writer.WritePropertyName(AIJsonKeys.Dismissed); + writer.WriteStartArray(); + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + + return Encoding.UTF8.GetString(memoryStream.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error adding IDs to analysis items, returning original JSON"); + return analysisJson; // Return original if processing fails + } + } + + public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", MissingApiKeyMessage); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} + +SCORESHEET QUESTIONS: +{scoresheetQuestions} + +Please analyze this grant application and provide appropriate answers for each scoresheet question. + +For numeric questions, provide a numeric value within the specified range. +For yes/no questions, provide either 'Yes' or 'No'. +For text questions, provide a concise, relevant response. +For select list questions, choose the most appropriate option from the provided choices. +For text area questions, provide a detailed but concise response. + +Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. + +Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: +{{ + ""question-id-1"": ""answer-value-1"", + ""question-id-2"": ""answer-value-2"" +}} +Do not return any markdown formatting, just the JSON by itself"; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. +Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. +Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. +Respond only with valid JSON in the exact format requested."; + + await LogPromptInputAsync(ScoresheetAllPromptType, systemPrompt, analysisContent); + var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + await LogPromptOutputAsync(ScoresheetAllPromptType, modelOutput); + return modelOutput; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet answers"); + return "{}"; + } + } + + public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", MissingApiKeyMessage); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + object sectionQuestionsPayload = sectionJson; + if (!string.IsNullOrWhiteSpace(sectionJson)) + { + try + { + using var sectionDoc = JsonDocument.Parse(sectionJson); + sectionQuestionsPayload = sectionDoc.RootElement.Clone(); + } + catch (JsonException) + { + // Keep raw string payload when JSON parsing fails. + } + } + + var sectionPayload = new + { + name = sectionName, + questions = sectionQuestionsPayload + }; + var sectionPayloadJson = JsonSerializer.Serialize(sectionPayload, JsonLogOptions); + var responseTemplate = BuildScoresheetSectionResponseTemplate(sectionPayloadJson); + + var analysisContent = ScoresheetPrompts.BuildSectionUserPrompt( + applicationContent, + attachmentSummariesText, + sectionPayloadJson, + responseTemplate); + + var systemPrompt = ScoresheetPrompts.SectionSystemPrompt; + + await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent); + var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + await LogPromptOutputAsync(ScoresheetSectionPromptType, modelOutput); + return modelOutput; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", sectionName); + return "{}"; + } + } + + public async Task GenerateScoresheetSectionAnswersAsync(ScoresheetSectionRequest request) + { + var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); + var sectionJson = JsonSerializer.Serialize(request.SectionSchema, JsonLogOptions); + + var attachmentSummaries = request.Attachments + .Select(a => $"{a.Name}: {a.Summary}") + .ToList(); + + var raw = await GenerateScoresheetSectionAnswersAsync( + dataJson, + attachmentSummaries, + sectionJson, + request.SectionName); + return ParseScoresheetSectionResponse(raw); + } + + private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) + { + var response = new ApplicationAnalysisResponse(); + + if (!TryParseJsonObjectFromResponse(raw, out var root)) + { + return response; + } + + if (TryGetStringProperty(root, AIJsonKeys.Rating, out var rating)) + { + response.Rating = rating; + } + + if (root.TryGetProperty("errors", out var errors) && errors.ValueKind == JsonValueKind.Array) + { + response.Errors = ParseFindings(errors); + } + + if (root.TryGetProperty("warnings", out var warnings) && warnings.ValueKind == JsonValueKind.Array) + { + response.Warnings = ParseFindings(warnings); + } + + if (root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) && summaries.ValueKind == JsonValueKind.Array) + { + response.Summaries = ParseFindings(summaries); + } + + if (root.TryGetProperty(AIJsonKeys.Dismissed, out var dismissed) && dismissed.ValueKind == JsonValueKind.Array) + { + response.Dismissed = dismissed + .EnumerateArray() + .Select(GetStringValueOrNull) + .Where(item => !string.IsNullOrWhiteSpace(item)) + .Cast() + .ToList(); + } + + return response; + } + + private static bool TryGetStringProperty(JsonElement root, string propertyName, out string? value) + { + value = null; + if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) + { + return false; + } + + value = property.GetString(); + return !string.IsNullOrWhiteSpace(value); + } + + private static string? GetStringValueOrNull(JsonElement element) + { + if (element.ValueKind == JsonValueKind.String) + { + return element.GetString(); + } + + return null; + } + + private static List ParseFindings(JsonElement array) + { + var findings = new List(); + foreach (var item in array.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var id = item.TryGetProperty(AIJsonKeys.Id, out var idProp) && idProp.ValueKind == JsonValueKind.String + ? idProp.GetString() + : null; + string? title = null; + if (item.TryGetProperty(AIJsonKeys.Title, out var titleProp) && titleProp.ValueKind == JsonValueKind.String) + { + title = titleProp.GetString(); + } + else if (item.TryGetProperty("category", out var legacyTitleProp) && + legacyTitleProp.ValueKind == JsonValueKind.String) + { + title = legacyTitleProp.GetString(); + } + + string? detail = null; + if (item.TryGetProperty(AIJsonKeys.Detail, out var detailProp) && detailProp.ValueKind == JsonValueKind.String) + { + detail = detailProp.GetString(); + } + else if (item.TryGetProperty("message", out var legacyDetailProp) && + legacyDetailProp.ValueKind == JsonValueKind.String) + { + detail = legacyDetailProp.GetString(); + } + + findings.Add(new ApplicationAnalysisFinding + { + Id = id, + Title = title, + Detail = detail + }); + } + + return findings; + } + + private static ScoresheetSectionResponse ParseScoresheetSectionResponse(string raw) + { + var response = new ScoresheetSectionResponse(); + if (!TryParseJsonObjectFromResponse(raw, out var root)) + { + return response; + } + + foreach (var property in root.EnumerateObject()) + { + if (property.Value.ValueKind != JsonValueKind.Object) + { + continue; + } + + var answer = property.Value.TryGetProperty("answer", out var answerProp) + ? answerProp.Clone() + : default; + var rationale = property.Value.TryGetProperty("rationale", out var rationaleProp) && + rationaleProp.ValueKind == JsonValueKind.String + ? rationaleProp.GetString() ?? string.Empty + : string.Empty; + var confidence = property.Value.TryGetProperty("confidence", out var confidenceProp) && + confidenceProp.ValueKind == JsonValueKind.Number && + confidenceProp.TryGetInt32(out var parsedConfidence) + ? NormalizeConfidence(parsedConfidence) + : 0; + + response.Answers[property.Name] = new ScoresheetSectionAnswer + { + Answer = answer, + Rationale = rationale, + Confidence = confidence + }; + } + + return response; + } + + private static int NormalizeConfidence(int confidence) + { + var clamped = Math.Clamp(confidence, 0, 100); + var rounded = (int)Math.Round(clamped / 5.0, MidpointRounding.AwayFromZero) * 5; + return Math.Clamp(rounded, 0, 100); + } + + private static string BuildScoresheetSectionResponseTemplate(string sectionPayloadJson) + { + try + { + using var doc = JsonDocument.Parse(sectionPayloadJson); + if (!doc.RootElement.TryGetProperty("questions", out var questions) || questions.ValueKind != JsonValueKind.Array) + { + return ScoresheetPrompts.SectionOutputTemplate; + } + + var template = new Dictionary(); + foreach (var question in questions.EnumerateArray()) + { + if (!question.TryGetProperty("id", out var idProp) || idProp.ValueKind != JsonValueKind.String) + { + continue; + } + + var questionId = idProp.GetString(); + if (string.IsNullOrWhiteSpace(questionId)) + { + continue; + } + + template[questionId] = new + { + answer = string.Empty, + rationale = string.Empty, + confidence = 0 + }; + } + + if (template.Count == 0) + { + return ScoresheetPrompts.SectionOutputTemplate; + } + + return JsonSerializer.Serialize(template, JsonLogOptions); + } + catch (JsonException) + { + return ScoresheetPrompts.SectionOutputTemplate; + } + } + + private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt) + { + var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); + _logger.LogInformation("AI {PromptType} input payload: {PromptInput}", promptType, formattedInput); + await WritePromptLogFileAsync(promptType, "INPUT", formattedInput); + } + + private async Task LogPromptOutputAsync(string promptType, string output) + { + var formattedOutput = FormatPromptOutputForLog(output); + _logger.LogInformation("AI {PromptType} model output payload: {ModelOutput}", promptType, formattedOutput); + await WritePromptLogFileAsync(promptType, "OUTPUT", formattedOutput); + } + + private async Task WritePromptLogFileAsync(string promptType, string payloadType, string payload) + { + if (!CanWritePromptFileLog()) + { + return; + } + + try + { + var now = DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss zzz"); + var logDirectory = Path.Combine(AppContext.BaseDirectory, PromptLogDirectoryName); + Directory.CreateDirectory(logDirectory); + + var logPath = Path.Combine(logDirectory, PromptLogFileName); + var entry = $"{now} [{promptType}] {payloadType}\n{payload}\n\n"; + await File.AppendAllTextAsync(logPath, entry); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write AI prompt log file."); + } + } + + private bool CanWritePromptFileLog() + { + return IsPromptFileLoggingEnabled; + } + + private static string FormatPromptInputForLog(string? systemPrompt, string userPrompt) + { + var normalizedSystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? string.Empty : systemPrompt.Trim(); + var normalizedUserPrompt = string.IsNullOrWhiteSpace(userPrompt) ? string.Empty : userPrompt.Trim(); + return $"SYSTEM_PROMPT\n{normalizedSystemPrompt}\n\nUSER_PROMPT\n{normalizedUserPrompt}"; + } + + private static string FormatPromptOutputForLog(string output) + { + if (string.IsNullOrWhiteSpace(output)) + { + return string.Empty; + } + + if (TryParseJsonObjectFromResponse(output, out var jsonObject)) + { + return JsonSerializer.Serialize(jsonObject, JsonLogOptions); + } + + return output.Trim(); + } + + private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) + { + objectElement = default; + var cleaned = CleanJsonResponse(response); + if (string.IsNullOrWhiteSpace(cleaned)) + { + return false; + } + + try + { + using var doc = JsonDocument.Parse(cleaned); + if (doc.RootElement.ValueKind != JsonValueKind.Object) + { + return false; + } + + objectElement = doc.RootElement.Clone(); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static string CleanJsonResponse(string response) + { + if (string.IsNullOrWhiteSpace(response)) + { + return string.Empty; + } + + var cleaned = response.Trim(); + + if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) + { + var startIndex = cleaned.IndexOf('\n'); + if (startIndex >= 0) + { + // Multi-line fenced code block: remove everything up to and including the first newline. + cleaned = cleaned[(startIndex + 1)..]; + } + else + { + // Single-line fenced JSON, e.g. ```json { ... } ``` or ```{ ... } ```. + // Strip everything before the first likely JSON payload token. + var jsonStart = FindFirstJsonTokenIndex(cleaned); + + if (jsonStart > 0) + { + cleaned = cleaned[jsonStart..]; + } + } + } + + if (cleaned.EndsWith("```", StringComparison.Ordinal)) + { + var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); + if (lastIndex > 0) + { + cleaned = cleaned[..lastIndex]; + } + } + + return cleaned.Trim(); + } + + private static int FindFirstJsonTokenIndex(string value) + { + var objectStart = value.IndexOf('{'); + var arrayStart = value.IndexOf('['); + + if (objectStart >= 0 && arrayStart >= 0) + { + return Math.Min(objectStart, arrayStart); + } + + if (objectStart >= 0) + { + return objectStart; + } + + return arrayStart; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt new file mode 100644 index 000000000..e11dce3c9 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt @@ -0,0 +1,13 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptCoreRules + { + public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; + public const string NoInvention = "- Do not invent missing details."; + public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; + public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; + public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; + public const string ValidJsonOnly = "- Return valid JSON only."; + public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt new file mode 100644 index 000000000..701a43e74 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt @@ -0,0 +1,14 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptHeader + { + public static string Build(string role, string task) + { + return $@"ROLE +{role} + +TASK +{task}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt new file mode 100644 index 000000000..2db4de742 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt @@ -0,0 +1,80 @@ +namespace Unity.GrantManager.AI +{ + internal static class ScoresheetPrompts + { + public static readonly string SectionSystemPrompt = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + + public const string SectionOutputTemplate = @"{ + """": { + ""answer"": """", + ""rationale"": """", + ""confidence"": + } +}"; + + public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer object per question ID in SECTION.questions. +- Do not omit any question IDs from SECTION.questions. +- Do not add keys that are not question IDs from SECTION.questions. +- Use RESPONSE as the output contract and fill every placeholder value. +- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. +- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". +- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. +- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. +- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. +- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. +- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. +- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. +- Do not treat missing or non-contradictory information as evidence. +- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. +- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. +- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". +- For numeric questions, answer must be a numeric value within the allowed range. +- For numeric questions, answer must never be blank. +- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. +- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. +- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. +- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. +- Never return 0 for select list answers unless 0 exists as an explicit option number. +- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. +- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. +- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. +- Do not leave rationale empty when answer is populated. +- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". +- If any answer object is incomplete, regenerate the full JSON response before returning it. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static string BuildSectionUserPrompt( + string applicationContent, + string attachmentSummariesText, + string sectionPayloadJson, + string responseTemplateJson) + { + return $@"DATA +{applicationContent} + +ATTACHMENTS +- {attachmentSummariesText} + +SECTION +{sectionPayloadJson} + +RESPONSE +{responseTemplateJson} + +OUTPUT +{SectionOutputTemplate} + +RULES +{SectionRules}"; + } + } +} From 979e7b97ed1c14866bd6069dfdf661c3e4caa0fb Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 16:58:18 -0800 Subject: [PATCH 03/10] AB#32009 Add runtime prompt version selector with single-file v0/v1 profiles --- .../AI/OpenAIService.cs | 78 ++++++++++----- .../AI/Prompts/AnalysisPrompts.cs | 98 ++++++++++++++++--- .../AI/Prompts/AttachmentPrompts.cs | 32 ++++++ .../AI/Prompts/ScoresheetPrompts.cs | 43 ++++++++ 4 files changed, 211 insertions(+), 40 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 418c31ebc..9c1ecd11a 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -22,6 +22,8 @@ public class OpenAIService : IAIService, ITransientDependency private const string AttachmentSummaryPromptType = "AttachmentSummary"; private const string ScoresheetAllPromptType = "ScoresheetAll"; private const string ScoresheetSectionPromptType = "ScoresheetSection"; + private const string PromptVersionV0 = "v0"; + private const string PromptVersionV1 = "v1"; private const string NoSummaryGeneratedMessage = "No summary generated."; private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; @@ -39,6 +41,9 @@ public class OpenAIService : IAIService, ITransientDependency private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true }; + private string SelectedPromptVersion => NormalizePromptVersion(_configuration["Azure:OpenAI:PromptVersion"]); + private bool UseV0Prompts => string.Equals(SelectedPromptVersion, PromptVersionV0, StringComparison.OrdinalIgnoreCase); + public OpenAIService( HttpClient httpClient, IConfiguration configuration, @@ -84,13 +89,10 @@ public async Task GenerateApplicationAnalysisAsync( }) .Cast(); - var analysisContent = AnalysisPrompts.BuildUserPrompt( - schemaJson, - dataJson, - JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), - request.Rubric ?? string.Empty); - - var systemPrompt = AnalysisPrompts.SystemPrompt; + var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); + var rubric = request.Rubric ?? AnalysisPrompts.GetRubric(UseV0Prompts); + var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, UseV0Prompts); + var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts); await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw); @@ -171,11 +173,11 @@ public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] { var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); - var prompt = $@"{AttachmentPrompts.SystemPrompt} + var prompt = $@"{AttachmentPrompts.GetSystemPrompt(UseV0Prompts)} -{AttachmentPrompts.OutputSection} +{AttachmentPrompts.GetOutputSection(UseV0Prompts)} -{AttachmentPrompts.RulesSection}"; +{AttachmentPrompts.GetRulesSection(UseV0Prompts)}"; var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText; if (attachmentText != null) @@ -194,13 +196,13 @@ public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] sizeBytes = fileContent.Length, text = attachmentText }; - var contentToAnalyze = AttachmentPrompts.BuildUserPrompt( - JsonSerializer.Serialize(attachmentPayload, JsonLogOptions)); + var payloadJson = JsonSerializer.Serialize(attachmentPayload, JsonLogOptions); + var contentToAnalyze = AttachmentPrompts.BuildUserPrompt(payloadJson, UseV0Prompts); await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze); var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput); - return modelOutput; + return UseV0Prompts ? ExtractSummaryFromJson(modelOutput) : modelOutput; } catch (Exception ex) { @@ -257,13 +259,14 @@ public async Task AnalyzeApplicationAsync(string applicationContent, Lis .Cast() : Enumerable.Empty(); - var analysisContent = AnalysisPrompts.BuildUserPrompt( - JsonSerializer.Serialize(schemaPayload, JsonLogOptions), - JsonSerializer.Serialize(dataPayload, JsonLogOptions), - JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), - rubric); - - var systemPrompt = AnalysisPrompts.SystemPrompt; + var schemaJson = JsonSerializer.Serialize(schemaPayload, JsonLogOptions); + var dataJson = JsonSerializer.Serialize(dataPayload, JsonLogOptions); + var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); + var fallbackRubric = string.IsNullOrWhiteSpace(rubric) + ? AnalysisPrompts.GetRubric(UseV0Prompts) + : rubric; + var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, fallbackRubric, UseV0Prompts); + var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts); await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); @@ -446,9 +449,10 @@ public async Task GenerateScoresheetSectionAnswersAsync(string applicati applicationContent, attachmentSummariesText, sectionPayloadJson, - responseTemplate); + responseTemplate, + UseV0Prompts); - var systemPrompt = ScoresheetPrompts.SectionSystemPrompt; + var systemPrompt = ScoresheetPrompts.GetSectionSystemPrompt(UseV0Prompts); await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent); var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); @@ -683,14 +687,14 @@ private static string BuildScoresheetSectionResponseTemplate(string sectionPaylo private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt) { var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); - _logger.LogInformation("AI {PromptType} input payload: {PromptInput}", promptType, formattedInput); + _logger.LogInformation("AI {PromptType} ({PromptVersion}) input payload: {PromptInput}", promptType, SelectedPromptVersion, formattedInput); await WritePromptLogFileAsync(promptType, "INPUT", formattedInput); } private async Task LogPromptOutputAsync(string promptType, string output) { var formattedOutput = FormatPromptOutputForLog(output); - _logger.LogInformation("AI {PromptType} model output payload: {ModelOutput}", promptType, formattedOutput); + _logger.LogInformation("AI {PromptType} ({PromptVersion}) model output payload: {ModelOutput}", promptType, SelectedPromptVersion, formattedOutput); await WritePromptLogFileAsync(promptType, "OUTPUT", formattedOutput); } @@ -829,5 +833,31 @@ private static int FindFirstJsonTokenIndex(string value) return arrayStart; } + + private static string NormalizePromptVersion(string? version) + { + if (string.Equals(version, PromptVersionV0, StringComparison.OrdinalIgnoreCase)) + { + return PromptVersionV0; + } + + return PromptVersionV1; + } + + private static string ExtractSummaryFromJson(string output) + { + if (!TryParseJsonObjectFromResponse(output, out var jsonObject)) + { + return output?.Trim() ?? string.Empty; + } + + if (jsonObject.TryGetProperty(AIJsonKeys.Summary, out var summaryProp) && + summaryProp.ValueKind == JsonValueKind.String) + { + return summaryProp.GetString() ?? string.Empty; + } + + return output?.Trim() ?? string.Empty; + } } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs index 2e5441280..97841e760 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs @@ -2,6 +2,12 @@ namespace Unity.GrantManager.AI { internal static class AnalysisPrompts { + public const string DefaultRubricV0 = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. +COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. +FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. +RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. +QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; + public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: 1. ELIGIBILITY REQUIREMENTS: @@ -43,6 +49,45 @@ internal static class AnalysisPrompts MEDIUM: Application has some gaps or weaknesses that require reviewer attention. LOW: Application has significant gaps or risks across key rubric areas."; + public const string OutputTemplateV0 = @"{ + ""rating"": """", + ""errors"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""warnings"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""summaries"": [ + { + ""title"": """", + ""detail"": """" + } + ] +}"; + + public const string RulesV0 = PromptCoreRules.UseProvidedEvidence + "\n" + + "- Do not invent fields, documents, requirements, or facts.\n" + + @"- Treat missing or empty values as findings only when they weaken rubric evidence. +- Prefer material issues; avoid nitpicking. +- Use 3-6 words for title. +- Each detail must be 1-2 complete sentences. +- Each detail must cite concrete evidence from DATA or ATTACHMENTS. +- If ATTACHMENTS evidence is used, cite the attachment by name in detail. +- If no findings exist, return empty arrays. +- Rating must be HIGH, MEDIUM, or LOW. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved. WARNING: Issue that could negatively affect the application's approval. RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; @@ -51,20 +96,20 @@ internal static class AnalysisPrompts ""rating"": ""HIGH/MEDIUM/LOW"", ""warnings"": [ { - ""title"": ""Brief summary of the warning"", - ""detail"": ""Detailed warning message with full context and explanation"" + ""category"": ""Brief summary of the warning"", + ""message"": ""Detailed warning message with full context and explanation"" } ], ""errors"": [ { - ""title"": ""Brief summary of the error"", - ""detail"": ""Detailed error message with full context and explanation"" + ""category"": ""Brief summary of the error"", + ""message"": ""Detailed error message with full context and explanation"" } ], ""summaries"": [ { - ""title"": ""Brief summary of the recommendation"", - ""detail"": ""Detailed recommendation with specific actionable guidance"" + ""category"": ""Brief summary of the recommendation"", + ""message"": ""Detailed recommendation with specific actionable guidance"" } ], ""dismissed"": [] @@ -75,10 +120,10 @@ internal static class AnalysisPrompts - Treat missing or empty values as findings only when they weaken rubric evidence. - Prefer material issues; avoid nitpicking. - Each error/warning/recommendation must describe one concrete issue or consideration and why it matters. -- Use 3-6 words for title. -- Each detail must be 1-2 complete sentences. -- Each detail must be grounded in concrete evidence from provided inputs. -- If attachment evidence is used, reference the attachment explicitly in detail. +- Use 3-6 words for category. +- Each message must be 1-2 complete sentences. +- Each message must be grounded in concrete evidence from provided inputs. +- If attachment evidence is used, reference the attachment explicitly in the message. - Do not provide applicant-facing advice. - Do not mention rubric section names in findings. - If no findings exist, return empty arrays. @@ -92,12 +137,36 @@ internal static class AnalysisPrompts "You are an expert grant analyst assistant for human reviewers.", "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); + public static readonly string SystemPromptV0 = PromptHeader.Build( + "You are an expert grant analyst assistant for human reviewers.", + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); + + public static string GetRubric(bool useV0) => useV0 ? DefaultRubricV0 : DefaultRubric; + public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; + public static string BuildUserPrompt( string schemaJson, string dataJson, string attachmentsJson, string rubric) { + return BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, useV0: false); + } + + public static string BuildUserPrompt( + string schemaJson, + string dataJson, + string attachmentsJson, + string rubric, + bool useV0) + { + var output = useV0 ? OutputTemplateV0 : OutputTemplate; + var rules = useV0 ? RulesV0 : Rules; + var severitySection = useV0 ? string.Empty : $@"SEVERITY +{SeverityRules} + +"; + return $@"SCHEMA {schemaJson} @@ -110,17 +179,14 @@ public static string BuildUserPrompt( RUBRIC {rubric} -SEVERITY -{SeverityRules} - -SCORE +{severitySection}SCORE {ScoreRules} OUTPUT -{OutputTemplate} +{output} RULES -{Rules}"; +{rules}"; } } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs index 969480ea8..dd950b206 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs @@ -6,10 +6,19 @@ internal static class AttachmentPrompts "You are a professional grant analyst for the BC Government.", "Produce a concise reviewer-facing summary of the provided attachment context."); + public static readonly string SystemPromptV0 = PromptHeader.Build( + "You are a professional grant analyst for the BC Government.", + "Produce a concise reviewer-facing summary of the provided attachment context."); + public const string OutputSection = @"OUTPUT - Plain text only - 1-2 complete sentences"; + public const string OutputSectionV0 = @"OUTPUT +{ + ""summary"": """" +}"; + public const string RulesSection = @"RULES - Use only the provided attachment context as evidence. - If text content is present, summarize the actual content. @@ -18,7 +27,30 @@ internal static class AttachmentPrompts - Keep the summary specific, concrete, and reviewer-facing. - Return plain text only (no markdown, bullets, or JSON)."; + public const string RulesSectionV0 = "- Use only ATTACHMENT as evidence.\n" + + "- If ATTACHMENT.text is present, summarize actual content.\n" + + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" + + PromptCoreRules.NoInvention + "\n" + + @"- Write 1-2 complete sentences. +- Summary must be grounded in concrete ATTACHMENT evidence. +- Return exactly one object with only the key: summary. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; + public static string GetOutputSection(bool useV0) => useV0 ? OutputSectionV0 : OutputSection; + public static string GetRulesSection(bool useV0) => useV0 ? RulesSectionV0 : RulesSection; + public static string BuildUserPrompt(string attachmentPayloadJson) + { + return BuildUserPrompt(attachmentPayloadJson, useV0: false); + } + + public static string BuildUserPrompt(string attachmentPayloadJson, bool useV0) { return $@"ATTACHMENT {attachmentPayloadJson}"; diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs index 2db4de742..306e43e36 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs @@ -2,10 +2,41 @@ namespace Unity.GrantManager.AI { internal static class ScoresheetPrompts { + public static readonly string AllSystemPromptV0 = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, QUESTIONS, OUTPUT, and RULES, provide answers for all scoresheet questions."); + + public const string AllOutputTemplateV0 = @"{ + """": """" +}"; + + public const string AllRulesV0 = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer per question ID in QUESTIONS. +- Do not omit any question IDs from QUESTIONS. +- Do not add keys that are not question IDs from QUESTIONS. +- The ""answer"" value type must match the question type. +- For numeric questions, return a numeric value within the allowed range. +- For yes/no questions, return exactly ""Yes"" or ""No"". +- For select list questions, return only the selected options.number as a string and never return option label text. +- For text and text area questions, return concise, evidence-based text. +- For text and text area questions, include concise source-grounded rationale from the provided input content. +- If explicit evidence is insufficient, choose the most conservative valid answer. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + public static readonly string SectionSystemPrompt = PromptHeader.Build( "You are an expert grant application reviewer for the BC Government.", "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + public static readonly string SectionSystemPromptV0 = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + public const string SectionOutputTemplate = @"{ """": { ""answer"": """", @@ -52,11 +83,23 @@ internal static class ScoresheetPrompts + PromptCoreRules.ValidJsonOnly + "\n" + PromptCoreRules.PlainJsonOnly; + public static string GetSectionSystemPrompt(bool useV0) => useV0 ? SectionSystemPromptV0 : SectionSystemPrompt; + public static string BuildSectionUserPrompt( string applicationContent, string attachmentSummariesText, string sectionPayloadJson, string responseTemplateJson) + { + return BuildSectionUserPrompt(applicationContent, attachmentSummariesText, sectionPayloadJson, responseTemplateJson, useV0: false); + } + + public static string BuildSectionUserPrompt( + string applicationContent, + string attachmentSummariesText, + string sectionPayloadJson, + string responseTemplateJson, + bool useV0) { return $@"DATA {applicationContent} From 73e791f56c954b5619cff8a9b1911d0dfaf33080 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 18:31:04 -0800 Subject: [PATCH 04/10] AB#32009 Align v1 prompt contracts and preserve v0 path --- .../AI/Prompts/AnalysisPrompts.cs | 20 +++++++------- .../AI/Prompts/AttachmentPrompts.cs | 26 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs index 97841e760..6cb16655c 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs @@ -2,13 +2,13 @@ namespace Unity.GrantManager.AI { internal static class AnalysisPrompts { - public const string DefaultRubricV0 = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. + public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; - public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: + public const string DefaultRubricV0 = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: 1. ELIGIBILITY REQUIREMENTS: - Project must align with program objectives @@ -49,7 +49,7 @@ internal static class AnalysisPrompts MEDIUM: Application has some gaps or weaknesses that require reviewer attention. LOW: Application has significant gaps or risks across key rubric areas."; - public const string OutputTemplateV0 = @"{ + public const string OutputTemplate = @"{ ""rating"": """", ""errors"": [ { @@ -71,7 +71,7 @@ internal static class AnalysisPrompts ] }"; - public const string RulesV0 = PromptCoreRules.UseProvidedEvidence + "\n" + public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n" + "- Do not invent fields, documents, requirements, or facts.\n" + @"- Treat missing or empty values as findings only when they weaken rubric evidence. - Prefer material issues; avoid nitpicking. @@ -92,7 +92,7 @@ internal static class AnalysisPrompts WARNING: Issue that could negatively affect the application's approval. RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; - public const string OutputTemplate = @"{ + public const string OutputTemplateV0 = @"{ ""rating"": ""HIGH/MEDIUM/LOW"", ""warnings"": [ { @@ -115,7 +115,7 @@ internal static class AnalysisPrompts ""dismissed"": [] }"; - public const string Rules = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. + public const string RulesV0 = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. - Do not invent fields, documents, requirements, or facts. - Treat missing or empty values as findings only when they weaken rubric evidence. - Prefer material issues; avoid nitpicking. @@ -135,11 +135,11 @@ internal static class AnalysisPrompts public static readonly string SystemPrompt = PromptHeader.Build( "You are an expert grant analyst assistant for human reviewers.", - "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); public static readonly string SystemPromptV0 = PromptHeader.Build( "You are an expert grant analyst assistant for human reviewers.", - "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); public static string GetRubric(bool useV0) => useV0 ? DefaultRubricV0 : DefaultRubric; public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; @@ -162,10 +162,10 @@ public static string BuildUserPrompt( { var output = useV0 ? OutputTemplateV0 : OutputTemplate; var rules = useV0 ? RulesV0 : Rules; - var severitySection = useV0 ? string.Empty : $@"SEVERITY + var severitySection = useV0 ? $@"SEVERITY {SeverityRules} -"; +" : string.Empty; return $@"SCHEMA {schemaJson} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs index dd950b206..6e83ea6a1 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs @@ -11,23 +11,11 @@ internal static class AttachmentPrompts "Produce a concise reviewer-facing summary of the provided attachment context."); public const string OutputSection = @"OUTPUT -- Plain text only -- 1-2 complete sentences"; - - public const string OutputSectionV0 = @"OUTPUT { ""summary"": """" }"; - public const string RulesSection = @"RULES -- Use only the provided attachment context as evidence. -- If text content is present, summarize the actual content. -- If text content is missing or empty, provide a conservative metadata-based summary. -- Do not invent missing details. -- Keep the summary specific, concrete, and reviewer-facing. -- Return plain text only (no markdown, bullets, or JSON)."; - - public const string RulesSectionV0 = "- Use only ATTACHMENT as evidence.\n" + public const string RulesSection = "- Use only ATTACHMENT as evidence.\n" + "- If ATTACHMENT.text is present, summarize actual content.\n" + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" + PromptCoreRules.NoInvention + "\n" @@ -41,6 +29,18 @@ internal static class AttachmentPrompts + PromptCoreRules.ValidJsonOnly + "\n" + PromptCoreRules.PlainJsonOnly; + public const string OutputSectionV0 = @"OUTPUT +- Plain text only +- 1-2 complete sentences"; + + public const string RulesSectionV0 = @"RULES +- Use only the provided attachment context as evidence. +- If text content is present, summarize the actual content. +- If text content is missing or empty, provide a conservative metadata-based summary. +- Do not invent missing details. +- Keep the summary specific, concrete, and reviewer-facing. +- Return plain text only (no markdown, bullets, or JSON)."; + public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; public static string GetOutputSection(bool useV0) => useV0 ? OutputSectionV0 : OutputSection; public static string GetRulesSection(bool useV0) => useV0 ? RulesSectionV0 : RulesSection; From c0e25bdf5ffd3e603fe4846268edca877d59e955 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Fri, 6 Mar 2026 18:49:26 -0800 Subject: [PATCH 05/10] AB#32009 Fix async action authorization loop and centralize extractor dispatch --- .../AI/TextExtractionService.cs | 57 +++++++++++-------- .../GrantApplicationAppService.cs | 10 ++-- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs index 9aad041a5..e2353ca79 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs @@ -23,10 +23,22 @@ public partial class TextExtractionService : ITextExtractionService, ITransientD private const int MaxDocxTableRows = 2000; private const int MaxDocxTableCellsPerRow = 50; private readonly ILogger _logger; + private readonly Dictionary> _extractorsByExtension; public TextExtractionService(ILogger logger) { _logger = logger; + _extractorsByExtension = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + [".txt"] = (_, content) => ExtractTextFromTextFile(content), + [".csv"] = (_, content) => ExtractTextFromTextFile(content), + [".json"] = (_, content) => ExtractTextFromTextFile(content), + [".xml"] = (_, content) => ExtractTextFromTextFile(content), + [".pdf"] = ExtractTextFromPdfFile, + [".docx"] = (name, content) => ExtractTextFromWordDocx(name, content), + [".xls"] = ExtractTextFromExcelFile, + [".xlsx"] = ExtractTextFromExcelFile + }; } public Task ExtractTextAsync(string fileName, byte[] fileContent, string contentType) @@ -42,46 +54,41 @@ public Task ExtractTextAsync(string fileName, byte[] fileContent, string var normalizedContentType = contentType?.ToLowerInvariant() ?? string.Empty; var extension = Path.GetExtension(fileName)?.ToLowerInvariant() ?? string.Empty; - string rawText; + if (extension == ".doc") + { + _logger.LogDebug("Legacy .doc extraction is not supported for {FileName}", fileName); + return Task.FromResult(string.Empty); + } + + if (_extractorsByExtension.TryGetValue(extension, out var extractor)) + { + var rawText = extractor(fileName, fileContent); + return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); + } - if (normalizedContentType.Contains("text/") || - extension == ".txt" || - extension == ".csv" || - extension == ".json" || - extension == ".xml") + if (normalizedContentType.Contains("text/")) { - rawText = ExtractTextFromTextFile(fileContent); + var rawText = ExtractTextFromTextFile(fileContent); return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } - if (normalizedContentType.Contains("pdf") || extension == ".pdf") + if (normalizedContentType.Contains("pdf")) { - rawText = ExtractTextFromPdfFile(fileName, fileContent); + var rawText = ExtractTextFromPdfFile(fileName, fileContent); return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } if (normalizedContentType.Contains("word") || normalizedContentType.Contains("msword") || - normalizedContentType.Contains("officedocument.wordprocessingml") || - extension == ".doc" || - extension == ".docx") + normalizedContentType.Contains("officedocument.wordprocessingml")) { - if (extension == ".docx" || normalizedContentType.Contains("officedocument.wordprocessingml")) - { - rawText = ExtractTextFromWordDocx(fileName, fileContent); - return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); - } - - _logger.LogDebug("Legacy .doc extraction is not supported for {FileName}", fileName); - return Task.FromResult(string.Empty); + var rawText = ExtractTextFromWordDocx(fileName, fileContent); + return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } - if (normalizedContentType.Contains("excel") || - normalizedContentType.Contains("spreadsheet") || - extension == ".xls" || - extension == ".xlsx") + if (normalizedContentType.Contains("excel") || normalizedContentType.Contains("spreadsheet")) { - rawText = ExtractTextFromExcelFile(fileName, fileContent); + var rawText = ExtractTextFromExcelFile(fileName, fileContent); return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs index 075d56866..f6c4943e8 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs @@ -974,11 +974,11 @@ public async Task> GetActions(Guid applicati // NOTE: Authorization is applied on the AppService layer and is false by default // AUTHORIZATION HANDLING - actionDtos.ForEach(async item => - { - item.IsPermitted = item.IsPermitted && (await AuthorizationService.IsGrantedAsync(application, GetActionAuthorizationRequirement(item.ApplicationAction))); - item.IsAuthorized = true; - }); + foreach (var item in actionDtos) + { + item.IsPermitted = item.IsPermitted && (await AuthorizationService.IsGrantedAsync(application, GetActionAuthorizationRequirement(item.ApplicationAction))); + item.IsAuthorized = true; + } return new ListResultDto(actionDtos); } From 10e8ec7b374cce0b3f7b7fc27e1ebb0c1bf7ed7b Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Tue, 10 Mar 2026 08:29:43 -0700 Subject: [PATCH 06/10] AB#32009 Normalize baseline snapshot filenames to .txt convention --- .../Baselines/v0/{OpenAIService.cs.txt => OpenAIService.txt} | 0 .../Baselines/v1/{AnalysisPrompts.cs.txt => AnalysisPrompts.txt} | 0 .../v1/{AttachmentPrompts.cs.txt => AttachmentPrompts.txt} | 0 .../Baselines/v1/{OpenAIService.cs.txt => OpenAIService.txt} | 0 .../Baselines/v1/{PromptCoreRules.cs.txt => PromptCoreRules.txt} | 0 .../Baselines/v1/{PromptHeader.cs.txt => PromptHeader.txt} | 0 .../v1/{ScoresheetPrompts.cs.txt => ScoresheetPrompts.txt} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/{OpenAIService.cs.txt => OpenAIService.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/{AnalysisPrompts.cs.txt => AnalysisPrompts.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/{AttachmentPrompts.cs.txt => AttachmentPrompts.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/{OpenAIService.cs.txt => OpenAIService.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/{PromptCoreRules.cs.txt => PromptCoreRules.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/{PromptHeader.cs.txt => PromptHeader.txt} (100%) rename applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/{ScoresheetPrompts.cs.txt => ScoresheetPrompts.txt} (100%) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.txt similarity index 100% rename from applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt rename to applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.txt From e7f0df16bbc957c923f41b9f2639933339127359 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Tue, 10 Mar 2026 10:51:22 -0700 Subject: [PATCH 07/10] AB#32009 Normalize versioned AI prompt template architecture --- .../AI/IAIService.cs | 1 - .../AI/OpenAIService.cs | 434 ++++++--- .../AI/Prompts/AnalysisPrompts.cs | 192 ---- .../AI/Prompts/AttachmentPrompts.cs | 59 -- .../AI/Prompts/Baselines/README.md | 8 - .../Prompts/Baselines/v0/AnalysisPrompts.txt | 58 -- .../Baselines/v0/AttachmentPrompts.txt | 29 - .../AI/Prompts/Baselines/v0/OpenAIService.txt | 406 --------- .../Prompts/Baselines/v0/PromptCoreRules.txt | 13 - .../AI/Prompts/Baselines/v0/PromptHeader.txt | 14 - .../Baselines/v0/ScoresheetPrompts.txt | 85 -- .../Prompts/Baselines/v1/AnalysisPrompts.txt | 126 --- .../Baselines/v1/AttachmentPrompts.txt | 27 - .../AI/Prompts/Baselines/v1/OpenAIService.txt | 833 ------------------ .../Prompts/Baselines/v1/PromptCoreRules.txt | 13 - .../AI/Prompts/Baselines/v1/PromptHeader.txt | 14 - .../Baselines/v1/ScoresheetPrompts.txt | 80 -- .../AI/Prompts/PromptCoreRules.cs | 13 - .../AI/Prompts/PromptHeader.cs | 14 - .../AI/Prompts/ScoresheetPrompts.cs | 123 --- .../AI/Prompts/Versions/README.md | 54 ++ .../Prompts/Versions/v0/analysis.system.txt | 12 + .../AI/Prompts/Versions/v0/analysis.user.txt | 101 +++ .../Prompts/Versions/v0/attachment.system.txt | 22 + .../Prompts/Versions/v0/attachment.user.txt | 2 + .../Prompts/Versions/v0/scoresheet.system.txt | 5 + .../Prompts/Versions/v0/scoresheet.user.txt | 52 ++ .../Prompts/Versions/v1/analysis.output.txt | 22 + .../Prompts/Versions/v1/analysis.rubric.txt | 5 + .../AI/Prompts/Versions/v1/analysis.rules.txt | 10 + .../AI/Prompts/Versions/v1/analysis.score.txt | 3 + .../Prompts/Versions/v1/analysis.system.txt | 5 + .../AI/Prompts/Versions/v1/analysis.user.txt | 21 + .../Prompts/Versions/v1/attachment.output.txt | 3 + .../Prompts/Versions/v1/attachment.rules.txt | 7 + .../Prompts/Versions/v1/attachment.system.txt | 5 + .../Prompts/Versions/v1/attachment.user.txt | 9 + .../AI/Prompts/Versions/v1/common.rules.txt | 5 + .../Prompts/Versions/v1/scoresheet.output.txt | 7 + .../Prompts/Versions/v1/scoresheet.rules.txt | 31 + .../Prompts/Versions/v1/scoresheet.system.txt | 5 + .../Prompts/Versions/v1/scoresheet.user.txt | 18 + .../Handlers/GenerateAIContentHandler.cs | 2 +- .../Unity.GrantManager.Application.csproj | 6 + 44 files changed, 737 insertions(+), 2217 deletions(-) delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.txt delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptCoreRules.cs delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptHeader.cs delete mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v0/analysis.system.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v0/analysis.user.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v0/attachment.system.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v0/attachment.user.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v0/scoresheet.system.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v0/scoresheet.user.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.output.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rubric.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.rules.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.score.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.system.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/analysis.user.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.output.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.rules.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.system.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/attachment.user.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/common.rules.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.output.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.rules.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.system.txt create mode 100644 applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/v1/scoresheet.user.txt diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/IAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/IAIService.cs index 160f8ed23..117eb080a 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/IAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application.Contracts/AI/IAIService.cs @@ -17,6 +17,5 @@ public interface IAIService // Legacy compatibility methods retained until flow orchestration refactor. Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150); Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null); - Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions); } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 9c1ecd11a..da42056d7 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -20,10 +20,25 @@ public class OpenAIService : IAIService, ITransientDependency private readonly ITextExtractionService _textExtractionService; private const string ApplicationAnalysisPromptType = "ApplicationAnalysis"; private const string AttachmentSummaryPromptType = "AttachmentSummary"; - private const string ScoresheetAllPromptType = "ScoresheetAll"; private const string ScoresheetSectionPromptType = "ScoresheetSection"; private const string PromptVersionV0 = "v0"; private const string PromptVersionV1 = "v1"; + private const string PromptTemplatesFolder = "AI\\Prompts\\Versions"; + private const string AnalysisSystemTemplateName = "analysis.system"; + private const string AnalysisUserTemplateName = "analysis.user"; + private const string AnalysisRubricTemplateName = "analysis.rubric"; + private const string AttachmentSystemTemplateName = "attachment.system"; + private const string AttachmentUserTemplateName = "attachment.user"; + private const string ScoresheetSystemTemplateName = "scoresheet.system"; + private const string ScoresheetUserTemplateName = "scoresheet.user"; + private const string ScoresheetOutputTemplateName = "scoresheet.output"; + private const string DefaultScoresheetOutputTemplate = @"{ + """": { + ""answer"": """", + ""rationale"": """", + ""confidence"": + } +}"; private const string NoSummaryGeneratedMessage = "No summary generated."; private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; @@ -41,8 +56,15 @@ public class OpenAIService : IAIService, ITransientDependency private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true }; - private string SelectedPromptVersion => NormalizePromptVersion(_configuration["Azure:OpenAI:PromptVersion"]); - private bool UseV0Prompts => string.Equals(SelectedPromptVersion, PromptVersionV0, StringComparison.OrdinalIgnoreCase); + private static readonly IReadOnlyDictionary PromptProfiles = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [PromptVersionV0] = PromptVersionV0, + [PromptVersionV1] = PromptVersionV1 + }; + private static readonly Dictionary PromptTemplateCache = new(StringComparer.OrdinalIgnoreCase); + + private string SelectedPromptVersion => ResolvePromptVersion(_configuration["Azure:OpenAI:PromptVersion"]); public OpenAIService( HttpClient httpClient, @@ -78,8 +100,8 @@ public async Task GenerateCompletionAsync(AICompletionRequ public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) { - var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); - var schemaJson = JsonSerializer.Serialize(request.Schema, JsonLogOptions); + var data = JsonSerializer.Serialize(request.Data, JsonLogOptions); + var schema = JsonSerializer.Serialize(request.Schema, JsonLogOptions); var attachmentsPayload = request.Attachments .Select(a => new @@ -89,10 +111,14 @@ public async Task GenerateApplicationAnalysisAsync( }) .Cast(); - var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); - var rubric = request.Rubric ?? AnalysisPrompts.GetRubric(UseV0Prompts); - var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, UseV0Prompts); - var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts); + var attachments = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); + var systemPrompt = BuildAnalysisSystemPrompt(SelectedPromptVersion); + var analysisContent = BuildAnalysisUserPrompt( + SelectedPromptVersion, + schema, + data, + attachments, + request.Rubric); await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw); @@ -173,11 +199,7 @@ public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] { var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); - var prompt = $@"{AttachmentPrompts.GetSystemPrompt(UseV0Prompts)} - -{AttachmentPrompts.GetOutputSection(UseV0Prompts)} - -{AttachmentPrompts.GetRulesSection(UseV0Prompts)}"; + var prompt = BuildAttachmentSystemPrompt(SelectedPromptVersion); var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText; if (attachmentText != null) @@ -196,13 +218,13 @@ public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] sizeBytes = fileContent.Length, text = attachmentText }; - var payloadJson = JsonSerializer.Serialize(attachmentPayload, JsonLogOptions); - var contentToAnalyze = AttachmentPrompts.BuildUserPrompt(payloadJson, UseV0Prompts); + var attachment = JsonSerializer.Serialize(attachmentPayload, JsonLogOptions); + var contentToAnalyze = BuildAttachmentUserPrompt(SelectedPromptVersion, attachment); await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze); var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput); - return UseV0Prompts ? ExtractSummaryFromJson(modelOutput) : modelOutput; + return ExtractSummaryFromJson(modelOutput); } catch (Exception ex) { @@ -259,14 +281,16 @@ public async Task AnalyzeApplicationAsync(string applicationContent, Lis .Cast() : Enumerable.Empty(); - var schemaJson = JsonSerializer.Serialize(schemaPayload, JsonLogOptions); - var dataJson = JsonSerializer.Serialize(dataPayload, JsonLogOptions); - var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); - var fallbackRubric = string.IsNullOrWhiteSpace(rubric) - ? AnalysisPrompts.GetRubric(UseV0Prompts) - : rubric; - var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, fallbackRubric, UseV0Prompts); - var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts); + var schema = JsonSerializer.Serialize(schemaPayload, JsonLogOptions); + var data = JsonSerializer.Serialize(dataPayload, JsonLogOptions); + var attachments = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); + var analysisContent = BuildAnalysisUserPrompt( + SelectedPromptVersion, + schema, + data, + attachments, + rubric); + var systemPrompt = BuildAnalysisSystemPrompt(SelectedPromptVersion); await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); @@ -352,63 +376,6 @@ private string AddIdsToAnalysisItems(string analysisJson) } } - public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", MissingApiKeyMessage); - return "{}"; - } - - try - { - var attachmentSummariesText = attachmentSummaries?.Count > 0 - ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) - : "No attachments provided."; - - var analysisContent = $@"APPLICATION CONTENT: -{applicationContent} - -ATTACHMENT SUMMARIES: -- {attachmentSummariesText} - -SCORESHEET QUESTIONS: -{scoresheetQuestions} - -Please analyze this grant application and provide appropriate answers for each scoresheet question. - -For numeric questions, provide a numeric value within the specified range. -For yes/no questions, provide either 'Yes' or 'No'. -For text questions, provide a concise, relevant response. -For select list questions, choose the most appropriate option from the provided choices. -For text area questions, provide a detailed but concise response. - -Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. - -Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: -{{ - ""question-id-1"": ""answer-value-1"", - ""question-id-2"": ""answer-value-2"" -}} -Do not return any markdown formatting, just the JSON by itself"; - - var systemPrompt = @"You are an expert grant application reviewer for the BC Government. -Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. -Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. -Respond only with valid JSON in the exact format requested."; - - await LogPromptInputAsync(ScoresheetAllPromptType, systemPrompt, analysisContent); - var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); - await LogPromptOutputAsync(ScoresheetAllPromptType, modelOutput); - return modelOutput; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet answers"); - return "{}"; - } - } - public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) { if (string.IsNullOrEmpty(ApiKey)) @@ -419,7 +386,7 @@ public async Task GenerateScoresheetSectionAnswersAsync(string applicati try { - var attachmentSummariesText = attachmentSummaries?.Count > 0 + var attachments = attachmentSummaries?.Count > 0 ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) : "No attachments provided."; @@ -442,17 +409,17 @@ public async Task GenerateScoresheetSectionAnswersAsync(string applicati name = sectionName, questions = sectionQuestionsPayload }; - var sectionPayloadJson = JsonSerializer.Serialize(sectionPayload, JsonLogOptions); - var responseTemplate = BuildScoresheetSectionResponseTemplate(sectionPayloadJson); + var section = JsonSerializer.Serialize(sectionPayload, JsonLogOptions); + var response = BuildScoresheetSectionResponseTemplate(SelectedPromptVersion, section); - var analysisContent = ScoresheetPrompts.BuildSectionUserPrompt( + var analysisContent = BuildScoresheetSectionUserPrompt( + SelectedPromptVersion, applicationContent, - attachmentSummariesText, - sectionPayloadJson, - responseTemplate, - UseV0Prompts); + attachments, + section, + response); - var systemPrompt = ScoresheetPrompts.GetSectionSystemPrompt(UseV0Prompts); + var systemPrompt = BuildScoresheetSectionSystemPrompt(SelectedPromptVersion); await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent); var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); @@ -565,22 +532,12 @@ private static List ParseFindings(JsonElement array) { title = titleProp.GetString(); } - else if (item.TryGetProperty("category", out var legacyTitleProp) && - legacyTitleProp.ValueKind == JsonValueKind.String) - { - title = legacyTitleProp.GetString(); - } string? detail = null; if (item.TryGetProperty(AIJsonKeys.Detail, out var detailProp) && detailProp.ValueKind == JsonValueKind.String) { detail = detailProp.GetString(); } - else if (item.TryGetProperty("message", out var legacyDetailProp) && - legacyDetailProp.ValueKind == JsonValueKind.String) - { - detail = legacyDetailProp.GetString(); - } findings.Add(new ApplicationAnalysisFinding { @@ -639,14 +596,14 @@ private static int NormalizeConfidence(int confidence) return Math.Clamp(rounded, 0, 100); } - private static string BuildScoresheetSectionResponseTemplate(string sectionPayloadJson) + private static string BuildScoresheetSectionResponseTemplate(string version, string sectionPayloadJson) { try { using var doc = JsonDocument.Parse(sectionPayloadJson); if (!doc.RootElement.TryGetProperty("questions", out var questions) || questions.ValueKind != JsonValueKind.Array) { - return ScoresheetPrompts.SectionOutputTemplate; + return BuildScoresheetSectionOutputTemplate(version); } var template = new Dictionary(); @@ -673,15 +630,31 @@ private static string BuildScoresheetSectionResponseTemplate(string sectionPaylo if (template.Count == 0) { - return ScoresheetPrompts.SectionOutputTemplate; + return BuildScoresheetSectionOutputTemplate(version); } return JsonSerializer.Serialize(template, JsonLogOptions); } catch (JsonException) { - return ScoresheetPrompts.SectionOutputTemplate; + return BuildScoresheetSectionOutputTemplate(version); + } + } + + private static string BuildScoresheetSectionOutputTemplate(string version) + { + if (TryGetPromptTemplate(version, ScoresheetOutputTemplateName, out var template)) + { + return template; + } + + if (string.Equals(version, PromptVersionV0, StringComparison.OrdinalIgnoreCase)) + { + return DefaultScoresheetOutputTemplate; } + + throw new InvalidOperationException( + $"Missing required prompt template '{ScoresheetOutputTemplateName}.txt' for prompt version '{version}'."); } private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt) @@ -834,16 +807,261 @@ private static int FindFirstJsonTokenIndex(string value) return arrayStart; } - private static string NormalizePromptVersion(string? version) + private static string ResolvePromptVersion(string? version) { - if (string.Equals(version, PromptVersionV0, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrWhiteSpace(version) && + PromptProfiles.TryGetValue(version.Trim(), out var selectedVersion)) { - return PromptVersionV0; + return selectedVersion; } return PromptVersionV1; } + private static string BuildAnalysisSystemPrompt(string version) + { + return GetRequiredPromptTemplate(version, AnalysisSystemTemplateName); + } + + private static string BuildAnalysisUserPrompt( + string version, + string schema, + string data, + string attachments, + string? rubric) + { + var replacements = new Dictionary + { + ["SCHEMA"] = schema, + ["DATA"] = data, + ["ATTACHMENTS"] = attachments, + ["RUBRIC"] = ResolveAnalysisRubric(rubric, version) + }; + + return RenderPromptTemplate(version, AnalysisUserTemplateName, replacements); + } + + private static string ResolveAnalysisRubric(string? providedRubric, string version) + { + if (!string.IsNullOrWhiteSpace(providedRubric)) + { + return providedRubric; + } + + return GetRequiredPromptTemplate(version, AnalysisRubricTemplateName); + } + + private static string BuildAttachmentSystemPrompt(string version) + { + return GetRequiredPromptTemplate(version, AttachmentSystemTemplateName); + } + + private static string BuildAttachmentUserPrompt(string version, string attachment) + { + return RenderPromptTemplate(version, AttachmentUserTemplateName, new Dictionary + { + ["ATTACHMENT"] = attachment + }); + } + + private static string BuildScoresheetSectionSystemPrompt(string version) + { + return GetRequiredPromptTemplate(version, ScoresheetSystemTemplateName); + } + + private static string BuildScoresheetSectionUserPrompt( + string version, + string data, + string attachments, + string section, + string response) + { + return RenderPromptTemplate(version, ScoresheetUserTemplateName, new Dictionary + { + ["DATA"] = data, + ["ATTACHMENTS"] = attachments, + ["SECTION"] = section, + ["RESPONSE"] = response + }); + } + + private static bool TryGetPromptTemplate(string version, string templateName, out string template) + { + template = string.Empty; + var cacheKey = $"{version}/{templateName}"; + if (PromptTemplateCache.TryGetValue(cacheKey, out var cachedTemplate)) + { + template = cachedTemplate; + return true; + } + + var path = Path.Combine(AppContext.BaseDirectory, PromptTemplatesFolder, version, $"{templateName}.txt"); + if (!File.Exists(path)) + { + return false; + } + + var loaded = File.ReadAllText(path); + if (string.IsNullOrWhiteSpace(loaded)) + { + return false; + } + + template = loaded; + PromptTemplateCache[cacheKey] = loaded; + return true; + } + + private static string GetRequiredPromptTemplate(string version, string templateName) + { + if (TryGetPromptTemplate(version, templateName, out var template)) + { + return template; + } + + throw new InvalidOperationException( + $"Missing required prompt template '{templateName}.txt' for prompt version '{version}'."); + } + + private static string RenderPromptTemplate( + string version, + string templateName, + IReadOnlyDictionary runtimeReplacements) + { + return RenderPromptTemplateInternal( + version, + templateName, + runtimeReplacements, + new HashSet(StringComparer.OrdinalIgnoreCase)); + } + + private static string RenderPromptTemplateInternal( + string version, + string templateName, + IReadOnlyDictionary runtimeReplacements, + ISet resolutionStack) + { + if (!resolutionStack.Add(templateName)) + { + throw new InvalidOperationException( + $"Detected cyclic prompt fragment reference while resolving '{templateName}.txt' for prompt version '{version}'."); + } + + var template = GetRequiredPromptTemplate(version, templateName); + var replacements = new Dictionary(runtimeReplacements, StringComparer.Ordinal); + var baseTemplateName = GetTemplateBaseName(templateName); + + foreach (var placeholder in GetTemplatePlaceholders(template)) + { + if (replacements.ContainsKey(placeholder)) + { + continue; + } + + var fragmentTemplateName = ResolveFragmentTemplateName(version, baseTemplateName, placeholder); + if (!string.IsNullOrWhiteSpace(fragmentTemplateName)) + { + replacements[placeholder] = RenderPromptTemplateInternal( + version, + fragmentTemplateName, + new Dictionary(StringComparer.Ordinal), + resolutionStack); + } + } + + var rendered = template; + foreach (var replacement in replacements) + { + rendered = rendered.Replace($"{{{{{replacement.Key}}}}}", replacement.Value ?? string.Empty, StringComparison.Ordinal); + } + + var unresolved = GetTemplatePlaceholders(rendered); + if (unresolved.Count > 0) + { + throw new InvalidOperationException( + $"Unresolved prompt placeholders in '{templateName}.txt' for prompt version '{version}': {string.Join(", ", unresolved.OrderBy(item => item))}"); + } + + resolutionStack.Remove(templateName); + return rendered; + } + + private static string? ResolveFragmentTemplateName(string version, string baseTemplateName, string placeholderName) + { + var normalizedPlaceholder = placeholderName.ToLowerInvariant(); + var baseScopedCandidate = $"{baseTemplateName}.{normalizedPlaceholder}"; + if (TryGetPromptTemplate(version, baseScopedCandidate, out _)) + { + return baseScopedCandidate; + } + + if (TryResolveCommonTemplateName(placeholderName, out var commonTemplateName)) + { + if (TryGetPromptTemplate(version, commonTemplateName, out _)) + { + return commonTemplateName; + } + } + + return null; + } + + private static bool TryResolveCommonTemplateName(string placeholderName, out string commonTemplateName) + { + commonTemplateName = string.Empty; + if (!placeholderName.StartsWith("COMMON_", StringComparison.Ordinal)) + { + return false; + } + + var suffix = placeholderName.Substring("COMMON_".Length).ToLowerInvariant(); + suffix = suffix.Replace('_', '.'); + commonTemplateName = $"common.{suffix}"; + return true; + } + + private static string GetTemplateBaseName(string templateName) + { + var separatorIndex = templateName.IndexOf('.', StringComparison.Ordinal); + if (separatorIndex <= 0) + { + return templateName; + } + + return templateName.Substring(0, separatorIndex); + } + + private static ISet GetTemplatePlaceholders(string template) + { + var placeholders = new HashSet(StringComparer.Ordinal); + var searchIndex = 0; + + while (searchIndex < template.Length) + { + var start = template.IndexOf("{{", searchIndex, StringComparison.Ordinal); + if (start < 0) + { + break; + } + + var end = template.IndexOf("}}", start + 2, StringComparison.Ordinal); + if (end < 0) + { + break; + } + + var placeholder = template.Substring(start + 2, end - start - 2).Trim(); + if (!string.IsNullOrWhiteSpace(placeholder)) + { + placeholders.Add(placeholder); + } + + searchIndex = end + 2; + } + + return placeholders; + } + private static string ExtractSummaryFromJson(string output) { if (!TryParseJsonObjectFromResponse(output, out var jsonObject)) diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs deleted file mode 100644 index 6cb16655c..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class AnalysisPrompts - { - public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. -COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. -FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. -RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. -QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; - - public const string DefaultRubricV0 = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: - -1. ELIGIBILITY REQUIREMENTS: - - Project must align with program objectives - - Applicant must be eligible entity type - - Budget must be reasonable and well-justified - - Project timeline must be realistic - -2. COMPLETENESS CHECKS: - - All required fields completed - - Necessary supporting documents provided - - Budget breakdown detailed and accurate - - Project description clear and comprehensive - -3. FINANCIAL REVIEW: - - Requested amount is within program limits - - Budget is reasonable for scope of work - - Matching funds or in-kind contributions identified - - Cost per outcome/beneficiary is reasonable - -4. RISK ASSESSMENT: - - Applicant capacity to deliver project - - Technical feasibility of proposed work - - Environmental or regulatory compliance - - Potential for cost overruns or delays - -5. QUALITY INDICATORS: - - Clear project objectives and outcomes - - Well-defined target audience/beneficiaries - - Appropriate project methodology - - Sustainability plan for long-term impact - -EVALUATION CRITERIA: -- HIGH: Meets all requirements, well-prepared application, low risk -- MEDIUM: Meets most requirements, minor issues or missing elements -- LOW: Missing key requirements, significant concerns, high risk"; - - public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. -MEDIUM: Application has some gaps or weaknesses that require reviewer attention. -LOW: Application has significant gaps or risks across key rubric areas."; - - public const string OutputTemplate = @"{ - ""rating"": """", - ""errors"": [ - { - ""title"": """", - ""detail"": """" - } - ], - ""warnings"": [ - { - ""title"": """", - ""detail"": """" - } - ], - ""summaries"": [ - { - ""title"": """", - ""detail"": """" - } - ] -}"; - - public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n" - + "- Do not invent fields, documents, requirements, or facts.\n" - + @"- Treat missing or empty values as findings only when they weaken rubric evidence. -- Prefer material issues; avoid nitpicking. -- Use 3-6 words for title. -- Each detail must be 1-2 complete sentences. -- Each detail must cite concrete evidence from DATA or ATTACHMENTS. -- If ATTACHMENTS evidence is used, cite the attachment by name in detail. -- If no findings exist, return empty arrays. -- Rating must be HIGH, MEDIUM, or LOW. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved. -WARNING: Issue that could negatively affect the application's approval. -RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; - - public const string OutputTemplateV0 = @"{ - ""rating"": ""HIGH/MEDIUM/LOW"", - ""warnings"": [ - { - ""category"": ""Brief summary of the warning"", - ""message"": ""Detailed warning message with full context and explanation"" - } - ], - ""errors"": [ - { - ""category"": ""Brief summary of the error"", - ""message"": ""Detailed error message with full context and explanation"" - } - ], - ""summaries"": [ - { - ""category"": ""Brief summary of the recommendation"", - ""message"": ""Detailed recommendation with specific actionable guidance"" - } - ], - ""dismissed"": [] -}"; - - public const string RulesV0 = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. -- Do not invent fields, documents, requirements, or facts. -- Treat missing or empty values as findings only when they weaken rubric evidence. -- Prefer material issues; avoid nitpicking. -- Each error/warning/recommendation must describe one concrete issue or consideration and why it matters. -- Use 3-6 words for category. -- Each message must be 1-2 complete sentences. -- Each message must be grounded in concrete evidence from provided inputs. -- If attachment evidence is used, reference the attachment explicitly in the message. -- Do not provide applicant-facing advice. -- Do not mention rubric section names in findings. -- If no findings exist, return empty arrays. -- rating must be HIGH, MEDIUM, or LOW." - + "\n" + PromptCoreRules.ExactOutputShape - + "\n" + PromptCoreRules.NoExtraOutputKeys - + "\n" + PromptCoreRules.ValidJsonOnly - + "\n" + PromptCoreRules.PlainJsonOnly; - - public static readonly string SystemPrompt = PromptHeader.Build( - "You are an expert grant analyst assistant for human reviewers.", - "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); - - public static readonly string SystemPromptV0 = PromptHeader.Build( - "You are an expert grant analyst assistant for human reviewers.", - "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); - - public static string GetRubric(bool useV0) => useV0 ? DefaultRubricV0 : DefaultRubric; - public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; - - public static string BuildUserPrompt( - string schemaJson, - string dataJson, - string attachmentsJson, - string rubric) - { - return BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, useV0: false); - } - - public static string BuildUserPrompt( - string schemaJson, - string dataJson, - string attachmentsJson, - string rubric, - bool useV0) - { - var output = useV0 ? OutputTemplateV0 : OutputTemplate; - var rules = useV0 ? RulesV0 : Rules; - var severitySection = useV0 ? $@"SEVERITY -{SeverityRules} - -" : string.Empty; - - return $@"SCHEMA -{schemaJson} - -DATA -{dataJson} - -ATTACHMENTS -{attachmentsJson} - -RUBRIC -{rubric} - -{severitySection}SCORE -{ScoreRules} - -OUTPUT -{output} - -RULES -{rules}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs deleted file mode 100644 index 6e83ea6a1..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs +++ /dev/null @@ -1,59 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class AttachmentPrompts - { - public static readonly string SystemPrompt = PromptHeader.Build( - "You are a professional grant analyst for the BC Government.", - "Produce a concise reviewer-facing summary of the provided attachment context."); - - public static readonly string SystemPromptV0 = PromptHeader.Build( - "You are a professional grant analyst for the BC Government.", - "Produce a concise reviewer-facing summary of the provided attachment context."); - - public const string OutputSection = @"OUTPUT -{ - ""summary"": """" -}"; - - public const string RulesSection = "- Use only ATTACHMENT as evidence.\n" - + "- If ATTACHMENT.text is present, summarize actual content.\n" - + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" - + PromptCoreRules.NoInvention + "\n" - + @"- Write 1-2 complete sentences. -- Summary must be grounded in concrete ATTACHMENT evidence. -- Return exactly one object with only the key: summary. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public const string OutputSectionV0 = @"OUTPUT -- Plain text only -- 1-2 complete sentences"; - - public const string RulesSectionV0 = @"RULES -- Use only the provided attachment context as evidence. -- If text content is present, summarize the actual content. -- If text content is missing or empty, provide a conservative metadata-based summary. -- Do not invent missing details. -- Keep the summary specific, concrete, and reviewer-facing. -- Return plain text only (no markdown, bullets, or JSON)."; - - public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; - public static string GetOutputSection(bool useV0) => useV0 ? OutputSectionV0 : OutputSection; - public static string GetRulesSection(bool useV0) => useV0 ? RulesSectionV0 : RulesSection; - - public static string BuildUserPrompt(string attachmentPayloadJson) - { - return BuildUserPrompt(attachmentPayloadJson, useV0: false); - } - - public static string BuildUserPrompt(string attachmentPayloadJson, bool useV0) - { - return $@"ATTACHMENT -{attachmentPayloadJson}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md deleted file mode 100644 index 4106b7aa0..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Prompt Baselines - -- `v0`: legacy prompt/service snapshots used as historical baseline. -- `v1`: current runtime prompt/service snapshots. - -Versioning convention: -- Folder name is the baseline version. -- Filenames inside each folder are normalized and do not include version suffixes. diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt deleted file mode 100644 index d267a1216..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt +++ /dev/null @@ -1,58 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class AnalysisPrompts - { - public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. -COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. -FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. -RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. -QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; - - public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. -MEDIUM: Application has some gaps or weaknesses that require reviewer attention. -LOW: Application has significant gaps or risks across key rubric areas."; - - public const string OutputTemplate = @"{ - ""rating"": """", - ""errors"": [ - { - ""title"": """", - ""detail"": """" - } - ], - ""warnings"": [ - { - ""title"": """", - ""detail"": """" - } - ], - ""summaries"": [ - { - ""title"": """", - ""detail"": """" - } - ] -}"; - - public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n" - + "- Do not invent fields, documents, requirements, or facts.\n" - + @"- Treat missing or empty values as findings only when they weaken rubric evidence. -- Prefer material issues; avoid nitpicking. -- Use 3-6 words for title. -- Each detail must be 1-2 complete sentences. -- Each detail must cite concrete evidence from DATA or ATTACHMENTS. -- If ATTACHMENTS evidence is used, cite the attachment by name in detail. -- If no findings exist, return empty arrays. -- Rating must be HIGH, MEDIUM, or LOW. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public static readonly string SystemPrompt = PromptHeader.Build( - "You are an expert grant analyst assistant for human reviewers.", - "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt deleted file mode 100644 index a61cc5084..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt +++ /dev/null @@ -1,29 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class AttachmentPrompts - { - public static readonly string SystemPrompt = PromptHeader.Build( - "You are a professional grant analyst for the BC Government.", - "Produce a concise reviewer-facing summary of the provided attachment context."); - - public const string OutputSection = @"OUTPUT -{ - ""summary"": """" -}"; - - public const string RulesSection = "- Use only ATTACHMENT as evidence.\n" - + "- If ATTACHMENT.text is present, summarize actual content.\n" - + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" - + PromptCoreRules.NoInvention + "\n" - + @"- Write 1-2 complete sentences. -- Summary must be grounded in concrete ATTACHMENT evidence. -- Return exactly one object with only the key: summary. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - } -} - diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.txt deleted file mode 100644 index 239db1062..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.txt +++ /dev/null @@ -1,406 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Volo.Abp.DependencyInjection; - -namespace Unity.GrantManager.AI -{ - public class OpenAIService : IAIService, ITransientDependency - { - private readonly HttpClient _httpClient; - private readonly IConfiguration _configuration; - private readonly ILogger _logger; - private readonly ITextExtractionService _textExtractionService; - - private string? ApiKey => _configuration["AI:OpenAI:ApiKey"]; - private string? ApiUrl => _configuration["AI:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; - private readonly string NoKeyError = "OpenAI API key is not configured"; - - public OpenAIService(HttpClient httpClient, IConfiguration configuration, ILogger logger, ITextExtractionService textExtractionService) - { - _httpClient = httpClient; - _configuration = configuration; - _logger = logger; - _textExtractionService = textExtractionService; - } - - public Task IsAvailableAsync() - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("Error: {Message}", NoKeyError); - return Task.FromResult(false); - } - - return Task.FromResult(true); - } - - public async Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("Error: {Message}", NoKeyError); - return "AI analysis not available - service not configured."; - } - - _logger.LogDebug("Calling OpenAI with prompt: {Prompt}", content); - - try - { - var systemPrompt = prompt ?? "You are a professional grant analyst for the BC Government."; - - var requestBody = new - { - messages = new[] - { - new { role = "system", content = systemPrompt }, - new { role = "user", content = content } - }, - max_tokens = maxTokens, - temperature = 0.3 - }; - - var json = JsonSerializer.Serialize(requestBody); - var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); - - _httpClient.DefaultRequestHeaders.Clear(); - _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); - - var response = await _httpClient.PostAsync(ApiUrl, httpContent); - var responseContent = await response.Content.ReadAsStringAsync(); - - _logger.LogDebug("Response: {Response}", responseContent); - - if (!response.IsSuccessStatusCode) - { - _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); - return "AI analysis failed - service temporarily unavailable."; - } - - using var jsonDoc = JsonDocument.Parse(responseContent); - var choices = jsonDoc.RootElement.GetProperty("choices"); - if (choices.GetArrayLength() > 0) - { - var message = choices[0].GetProperty("message"); - return message.GetProperty("content").GetString() ?? "No summary generated."; - } - - return "No summary generated."; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating AI summary"); - return "AI analysis failed - please try again later."; - } - } - - public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] fileContent, string contentType) - { - try - { - var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); - - string contentToAnalyze; - string prompt; - - if (!string.IsNullOrWhiteSpace(extractedText)) - { - _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText.Length, fileName); - - contentToAnalyze = $"Document: {fileName}\nType: {contentType}\nContent:\n{extractedText}"; - prompt = "Please analyze this document and provide a concise summary of its content, purpose, and key information, for use by your fellow grant analysts. It should be 1-2 sentences long and about 46 tokens."; - } - else - { - _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", fileName); - - contentToAnalyze = $"File: {fileName}, Type: {contentType}, Size: {fileContent.Length} bytes"; - prompt = "Please analyze this document and provide a concise summary of its content, purpose, and key information, for use by your fellow grant analysts. It should be 1-2 sentences long and about 46 tokens."; - } - - return await GenerateSummaryAsync(contentToAnalyze, prompt, 150); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); - return $"AI analysis not available for this attachment ({fileName})."; - } - } - - public async Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", NoKeyError); - return "AI analysis not available - service not configured."; - } - - try - { - var attachmentSummariesText = attachmentSummaries?.Count > 0 - ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) - : "No attachments provided."; - - var fieldConfigurationSection = !string.IsNullOrEmpty(formFieldConfiguration) - ? $@" -{formFieldConfiguration}" - : string.Empty; - - var analysisContent = $@"APPLICATION CONTENT: -{applicationContent} - -ATTACHMENT SUMMARIES: -- {attachmentSummariesText} -{fieldConfigurationSection} - -EVALUATION RUBRIC: -{rubric} - -Analyze this grant application comprehensively across all five rubric categories (Eligibility, Completeness, Financial Review, Risk Assessment, and Quality Indicators). Identify issues, concerns, and areas for improvement. Return your findings in the following JSON format: -{{ - ""overall_score"": ""HIGH/MEDIUM/LOW"", - ""warnings"": [ - {{ - ""category"": ""Brief summary of the warning"", - ""message"": ""Detailed warning message with full context and explanation"", - ""severity"": ""WARNING"" - }} - ], - ""errors"": [ - {{ - ""category"": ""Brief summary of the error"", - ""message"": ""Detailed error message with full context and explanation"", - ""severity"": ""ERROR"" - }} - ], - ""recommendations"": [ - {{ - ""category"": ""Brief summary of the recommendation"", - ""message"": ""Detailed recommendation with specific actionable guidance"" - }} - ] -}} - -Important: The 'category' field should be a concise summary (3-6 words) that captures the essence of the issue, while the 'message' field should contain the detailed explanation."; - - var systemPrompt = @"You are an expert grant application reviewer for the BC Government. - -Conduct a thorough, comprehensive analysis across all rubric categories. Identify substantive issues, concerns, and opportunities for improvement. - -Classify findings based on their impact on the application's evaluation and fundability: -- ERRORS: Important missing information, significant gaps in required content, compliance issues, or major concerns affecting eligibility -- WARNINGS: Areas needing clarification, moderate issues, or concerns that should be addressed - -Evaluate the quality, clarity, and appropriateness of all application content. Be thorough but fair - identify real issues while avoiding nitpicking. - -Respond only with valid JSON in the exact format requested."; - - var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); - - // Post-process the AI response to add unique IDs to errors and warnings - return AddIdsToAnalysisItems(rawAnalysis); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error analyzing application"); - return "AI analysis failed - please try again later."; - } - } - - private string AddIdsToAnalysisItems(string analysisJson) - { - try - { - using var jsonDoc = JsonDocument.Parse(analysisJson); - using var memoryStream = new System.IO.MemoryStream(); - using (var writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions { Indented = true })) - { - writer.WriteStartObject(); - - foreach (var property in jsonDoc.RootElement.EnumerateObject()) - { - if (property.Name == "errors" || property.Name == "warnings") - { - writer.WritePropertyName(property.Name); - writer.WriteStartArray(); - - foreach (var item in property.Value.EnumerateArray()) - { - writer.WriteStartObject(); - - // Add unique ID first - writer.WriteString("id", Guid.NewGuid().ToString()); - - // Copy existing properties - foreach (var itemProperty in item.EnumerateObject()) - { - itemProperty.WriteTo(writer); - } - - writer.WriteEndObject(); - } - - writer.WriteEndArray(); - } - else - { - property.WriteTo(writer); - } - } - - // Add dismissed_items array if not present - if (!jsonDoc.RootElement.TryGetProperty("dismissed_items", out _)) - { - writer.WritePropertyName("dismissed_items"); - writer.WriteStartArray(); - writer.WriteEndArray(); - } - - writer.WriteEndObject(); - } - - return Encoding.UTF8.GetString(memoryStream.ToArray()); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error adding IDs to analysis items, returning original JSON"); - return analysisJson; // Return original if processing fails - } - } - - public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", NoKeyError); - return "{}"; - } - - try - { - var attachmentSummariesText = attachmentSummaries?.Count > 0 - ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) - : "No attachments provided."; - - var analysisContent = $@"APPLICATION CONTENT: -{applicationContent} - -ATTACHMENT SUMMARIES: -- {attachmentSummariesText} - -SCORESHEET QUESTIONS: -{scoresheetQuestions} - -Please analyze this grant application and provide appropriate answers for each scoresheet question. - -For numeric questions, provide a numeric value within the specified range. -For yes/no questions, provide either 'Yes' or 'No'. -For text questions, provide a concise, relevant response. -For select list questions, choose the most appropriate option from the provided choices. -For text area questions, provide a detailed but concise response. - -Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. - -Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: -{{ - ""question-id-1"": ""answer-value-1"", - ""question-id-2"": ""answer-value-2"" -}} -Do not return any markdown formatting, just the JSON by itself"; - - var systemPrompt = @"You are an expert grant application reviewer for the BC Government. -Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. -Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. -Respond only with valid JSON in the exact format requested."; - - return await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet answers"); - return "{}"; - } - } - - public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", NoKeyError); - return "{}"; - } - - try - { - var attachmentSummariesText = attachmentSummaries?.Count > 0 - ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) - : "No attachments provided."; - - var analysisContent = $@"APPLICATION CONTENT: -{applicationContent} - -ATTACHMENT SUMMARIES: -- {attachmentSummariesText} - -SCORESHEET SECTION: {sectionName} -{sectionJson} - -Please analyze this grant application and provide appropriate answers for each question in the ""{sectionName}"" section only. - -For each question, provide: -1. Your answer based on the application content -2. A brief cited description (1-2 sentences) explaining your reasoning with specific references to the application content -3. A confidence score from 0-100 indicating how confident you are in your answer based on available information - -Guidelines for answers: -- For numeric questions, provide a numeric value within the specified range -- For yes/no questions, provide either 'Yes' or 'No' -- For text questions, provide a concise, relevant response -- For select list questions, respond with ONLY the number from the 'number' field (1, 2, 3, etc.) of your chosen option. NEVER return 0 - the lowest valid answer is 1. For example: if you want '(0 pts) No outcomes provided', choose the option where number=1, not 0. -- For text area questions, provide a detailed but concise response -- Base your confidence score on how clearly the application content supports your answer - -Return your response as a JSON object where each key is the question ID and the value contains the answer, citation, and confidence: -{{ - ""question-id-1"": {{ - ""answer"": ""your-answer-here"", - ""citation"": ""Brief explanation with specific reference to application content"", - ""confidence"": 85 - }}, - ""question-id-2"": {{ - ""answer"": ""3"", - ""citation"": ""Based on the project budget of $50,000 mentioned in the application, this falls into the medium budget category"", - ""confidence"": 90 - }} -}} - -IMPORTANT FOR SELECT LIST QUESTIONS: If a question has availableOptions like: -[{{""number"":1,""value"":""Low (Under $25K)""}}, {{""number"":2,""value"":""Medium ($25K-$75K)""}}, {{""number"":3,""value"":""High (Over $75K)""}}] -Then respond with ONLY the number (e.g., ""3"" for ""High (Over $75K)""), not the text value. - -Do not return any markdown formatting, just the JSON by itself"; - - var systemPrompt = @"You are an expert grant application reviewer for the BC Government. -Analyze the provided application and generate appropriate answers for the scoresheet section questions based on the application content. -Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. -Always provide citations that reference specific parts of the application content to support your answers. -Be honest about your confidence level - if information is missing or unclear, reflect this in a lower confidence score. -Respond only with valid JSON in the exact format requested."; - - return await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", sectionName); - return "{}"; - } - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt deleted file mode 100644 index e11dce3c9..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt +++ /dev/null @@ -1,13 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class PromptCoreRules - { - public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; - public const string NoInvention = "- Do not invent missing details."; - public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; - public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; - public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; - public const string ValidJsonOnly = "- Return valid JSON only."; - public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt deleted file mode 100644 index 701a43e74..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt +++ /dev/null @@ -1,14 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class PromptHeader - { - public static string Build(string role, string task) - { - return $@"ROLE -{role} - -TASK -{task}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt deleted file mode 100644 index bfe883d64..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt +++ /dev/null @@ -1,85 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class ScoresheetPrompts - { - public static readonly string AllSystemPrompt = PromptHeader.Build( - "You are an expert grant application reviewer for the BC Government.", - "Using DATA, ATTACHMENTS, QUESTIONS, OUTPUT, and RULES, provide answers for all scoresheet questions."); - - public const string AllOutputTemplate = @"{ - """": """" -}"; - - public const string AllRules = "- Use only DATA and ATTACHMENTS as evidence.\n" - + "- Do not invent missing application details.\n" - + @"- Return exactly one answer per question ID in QUESTIONS. -- Do not omit any question IDs from QUESTIONS. -- Do not add keys that are not question IDs from QUESTIONS. -- The ""answer"" value type must match the question type. -- For numeric questions, return a numeric value within the allowed range. -- For yes/no questions, return exactly ""Yes"" or ""No"". -- For select list questions, return only the selected options.number as a string and never return option label text. -- For text and text area questions, return concise, evidence-based text. -- For text and text area questions, include concise source-grounded rationale from the provided input content. -- If explicit evidence is insufficient, choose the most conservative valid answer. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public static readonly string SectionSystemPrompt = PromptHeader.Build( - "You are an expert grant application reviewer for the BC Government.", - "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); - - public const string SectionOutputTemplate = @"{ - """": { - ""answer"": """", - ""rationale"": """", - ""confidence"": - } -}"; - - public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" - + "- Do not invent missing application details.\n" - + @"- Return exactly one answer object per question ID in SECTION.questions. -- Do not omit any question IDs from SECTION.questions. -- Do not add keys that are not question IDs from SECTION.questions. -- Use RESPONSE as the output contract and fill every placeholder value. -- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. -- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". -- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. -- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. -- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. -- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. -- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. -- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. -- Do not treat missing or non-contradictory information as evidence. -- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. -- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. -- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". -- For numeric questions, answer must be a numeric value within the allowed range. -- For numeric questions, answer must never be blank. -- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. -- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. -- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. -- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. -- Never return 0 for select list answers unless 0 exists as an explicit option number. -- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. -- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. -- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. -- Do not leave rationale empty when answer is populated. -- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". -- If any answer object is incomplete, regenerate the full JSON response before returning it. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - } -} - - - diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.txt deleted file mode 100644 index 2e5441280..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.txt +++ /dev/null @@ -1,126 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class AnalysisPrompts - { - public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: - -1. ELIGIBILITY REQUIREMENTS: - - Project must align with program objectives - - Applicant must be eligible entity type - - Budget must be reasonable and well-justified - - Project timeline must be realistic - -2. COMPLETENESS CHECKS: - - All required fields completed - - Necessary supporting documents provided - - Budget breakdown detailed and accurate - - Project description clear and comprehensive - -3. FINANCIAL REVIEW: - - Requested amount is within program limits - - Budget is reasonable for scope of work - - Matching funds or in-kind contributions identified - - Cost per outcome/beneficiary is reasonable - -4. RISK ASSESSMENT: - - Applicant capacity to deliver project - - Technical feasibility of proposed work - - Environmental or regulatory compliance - - Potential for cost overruns or delays - -5. QUALITY INDICATORS: - - Clear project objectives and outcomes - - Well-defined target audience/beneficiaries - - Appropriate project methodology - - Sustainability plan for long-term impact - -EVALUATION CRITERIA: -- HIGH: Meets all requirements, well-prepared application, low risk -- MEDIUM: Meets most requirements, minor issues or missing elements -- LOW: Missing key requirements, significant concerns, high risk"; - - public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. -MEDIUM: Application has some gaps or weaknesses that require reviewer attention. -LOW: Application has significant gaps or risks across key rubric areas."; - - public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved. -WARNING: Issue that could negatively affect the application's approval. -RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; - - public const string OutputTemplate = @"{ - ""rating"": ""HIGH/MEDIUM/LOW"", - ""warnings"": [ - { - ""title"": ""Brief summary of the warning"", - ""detail"": ""Detailed warning message with full context and explanation"" - } - ], - ""errors"": [ - { - ""title"": ""Brief summary of the error"", - ""detail"": ""Detailed error message with full context and explanation"" - } - ], - ""summaries"": [ - { - ""title"": ""Brief summary of the recommendation"", - ""detail"": ""Detailed recommendation with specific actionable guidance"" - } - ], - ""dismissed"": [] -}"; - - public const string Rules = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. -- Do not invent fields, documents, requirements, or facts. -- Treat missing or empty values as findings only when they weaken rubric evidence. -- Prefer material issues; avoid nitpicking. -- Each error/warning/recommendation must describe one concrete issue or consideration and why it matters. -- Use 3-6 words for title. -- Each detail must be 1-2 complete sentences. -- Each detail must be grounded in concrete evidence from provided inputs. -- If attachment evidence is used, reference the attachment explicitly in detail. -- Do not provide applicant-facing advice. -- Do not mention rubric section names in findings. -- If no findings exist, return empty arrays. -- rating must be HIGH, MEDIUM, or LOW." - + "\n" + PromptCoreRules.ExactOutputShape - + "\n" + PromptCoreRules.NoExtraOutputKeys - + "\n" + PromptCoreRules.ValidJsonOnly - + "\n" + PromptCoreRules.PlainJsonOnly; - - public static readonly string SystemPrompt = PromptHeader.Build( - "You are an expert grant analyst assistant for human reviewers.", - "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); - - public static string BuildUserPrompt( - string schemaJson, - string dataJson, - string attachmentsJson, - string rubric) - { - return $@"SCHEMA -{schemaJson} - -DATA -{dataJson} - -ATTACHMENTS -{attachmentsJson} - -RUBRIC -{rubric} - -SEVERITY -{SeverityRules} - -SCORE -{ScoreRules} - -OUTPUT -{OutputTemplate} - -RULES -{Rules}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.txt deleted file mode 100644 index 969480ea8..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.txt +++ /dev/null @@ -1,27 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class AttachmentPrompts - { - public static readonly string SystemPrompt = PromptHeader.Build( - "You are a professional grant analyst for the BC Government.", - "Produce a concise reviewer-facing summary of the provided attachment context."); - - public const string OutputSection = @"OUTPUT -- Plain text only -- 1-2 complete sentences"; - - public const string RulesSection = @"RULES -- Use only the provided attachment context as evidence. -- If text content is present, summarize the actual content. -- If text content is missing or empty, provide a conservative metadata-based summary. -- Do not invent missing details. -- Keep the summary specific, concrete, and reviewer-facing. -- Return plain text only (no markdown, bullets, or JSON)."; - - public static string BuildUserPrompt(string attachmentPayloadJson) - { - return $@"ATTACHMENT -{attachmentPayloadJson}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.txt deleted file mode 100644 index 418c31ebc..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.txt +++ /dev/null @@ -1,833 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Volo.Abp.DependencyInjection; - -namespace Unity.GrantManager.AI -{ - public class OpenAIService : IAIService, ITransientDependency - { - private readonly HttpClient _httpClient; - private readonly IConfiguration _configuration; - private readonly ILogger _logger; - private readonly ITextExtractionService _textExtractionService; - private const string ApplicationAnalysisPromptType = "ApplicationAnalysis"; - private const string AttachmentSummaryPromptType = "AttachmentSummary"; - private const string ScoresheetAllPromptType = "ScoresheetAll"; - private const string ScoresheetSectionPromptType = "ScoresheetSection"; - private const string NoSummaryGeneratedMessage = "No summary generated."; - private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; - private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; - private const string SummaryFailedRetryMessage = "AI analysis failed - please try again later."; - - private string? ApiKey => _configuration["Azure:OpenAI:ApiKey"]; - private string? ApiUrl => _configuration["Azure:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; - private readonly string MissingApiKeyMessage = "OpenAI API key is not configured"; - - // Optional local debugging sink for prompt payload logs to a local file. - // Not intended for deployed/shared environments. - private bool IsPromptFileLoggingEnabled => _configuration.GetValue("Azure:Logging:EnablePromptFileLog") ?? false; - private const string PromptLogDirectoryName = "logs"; - private static readonly string PromptLogFileName = $"ai-prompts-{DateTime.UtcNow:yyyyMMdd-HHmmss}-{Environment.ProcessId}.log"; - - private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true }; - - public OpenAIService( - HttpClient httpClient, - IConfiguration configuration, - ILogger logger, - ITextExtractionService textExtractionService) - { - _httpClient = httpClient; - _configuration = configuration; - _logger = logger; - _textExtractionService = textExtractionService; - } - - public Task IsAvailableAsync() - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); - return Task.FromResult(false); - } - - return Task.FromResult(true); - } - - public async Task GenerateCompletionAsync(AICompletionRequest request) - { - var content = await GenerateSummaryAsync( - request?.UserPrompt ?? string.Empty, - request?.SystemPrompt, - request?.MaxTokens ?? 150); - return new AICompletionResponse { Content = content }; - } - - public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) - { - var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); - var schemaJson = JsonSerializer.Serialize(request.Schema, JsonLogOptions); - - var attachmentsPayload = request.Attachments - .Select(a => new - { - name = string.IsNullOrWhiteSpace(a.Name) ? "attachment" : a.Name.Trim(), - summary = string.IsNullOrWhiteSpace(a.Summary) ? string.Empty : a.Summary.Trim() - }) - .Cast(); - - var analysisContent = AnalysisPrompts.BuildUserPrompt( - schemaJson, - dataJson, - JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), - request.Rubric ?? string.Empty); - - var systemPrompt = AnalysisPrompts.SystemPrompt; - await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); - var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); - await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw); - return ParseApplicationAnalysisResponse(AddIdsToAnalysisItems(raw)); - } - - public async Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); - return ServiceNotConfiguredMessage; - } - - _logger.LogDebug("Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", content?.Length ?? 0, maxTokens); - - try - { - var systemPrompt = prompt ?? "You are a professional grant analyst for the BC Government."; - var userPrompt = content ?? string.Empty; - - var requestBody = new - { - messages = new[] - { - new { role = "system", content = systemPrompt }, - new { role = "user", content = userPrompt } - }, - max_tokens = maxTokens, - temperature = 0.3 - }; - - var json = JsonSerializer.Serialize(requestBody); - var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); - - _httpClient.DefaultRequestHeaders.Clear(); - _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); - - var response = await _httpClient.PostAsync(ApiUrl, httpContent); - var responseContent = await response.Content.ReadAsStringAsync(); - - _logger.LogDebug( - "OpenAI chat completions response received. StatusCode: {StatusCode}, ResponseLength: {ResponseLength}", - response.StatusCode, - responseContent?.Length ?? 0); - - if (!response.IsSuccessStatusCode) - { - _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); - return ServiceTemporarilyUnavailableMessage; - } - - if (string.IsNullOrWhiteSpace(responseContent)) - { - return NoSummaryGeneratedMessage; - } - - using var jsonDoc = JsonDocument.Parse(responseContent); - var choices = jsonDoc.RootElement.GetProperty("choices"); - if (choices.GetArrayLength() > 0) - { - var message = choices[0].GetProperty("message"); - return message.GetProperty("content").GetString() ?? NoSummaryGeneratedMessage; - } - - return NoSummaryGeneratedMessage; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating AI summary"); - return SummaryFailedRetryMessage; - } - } - - public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] fileContent, string contentType) - { - try - { - var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); - - var prompt = $@"{AttachmentPrompts.SystemPrompt} - -{AttachmentPrompts.OutputSection} - -{AttachmentPrompts.RulesSection}"; - - var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText; - if (attachmentText != null) - { - _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText.Length, fileName); - } - else - { - _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", fileName); - } - - var attachmentPayload = new - { - name = fileName, - contentType, - sizeBytes = fileContent.Length, - text = attachmentText - }; - var contentToAnalyze = AttachmentPrompts.BuildUserPrompt( - JsonSerializer.Serialize(attachmentPayload, JsonLogOptions)); - - await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze); - var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); - await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput); - return modelOutput; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); - return $"AI analysis not available for this attachment ({fileName})."; - } - } - - public async Task GenerateAttachmentSummaryAsync(AttachmentSummaryRequest request) - { - var summary = await GenerateAttachmentSummaryAsync( - request?.FileName ?? string.Empty, - request?.FileContent ?? Array.Empty(), - request?.ContentType ?? "application/octet-stream"); - return new AttachmentSummaryResponse { Summary = summary }; - } - - public async Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", MissingApiKeyMessage); - return ServiceNotConfiguredMessage; - } - - try - { - object schemaPayload = new { }; - if (!string.IsNullOrWhiteSpace(formFieldConfiguration)) - { - try - { - using var schemaDoc = JsonDocument.Parse(formFieldConfiguration); - schemaPayload = schemaDoc.RootElement.Clone(); - } - catch (JsonException ex) - { - _logger.LogWarning(ex, "Invalid form field configuration JSON. Using empty schema payload."); - } - } - - var dataPayload = new - { - applicationContent - }; - - var attachmentsPayload = attachmentSummaries?.Count > 0 - ? attachmentSummaries - .Select((summary, index) => new - { - name = $"Attachment {index + 1}", - summary = summary - }) - .Cast() - : Enumerable.Empty(); - - var analysisContent = AnalysisPrompts.BuildUserPrompt( - JsonSerializer.Serialize(schemaPayload, JsonLogOptions), - JsonSerializer.Serialize(dataPayload, JsonLogOptions), - JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), - rubric); - - var systemPrompt = AnalysisPrompts.SystemPrompt; - - await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); - var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); - await LogPromptOutputAsync(ApplicationAnalysisPromptType, rawAnalysis); - - // Post-process the AI response to add unique IDs to errors and warnings - return AddIdsToAnalysisItems(rawAnalysis); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error analyzing application"); - return SummaryFailedRetryMessage; - } - } - - private string AddIdsToAnalysisItems(string analysisJson) - { - try - { - using var jsonDoc = JsonDocument.Parse(analysisJson); - using var memoryStream = new System.IO.MemoryStream(); - using (var writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions { Indented = true })) - { - writer.WriteStartObject(); - - foreach (var property in jsonDoc.RootElement.EnumerateObject()) - { - var outputPropertyName = property.Name; - - if (outputPropertyName == AIJsonKeys.Errors || outputPropertyName == AIJsonKeys.Warnings) - { - writer.WritePropertyName(outputPropertyName); - writer.WriteStartArray(); - - foreach (var item in property.Value.EnumerateArray()) - { - writer.WriteStartObject(); - - // Add unique ID first - writer.WriteString("id", Guid.NewGuid().ToString()); - - // Copy existing properties - foreach (var itemProperty in item.EnumerateObject()) - { - itemProperty.WriteTo(writer); - } - - writer.WriteEndObject(); - } - - writer.WriteEndArray(); - } - else - { - if (outputPropertyName != property.Name) - { - writer.WritePropertyName(outputPropertyName); - property.Value.WriteTo(writer); - continue; - } - - property.WriteTo(writer); - } - } - - // Add dismissed array if not present. - if (!jsonDoc.RootElement.TryGetProperty(AIJsonKeys.Dismissed, out _)) - { - writer.WritePropertyName(AIJsonKeys.Dismissed); - writer.WriteStartArray(); - writer.WriteEndArray(); - } - - writer.WriteEndObject(); - } - - return Encoding.UTF8.GetString(memoryStream.ToArray()); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error adding IDs to analysis items, returning original JSON"); - return analysisJson; // Return original if processing fails - } - } - - public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", MissingApiKeyMessage); - return "{}"; - } - - try - { - var attachmentSummariesText = attachmentSummaries?.Count > 0 - ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) - : "No attachments provided."; - - var analysisContent = $@"APPLICATION CONTENT: -{applicationContent} - -ATTACHMENT SUMMARIES: -- {attachmentSummariesText} - -SCORESHEET QUESTIONS: -{scoresheetQuestions} - -Please analyze this grant application and provide appropriate answers for each scoresheet question. - -For numeric questions, provide a numeric value within the specified range. -For yes/no questions, provide either 'Yes' or 'No'. -For text questions, provide a concise, relevant response. -For select list questions, choose the most appropriate option from the provided choices. -For text area questions, provide a detailed but concise response. - -Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. - -Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: -{{ - ""question-id-1"": ""answer-value-1"", - ""question-id-2"": ""answer-value-2"" -}} -Do not return any markdown formatting, just the JSON by itself"; - - var systemPrompt = @"You are an expert grant application reviewer for the BC Government. -Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. -Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. -Respond only with valid JSON in the exact format requested."; - - await LogPromptInputAsync(ScoresheetAllPromptType, systemPrompt, analysisContent); - var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); - await LogPromptOutputAsync(ScoresheetAllPromptType, modelOutput); - return modelOutput; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet answers"); - return "{}"; - } - } - - public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) - { - if (string.IsNullOrEmpty(ApiKey)) - { - _logger.LogWarning("{Message}", MissingApiKeyMessage); - return "{}"; - } - - try - { - var attachmentSummariesText = attachmentSummaries?.Count > 0 - ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) - : "No attachments provided."; - - object sectionQuestionsPayload = sectionJson; - if (!string.IsNullOrWhiteSpace(sectionJson)) - { - try - { - using var sectionDoc = JsonDocument.Parse(sectionJson); - sectionQuestionsPayload = sectionDoc.RootElement.Clone(); - } - catch (JsonException) - { - // Keep raw string payload when JSON parsing fails. - } - } - - var sectionPayload = new - { - name = sectionName, - questions = sectionQuestionsPayload - }; - var sectionPayloadJson = JsonSerializer.Serialize(sectionPayload, JsonLogOptions); - var responseTemplate = BuildScoresheetSectionResponseTemplate(sectionPayloadJson); - - var analysisContent = ScoresheetPrompts.BuildSectionUserPrompt( - applicationContent, - attachmentSummariesText, - sectionPayloadJson, - responseTemplate); - - var systemPrompt = ScoresheetPrompts.SectionSystemPrompt; - - await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent); - var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); - await LogPromptOutputAsync(ScoresheetSectionPromptType, modelOutput); - return modelOutput; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", sectionName); - return "{}"; - } - } - - public async Task GenerateScoresheetSectionAnswersAsync(ScoresheetSectionRequest request) - { - var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); - var sectionJson = JsonSerializer.Serialize(request.SectionSchema, JsonLogOptions); - - var attachmentSummaries = request.Attachments - .Select(a => $"{a.Name}: {a.Summary}") - .ToList(); - - var raw = await GenerateScoresheetSectionAnswersAsync( - dataJson, - attachmentSummaries, - sectionJson, - request.SectionName); - return ParseScoresheetSectionResponse(raw); - } - - private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) - { - var response = new ApplicationAnalysisResponse(); - - if (!TryParseJsonObjectFromResponse(raw, out var root)) - { - return response; - } - - if (TryGetStringProperty(root, AIJsonKeys.Rating, out var rating)) - { - response.Rating = rating; - } - - if (root.TryGetProperty("errors", out var errors) && errors.ValueKind == JsonValueKind.Array) - { - response.Errors = ParseFindings(errors); - } - - if (root.TryGetProperty("warnings", out var warnings) && warnings.ValueKind == JsonValueKind.Array) - { - response.Warnings = ParseFindings(warnings); - } - - if (root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) && summaries.ValueKind == JsonValueKind.Array) - { - response.Summaries = ParseFindings(summaries); - } - - if (root.TryGetProperty(AIJsonKeys.Dismissed, out var dismissed) && dismissed.ValueKind == JsonValueKind.Array) - { - response.Dismissed = dismissed - .EnumerateArray() - .Select(GetStringValueOrNull) - .Where(item => !string.IsNullOrWhiteSpace(item)) - .Cast() - .ToList(); - } - - return response; - } - - private static bool TryGetStringProperty(JsonElement root, string propertyName, out string? value) - { - value = null; - if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) - { - return false; - } - - value = property.GetString(); - return !string.IsNullOrWhiteSpace(value); - } - - private static string? GetStringValueOrNull(JsonElement element) - { - if (element.ValueKind == JsonValueKind.String) - { - return element.GetString(); - } - - return null; - } - - private static List ParseFindings(JsonElement array) - { - var findings = new List(); - foreach (var item in array.EnumerateArray()) - { - if (item.ValueKind != JsonValueKind.Object) - { - continue; - } - - var id = item.TryGetProperty(AIJsonKeys.Id, out var idProp) && idProp.ValueKind == JsonValueKind.String - ? idProp.GetString() - : null; - string? title = null; - if (item.TryGetProperty(AIJsonKeys.Title, out var titleProp) && titleProp.ValueKind == JsonValueKind.String) - { - title = titleProp.GetString(); - } - else if (item.TryGetProperty("category", out var legacyTitleProp) && - legacyTitleProp.ValueKind == JsonValueKind.String) - { - title = legacyTitleProp.GetString(); - } - - string? detail = null; - if (item.TryGetProperty(AIJsonKeys.Detail, out var detailProp) && detailProp.ValueKind == JsonValueKind.String) - { - detail = detailProp.GetString(); - } - else if (item.TryGetProperty("message", out var legacyDetailProp) && - legacyDetailProp.ValueKind == JsonValueKind.String) - { - detail = legacyDetailProp.GetString(); - } - - findings.Add(new ApplicationAnalysisFinding - { - Id = id, - Title = title, - Detail = detail - }); - } - - return findings; - } - - private static ScoresheetSectionResponse ParseScoresheetSectionResponse(string raw) - { - var response = new ScoresheetSectionResponse(); - if (!TryParseJsonObjectFromResponse(raw, out var root)) - { - return response; - } - - foreach (var property in root.EnumerateObject()) - { - if (property.Value.ValueKind != JsonValueKind.Object) - { - continue; - } - - var answer = property.Value.TryGetProperty("answer", out var answerProp) - ? answerProp.Clone() - : default; - var rationale = property.Value.TryGetProperty("rationale", out var rationaleProp) && - rationaleProp.ValueKind == JsonValueKind.String - ? rationaleProp.GetString() ?? string.Empty - : string.Empty; - var confidence = property.Value.TryGetProperty("confidence", out var confidenceProp) && - confidenceProp.ValueKind == JsonValueKind.Number && - confidenceProp.TryGetInt32(out var parsedConfidence) - ? NormalizeConfidence(parsedConfidence) - : 0; - - response.Answers[property.Name] = new ScoresheetSectionAnswer - { - Answer = answer, - Rationale = rationale, - Confidence = confidence - }; - } - - return response; - } - - private static int NormalizeConfidence(int confidence) - { - var clamped = Math.Clamp(confidence, 0, 100); - var rounded = (int)Math.Round(clamped / 5.0, MidpointRounding.AwayFromZero) * 5; - return Math.Clamp(rounded, 0, 100); - } - - private static string BuildScoresheetSectionResponseTemplate(string sectionPayloadJson) - { - try - { - using var doc = JsonDocument.Parse(sectionPayloadJson); - if (!doc.RootElement.TryGetProperty("questions", out var questions) || questions.ValueKind != JsonValueKind.Array) - { - return ScoresheetPrompts.SectionOutputTemplate; - } - - var template = new Dictionary(); - foreach (var question in questions.EnumerateArray()) - { - if (!question.TryGetProperty("id", out var idProp) || idProp.ValueKind != JsonValueKind.String) - { - continue; - } - - var questionId = idProp.GetString(); - if (string.IsNullOrWhiteSpace(questionId)) - { - continue; - } - - template[questionId] = new - { - answer = string.Empty, - rationale = string.Empty, - confidence = 0 - }; - } - - if (template.Count == 0) - { - return ScoresheetPrompts.SectionOutputTemplate; - } - - return JsonSerializer.Serialize(template, JsonLogOptions); - } - catch (JsonException) - { - return ScoresheetPrompts.SectionOutputTemplate; - } - } - - private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt) - { - var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); - _logger.LogInformation("AI {PromptType} input payload: {PromptInput}", promptType, formattedInput); - await WritePromptLogFileAsync(promptType, "INPUT", formattedInput); - } - - private async Task LogPromptOutputAsync(string promptType, string output) - { - var formattedOutput = FormatPromptOutputForLog(output); - _logger.LogInformation("AI {PromptType} model output payload: {ModelOutput}", promptType, formattedOutput); - await WritePromptLogFileAsync(promptType, "OUTPUT", formattedOutput); - } - - private async Task WritePromptLogFileAsync(string promptType, string payloadType, string payload) - { - if (!CanWritePromptFileLog()) - { - return; - } - - try - { - var now = DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss zzz"); - var logDirectory = Path.Combine(AppContext.BaseDirectory, PromptLogDirectoryName); - Directory.CreateDirectory(logDirectory); - - var logPath = Path.Combine(logDirectory, PromptLogFileName); - var entry = $"{now} [{promptType}] {payloadType}\n{payload}\n\n"; - await File.AppendAllTextAsync(logPath, entry); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to write AI prompt log file."); - } - } - - private bool CanWritePromptFileLog() - { - return IsPromptFileLoggingEnabled; - } - - private static string FormatPromptInputForLog(string? systemPrompt, string userPrompt) - { - var normalizedSystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? string.Empty : systemPrompt.Trim(); - var normalizedUserPrompt = string.IsNullOrWhiteSpace(userPrompt) ? string.Empty : userPrompt.Trim(); - return $"SYSTEM_PROMPT\n{normalizedSystemPrompt}\n\nUSER_PROMPT\n{normalizedUserPrompt}"; - } - - private static string FormatPromptOutputForLog(string output) - { - if (string.IsNullOrWhiteSpace(output)) - { - return string.Empty; - } - - if (TryParseJsonObjectFromResponse(output, out var jsonObject)) - { - return JsonSerializer.Serialize(jsonObject, JsonLogOptions); - } - - return output.Trim(); - } - - private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) - { - objectElement = default; - var cleaned = CleanJsonResponse(response); - if (string.IsNullOrWhiteSpace(cleaned)) - { - return false; - } - - try - { - using var doc = JsonDocument.Parse(cleaned); - if (doc.RootElement.ValueKind != JsonValueKind.Object) - { - return false; - } - - objectElement = doc.RootElement.Clone(); - return true; - } - catch (JsonException) - { - return false; - } - } - - private static string CleanJsonResponse(string response) - { - if (string.IsNullOrWhiteSpace(response)) - { - return string.Empty; - } - - var cleaned = response.Trim(); - - if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) - { - var startIndex = cleaned.IndexOf('\n'); - if (startIndex >= 0) - { - // Multi-line fenced code block: remove everything up to and including the first newline. - cleaned = cleaned[(startIndex + 1)..]; - } - else - { - // Single-line fenced JSON, e.g. ```json { ... } ``` or ```{ ... } ```. - // Strip everything before the first likely JSON payload token. - var jsonStart = FindFirstJsonTokenIndex(cleaned); - - if (jsonStart > 0) - { - cleaned = cleaned[jsonStart..]; - } - } - } - - if (cleaned.EndsWith("```", StringComparison.Ordinal)) - { - var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); - if (lastIndex > 0) - { - cleaned = cleaned[..lastIndex]; - } - } - - return cleaned.Trim(); - } - - private static int FindFirstJsonTokenIndex(string value) - { - var objectStart = value.IndexOf('{'); - var arrayStart = value.IndexOf('['); - - if (objectStart >= 0 && arrayStart >= 0) - { - return Math.Min(objectStart, arrayStart); - } - - if (objectStart >= 0) - { - return objectStart; - } - - return arrayStart; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.txt deleted file mode 100644 index e11dce3c9..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.txt +++ /dev/null @@ -1,13 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class PromptCoreRules - { - public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; - public const string NoInvention = "- Do not invent missing details."; - public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; - public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; - public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; - public const string ValidJsonOnly = "- Return valid JSON only."; - public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.txt deleted file mode 100644 index 701a43e74..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.txt +++ /dev/null @@ -1,14 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class PromptHeader - { - public static string Build(string role, string task) - { - return $@"ROLE -{role} - -TASK -{task}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.txt deleted file mode 100644 index 2db4de742..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.txt +++ /dev/null @@ -1,80 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class ScoresheetPrompts - { - public static readonly string SectionSystemPrompt = PromptHeader.Build( - "You are an expert grant application reviewer for the BC Government.", - "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); - - public const string SectionOutputTemplate = @"{ - """": { - ""answer"": """", - ""rationale"": """", - ""confidence"": - } -}"; - - public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" - + "- Do not invent missing application details.\n" - + @"- Return exactly one answer object per question ID in SECTION.questions. -- Do not omit any question IDs from SECTION.questions. -- Do not add keys that are not question IDs from SECTION.questions. -- Use RESPONSE as the output contract and fill every placeholder value. -- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. -- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". -- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. -- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. -- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. -- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. -- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. -- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. -- Do not treat missing or non-contradictory information as evidence. -- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. -- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. -- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". -- For numeric questions, answer must be a numeric value within the allowed range. -- For numeric questions, answer must never be blank. -- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. -- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. -- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. -- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. -- Never return 0 for select list answers unless 0 exists as an explicit option number. -- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. -- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. -- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. -- Do not leave rationale empty when answer is populated. -- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". -- If any answer object is incomplete, regenerate the full JSON response before returning it. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public static string BuildSectionUserPrompt( - string applicationContent, - string attachmentSummariesText, - string sectionPayloadJson, - string responseTemplateJson) - { - return $@"DATA -{applicationContent} - -ATTACHMENTS -- {attachmentSummariesText} - -SECTION -{sectionPayloadJson} - -RESPONSE -{responseTemplateJson} - -OUTPUT -{SectionOutputTemplate} - -RULES -{SectionRules}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptCoreRules.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptCoreRules.cs deleted file mode 100644 index e11dce3c9..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptCoreRules.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class PromptCoreRules - { - public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; - public const string NoInvention = "- Do not invent missing details."; - public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; - public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; - public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; - public const string ValidJsonOnly = "- Return valid JSON only."; - public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptHeader.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptHeader.cs deleted file mode 100644 index 701a43e74..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/PromptHeader.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class PromptHeader - { - public static string Build(string role, string task) - { - return $@"ROLE -{role} - -TASK -{task}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs deleted file mode 100644 index 306e43e36..000000000 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs +++ /dev/null @@ -1,123 +0,0 @@ -namespace Unity.GrantManager.AI -{ - internal static class ScoresheetPrompts - { - public static readonly string AllSystemPromptV0 = PromptHeader.Build( - "You are an expert grant application reviewer for the BC Government.", - "Using DATA, ATTACHMENTS, QUESTIONS, OUTPUT, and RULES, provide answers for all scoresheet questions."); - - public const string AllOutputTemplateV0 = @"{ - """": """" -}"; - - public const string AllRulesV0 = "- Use only DATA and ATTACHMENTS as evidence.\n" - + "- Do not invent missing application details.\n" - + @"- Return exactly one answer per question ID in QUESTIONS. -- Do not omit any question IDs from QUESTIONS. -- Do not add keys that are not question IDs from QUESTIONS. -- The ""answer"" value type must match the question type. -- For numeric questions, return a numeric value within the allowed range. -- For yes/no questions, return exactly ""Yes"" or ""No"". -- For select list questions, return only the selected options.number as a string and never return option label text. -- For text and text area questions, return concise, evidence-based text. -- For text and text area questions, include concise source-grounded rationale from the provided input content. -- If explicit evidence is insufficient, choose the most conservative valid answer. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public static readonly string SectionSystemPrompt = PromptHeader.Build( - "You are an expert grant application reviewer for the BC Government.", - "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); - - public static readonly string SectionSystemPromptV0 = PromptHeader.Build( - "You are an expert grant application reviewer for the BC Government.", - "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); - - public const string SectionOutputTemplate = @"{ - """": { - ""answer"": """", - ""rationale"": """", - ""confidence"": - } -}"; - - public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" - + "- Do not invent missing application details.\n" - + @"- Return exactly one answer object per question ID in SECTION.questions. -- Do not omit any question IDs from SECTION.questions. -- Do not add keys that are not question IDs from SECTION.questions. -- Use RESPONSE as the output contract and fill every placeholder value. -- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. -- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". -- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. -- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. -- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. -- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. -- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. -- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. -- Do not treat missing or non-contradictory information as evidence. -- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. -- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. -- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". -- For numeric questions, answer must be a numeric value within the allowed range. -- For numeric questions, answer must never be blank. -- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. -- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. -- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. -- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. -- Never return 0 for select list answers unless 0 exists as an explicit option number. -- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. -- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. -- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. -- Do not leave rationale empty when answer is populated. -- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". -- If any answer object is incomplete, regenerate the full JSON response before returning it. -" - + PromptCoreRules.MinimumNarrativeWords + "\n" - + PromptCoreRules.ExactOutputShape + "\n" - + PromptCoreRules.NoExtraOutputKeys + "\n" - + PromptCoreRules.ValidJsonOnly + "\n" - + PromptCoreRules.PlainJsonOnly; - - public static string GetSectionSystemPrompt(bool useV0) => useV0 ? SectionSystemPromptV0 : SectionSystemPrompt; - - public static string BuildSectionUserPrompt( - string applicationContent, - string attachmentSummariesText, - string sectionPayloadJson, - string responseTemplateJson) - { - return BuildSectionUserPrompt(applicationContent, attachmentSummariesText, sectionPayloadJson, responseTemplateJson, useV0: false); - } - - public static string BuildSectionUserPrompt( - string applicationContent, - string attachmentSummariesText, - string sectionPayloadJson, - string responseTemplateJson, - bool useV0) - { - return $@"DATA -{applicationContent} - -ATTACHMENTS -- {attachmentSummariesText} - -SECTION -{sectionPayloadJson} - -RESPONSE -{responseTemplateJson} - -OUTPUT -{SectionOutputTemplate} - -RULES -{SectionRules}"; - } - } -} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md new file mode 100644 index 000000000..e8f5b4de7 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Versions/README.md @@ -0,0 +1,54 @@ +# Runtime Prompt Templates + +These files are the source of truth for runtime prompts. +`OpenAIService` resolves templates from: + +- `AI/Prompts/Versions//