diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs index 418c31ebc..9c1ecd11a 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs @@ -22,6 +22,8 @@ public class OpenAIService : IAIService, ITransientDependency private const string AttachmentSummaryPromptType = "AttachmentSummary"; private const string ScoresheetAllPromptType = "ScoresheetAll"; private const string ScoresheetSectionPromptType = "ScoresheetSection"; + private const string PromptVersionV0 = "v0"; + private const string PromptVersionV1 = "v1"; private const string NoSummaryGeneratedMessage = "No summary generated."; private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; @@ -39,6 +41,9 @@ public class OpenAIService : IAIService, ITransientDependency private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true }; + private string SelectedPromptVersion => NormalizePromptVersion(_configuration["Azure:OpenAI:PromptVersion"]); + private bool UseV0Prompts => string.Equals(SelectedPromptVersion, PromptVersionV0, StringComparison.OrdinalIgnoreCase); + public OpenAIService( HttpClient httpClient, IConfiguration configuration, @@ -84,13 +89,10 @@ public async Task GenerateApplicationAnalysisAsync( }) .Cast(); - var analysisContent = AnalysisPrompts.BuildUserPrompt( - schemaJson, - dataJson, - JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), - request.Rubric ?? string.Empty); - - var systemPrompt = AnalysisPrompts.SystemPrompt; + var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); + var rubric = request.Rubric ?? AnalysisPrompts.GetRubric(UseV0Prompts); + var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, UseV0Prompts); + var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts); await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw); @@ -171,11 +173,11 @@ public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] { var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); - var prompt = $@"{AttachmentPrompts.SystemPrompt} + var prompt = $@"{AttachmentPrompts.GetSystemPrompt(UseV0Prompts)} -{AttachmentPrompts.OutputSection} +{AttachmentPrompts.GetOutputSection(UseV0Prompts)} -{AttachmentPrompts.RulesSection}"; +{AttachmentPrompts.GetRulesSection(UseV0Prompts)}"; var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText; if (attachmentText != null) @@ -194,13 +196,13 @@ public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] sizeBytes = fileContent.Length, text = attachmentText }; - var contentToAnalyze = AttachmentPrompts.BuildUserPrompt( - JsonSerializer.Serialize(attachmentPayload, JsonLogOptions)); + var payloadJson = JsonSerializer.Serialize(attachmentPayload, JsonLogOptions); + var contentToAnalyze = AttachmentPrompts.BuildUserPrompt(payloadJson, UseV0Prompts); await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze); var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput); - return modelOutput; + return UseV0Prompts ? ExtractSummaryFromJson(modelOutput) : modelOutput; } catch (Exception ex) { @@ -257,13 +259,14 @@ public async Task AnalyzeApplicationAsync(string applicationContent, Lis .Cast() : Enumerable.Empty(); - var analysisContent = AnalysisPrompts.BuildUserPrompt( - JsonSerializer.Serialize(schemaPayload, JsonLogOptions), - JsonSerializer.Serialize(dataPayload, JsonLogOptions), - JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), - rubric); - - var systemPrompt = AnalysisPrompts.SystemPrompt; + var schemaJson = JsonSerializer.Serialize(schemaPayload, JsonLogOptions); + var dataJson = JsonSerializer.Serialize(dataPayload, JsonLogOptions); + var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions); + var fallbackRubric = string.IsNullOrWhiteSpace(rubric) + ? AnalysisPrompts.GetRubric(UseV0Prompts) + : rubric; + var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, fallbackRubric, UseV0Prompts); + var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts); await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); @@ -446,9 +449,10 @@ public async Task GenerateScoresheetSectionAnswersAsync(string applicati applicationContent, attachmentSummariesText, sectionPayloadJson, - responseTemplate); + responseTemplate, + UseV0Prompts); - var systemPrompt = ScoresheetPrompts.SectionSystemPrompt; + var systemPrompt = ScoresheetPrompts.GetSectionSystemPrompt(UseV0Prompts); await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent); var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); @@ -683,14 +687,14 @@ private static string BuildScoresheetSectionResponseTemplate(string sectionPaylo private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt) { var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); - _logger.LogInformation("AI {PromptType} input payload: {PromptInput}", promptType, formattedInput); + _logger.LogInformation("AI {PromptType} ({PromptVersion}) input payload: {PromptInput}", promptType, SelectedPromptVersion, formattedInput); await WritePromptLogFileAsync(promptType, "INPUT", formattedInput); } private async Task LogPromptOutputAsync(string promptType, string output) { var formattedOutput = FormatPromptOutputForLog(output); - _logger.LogInformation("AI {PromptType} model output payload: {ModelOutput}", promptType, formattedOutput); + _logger.LogInformation("AI {PromptType} ({PromptVersion}) model output payload: {ModelOutput}", promptType, SelectedPromptVersion, formattedOutput); await WritePromptLogFileAsync(promptType, "OUTPUT", formattedOutput); } @@ -829,5 +833,31 @@ private static int FindFirstJsonTokenIndex(string value) return arrayStart; } + + private static string NormalizePromptVersion(string? version) + { + if (string.Equals(version, PromptVersionV0, StringComparison.OrdinalIgnoreCase)) + { + return PromptVersionV0; + } + + return PromptVersionV1; + } + + private static string ExtractSummaryFromJson(string output) + { + if (!TryParseJsonObjectFromResponse(output, out var jsonObject)) + { + return output?.Trim() ?? string.Empty; + } + + if (jsonObject.TryGetProperty(AIJsonKeys.Summary, out var summaryProp) && + summaryProp.ValueKind == JsonValueKind.String) + { + return summaryProp.GetString() ?? string.Empty; + } + + return output?.Trim() ?? string.Empty; + } } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs index 2e5441280..6cb16655c 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs @@ -2,7 +2,13 @@ namespace Unity.GrantManager.AI { internal static class AnalysisPrompts { - public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: + public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. +COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. +FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. +RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. +QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; + + public const string DefaultRubricV0 = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: 1. ELIGIBILITY REQUIREMENTS: - Project must align with program objectives @@ -43,42 +49,81 @@ internal static class AnalysisPrompts MEDIUM: Application has some gaps or weaknesses that require reviewer attention. LOW: Application has significant gaps or risks across key rubric areas."; + public const string OutputTemplate = @"{ + ""rating"": """", + ""errors"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""warnings"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""summaries"": [ + { + ""title"": """", + ""detail"": """" + } + ] +}"; + + public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n" + + "- Do not invent fields, documents, requirements, or facts.\n" + + @"- Treat missing or empty values as findings only when they weaken rubric evidence. +- Prefer material issues; avoid nitpicking. +- Use 3-6 words for title. +- Each detail must be 1-2 complete sentences. +- Each detail must cite concrete evidence from DATA or ATTACHMENTS. +- If ATTACHMENTS evidence is used, cite the attachment by name in detail. +- If no findings exist, return empty arrays. +- Rating must be HIGH, MEDIUM, or LOW. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved. WARNING: Issue that could negatively affect the application's approval. RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; - public const string OutputTemplate = @"{ + public const string OutputTemplateV0 = @"{ ""rating"": ""HIGH/MEDIUM/LOW"", ""warnings"": [ { - ""title"": ""Brief summary of the warning"", - ""detail"": ""Detailed warning message with full context and explanation"" + ""category"": ""Brief summary of the warning"", + ""message"": ""Detailed warning message with full context and explanation"" } ], ""errors"": [ { - ""title"": ""Brief summary of the error"", - ""detail"": ""Detailed error message with full context and explanation"" + ""category"": ""Brief summary of the error"", + ""message"": ""Detailed error message with full context and explanation"" } ], ""summaries"": [ { - ""title"": ""Brief summary of the recommendation"", - ""detail"": ""Detailed recommendation with specific actionable guidance"" + ""category"": ""Brief summary of the recommendation"", + ""message"": ""Detailed recommendation with specific actionable guidance"" } ], ""dismissed"": [] }"; - public const string Rules = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. + public const string RulesV0 = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. - Do not invent fields, documents, requirements, or facts. - Treat missing or empty values as findings only when they weaken rubric evidence. - Prefer material issues; avoid nitpicking. - Each error/warning/recommendation must describe one concrete issue or consideration and why it matters. -- Use 3-6 words for title. -- Each detail must be 1-2 complete sentences. -- Each detail must be grounded in concrete evidence from provided inputs. -- If attachment evidence is used, reference the attachment explicitly in detail. +- Use 3-6 words for category. +- Each message must be 1-2 complete sentences. +- Each message must be grounded in concrete evidence from provided inputs. +- If attachment evidence is used, reference the attachment explicitly in the message. - Do not provide applicant-facing advice. - Do not mention rubric section names in findings. - If no findings exist, return empty arrays. @@ -89,15 +134,39 @@ internal static class AnalysisPrompts + "\n" + PromptCoreRules.PlainJsonOnly; public static readonly string SystemPrompt = PromptHeader.Build( + "You are an expert grant analyst assistant for human reviewers.", + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); + + public static readonly string SystemPromptV0 = PromptHeader.Build( "You are an expert grant analyst assistant for human reviewers.", "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); + public static string GetRubric(bool useV0) => useV0 ? DefaultRubricV0 : DefaultRubric; + public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; + public static string BuildUserPrompt( string schemaJson, string dataJson, string attachmentsJson, string rubric) { + return BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, useV0: false); + } + + public static string BuildUserPrompt( + string schemaJson, + string dataJson, + string attachmentsJson, + string rubric, + bool useV0) + { + var output = useV0 ? OutputTemplateV0 : OutputTemplate; + var rules = useV0 ? RulesV0 : Rules; + var severitySection = useV0 ? $@"SEVERITY +{SeverityRules} + +" : string.Empty; + return $@"SCHEMA {schemaJson} @@ -110,17 +179,14 @@ public static string BuildUserPrompt( RUBRIC {rubric} -SEVERITY -{SeverityRules} - -SCORE +{severitySection}SCORE {ScoreRules} OUTPUT -{OutputTemplate} +{output} RULES -{Rules}"; +{rules}"; } } } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs index 969480ea8..6e83ea6a1 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs @@ -6,11 +6,34 @@ internal static class AttachmentPrompts "You are a professional grant analyst for the BC Government.", "Produce a concise reviewer-facing summary of the provided attachment context."); + public static readonly string SystemPromptV0 = PromptHeader.Build( + "You are a professional grant analyst for the BC Government.", + "Produce a concise reviewer-facing summary of the provided attachment context."); + public const string OutputSection = @"OUTPUT +{ + ""summary"": """" +}"; + + public const string RulesSection = "- Use only ATTACHMENT as evidence.\n" + + "- If ATTACHMENT.text is present, summarize actual content.\n" + + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" + + PromptCoreRules.NoInvention + "\n" + + @"- Write 1-2 complete sentences. +- Summary must be grounded in concrete ATTACHMENT evidence. +- Return exactly one object with only the key: summary. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public const string OutputSectionV0 = @"OUTPUT - Plain text only - 1-2 complete sentences"; - public const string RulesSection = @"RULES + public const string RulesSectionV0 = @"RULES - Use only the provided attachment context as evidence. - If text content is present, summarize the actual content. - If text content is missing or empty, provide a conservative metadata-based summary. @@ -18,7 +41,16 @@ internal static class AttachmentPrompts - Keep the summary specific, concrete, and reviewer-facing. - Return plain text only (no markdown, bullets, or JSON)."; + public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt; + public static string GetOutputSection(bool useV0) => useV0 ? OutputSectionV0 : OutputSection; + public static string GetRulesSection(bool useV0) => useV0 ? RulesSectionV0 : RulesSection; + public static string BuildUserPrompt(string attachmentPayloadJson) + { + return BuildUserPrompt(attachmentPayloadJson, useV0: false); + } + + public static string BuildUserPrompt(string attachmentPayloadJson, bool useV0) { return $@"ATTACHMENT {attachmentPayloadJson}"; diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md new file mode 100644 index 000000000..4106b7aa0 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md @@ -0,0 +1,8 @@ +# Prompt Baselines + +- `v0`: legacy prompt/service snapshots used as historical baseline. +- `v1`: current runtime prompt/service snapshots. + +Versioning convention: +- Folder name is the baseline version. +- Filenames inside each folder are normalized and do not include version suffixes. diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt new file mode 100644 index 000000000..d267a1216 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AnalysisPrompts.txt @@ -0,0 +1,58 @@ +namespace Unity.GrantManager.AI +{ + internal static class AnalysisPrompts + { + public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic. +COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear. +FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified. +RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks. +QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability."; + + public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. +MEDIUM: Application has some gaps or weaknesses that require reviewer attention. +LOW: Application has significant gaps or risks across key rubric areas."; + + public const string OutputTemplate = @"{ + ""rating"": """", + ""errors"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""warnings"": [ + { + ""title"": """", + ""detail"": """" + } + ], + ""summaries"": [ + { + ""title"": """", + ""detail"": """" + } + ] +}"; + + public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n" + + "- Do not invent fields, documents, requirements, or facts.\n" + + @"- Treat missing or empty values as findings only when they weaken rubric evidence. +- Prefer material issues; avoid nitpicking. +- Use 3-6 words for title. +- Each detail must be 1-2 complete sentences. +- Each detail must cite concrete evidence from DATA or ATTACHMENTS. +- If ATTACHMENTS evidence is used, cite the attachment by name in detail. +- If no findings exist, return empty arrays. +- Rating must be HIGH, MEDIUM, or LOW. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static readonly string SystemPrompt = PromptHeader.Build( + "You are an expert grant analyst assistant for human reviewers.", + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings."); + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt new file mode 100644 index 000000000..a61cc5084 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/AttachmentPrompts.txt @@ -0,0 +1,29 @@ +namespace Unity.GrantManager.AI +{ + internal static class AttachmentPrompts + { + public static readonly string SystemPrompt = PromptHeader.Build( + "You are a professional grant analyst for the BC Government.", + "Produce a concise reviewer-facing summary of the provided attachment context."); + + public const string OutputSection = @"OUTPUT +{ + ""summary"": """" +}"; + + public const string RulesSection = "- Use only ATTACHMENT as evidence.\n" + + "- If ATTACHMENT.text is present, summarize actual content.\n" + + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n" + + PromptCoreRules.NoInvention + "\n" + + @"- Write 1-2 complete sentences. +- Summary must be grounded in concrete ATTACHMENT evidence. +- Return exactly one object with only the key: summary. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + } +} + diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt new file mode 100644 index 000000000..239db1062 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/OpenAIService.cs.txt @@ -0,0 +1,406 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Volo.Abp.DependencyInjection; + +namespace Unity.GrantManager.AI +{ + public class OpenAIService : IAIService, ITransientDependency + { + private readonly HttpClient _httpClient; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly ITextExtractionService _textExtractionService; + + private string? ApiKey => _configuration["AI:OpenAI:ApiKey"]; + private string? ApiUrl => _configuration["AI:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; + private readonly string NoKeyError = "OpenAI API key is not configured"; + + public OpenAIService(HttpClient httpClient, IConfiguration configuration, ILogger logger, ITextExtractionService textExtractionService) + { + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + _textExtractionService = textExtractionService; + } + + public Task IsAvailableAsync() + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", NoKeyError); + return Task.FromResult(false); + } + + return Task.FromResult(true); + } + + public async Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", NoKeyError); + return "AI analysis not available - service not configured."; + } + + _logger.LogDebug("Calling OpenAI with prompt: {Prompt}", content); + + try + { + var systemPrompt = prompt ?? "You are a professional grant analyst for the BC Government."; + + var requestBody = new + { + messages = new[] + { + new { role = "system", content = systemPrompt }, + new { role = "user", content = content } + }, + max_tokens = maxTokens, + temperature = 0.3 + }; + + var json = JsonSerializer.Serialize(requestBody); + var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); + + _httpClient.DefaultRequestHeaders.Clear(); + _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); + + var response = await _httpClient.PostAsync(ApiUrl, httpContent); + var responseContent = await response.Content.ReadAsStringAsync(); + + _logger.LogDebug("Response: {Response}", responseContent); + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); + return "AI analysis failed - service temporarily unavailable."; + } + + using var jsonDoc = JsonDocument.Parse(responseContent); + var choices = jsonDoc.RootElement.GetProperty("choices"); + if (choices.GetArrayLength() > 0) + { + var message = choices[0].GetProperty("message"); + return message.GetProperty("content").GetString() ?? "No summary generated."; + } + + return "No summary generated."; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating AI summary"); + return "AI analysis failed - please try again later."; + } + } + + public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] fileContent, string contentType) + { + try + { + var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); + + string contentToAnalyze; + string prompt; + + if (!string.IsNullOrWhiteSpace(extractedText)) + { + _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText.Length, fileName); + + contentToAnalyze = $"Document: {fileName}\nType: {contentType}\nContent:\n{extractedText}"; + prompt = "Please analyze this document and provide a concise summary of its content, purpose, and key information, for use by your fellow grant analysts. It should be 1-2 sentences long and about 46 tokens."; + } + else + { + _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", fileName); + + contentToAnalyze = $"File: {fileName}, Type: {contentType}, Size: {fileContent.Length} bytes"; + prompt = "Please analyze this document and provide a concise summary of its content, purpose, and key information, for use by your fellow grant analysts. It should be 1-2 sentences long and about 46 tokens."; + } + + return await GenerateSummaryAsync(contentToAnalyze, prompt, 150); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); + return $"AI analysis not available for this attachment ({fileName})."; + } + } + + public async Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", NoKeyError); + return "AI analysis not available - service not configured."; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var fieldConfigurationSection = !string.IsNullOrEmpty(formFieldConfiguration) + ? $@" +{formFieldConfiguration}" + : string.Empty; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} +{fieldConfigurationSection} + +EVALUATION RUBRIC: +{rubric} + +Analyze this grant application comprehensively across all five rubric categories (Eligibility, Completeness, Financial Review, Risk Assessment, and Quality Indicators). Identify issues, concerns, and areas for improvement. Return your findings in the following JSON format: +{{ + ""overall_score"": ""HIGH/MEDIUM/LOW"", + ""warnings"": [ + {{ + ""category"": ""Brief summary of the warning"", + ""message"": ""Detailed warning message with full context and explanation"", + ""severity"": ""WARNING"" + }} + ], + ""errors"": [ + {{ + ""category"": ""Brief summary of the error"", + ""message"": ""Detailed error message with full context and explanation"", + ""severity"": ""ERROR"" + }} + ], + ""recommendations"": [ + {{ + ""category"": ""Brief summary of the recommendation"", + ""message"": ""Detailed recommendation with specific actionable guidance"" + }} + ] +}} + +Important: The 'category' field should be a concise summary (3-6 words) that captures the essence of the issue, while the 'message' field should contain the detailed explanation."; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. + +Conduct a thorough, comprehensive analysis across all rubric categories. Identify substantive issues, concerns, and opportunities for improvement. + +Classify findings based on their impact on the application's evaluation and fundability: +- ERRORS: Important missing information, significant gaps in required content, compliance issues, or major concerns affecting eligibility +- WARNINGS: Areas needing clarification, moderate issues, or concerns that should be addressed + +Evaluate the quality, clarity, and appropriateness of all application content. Be thorough but fair - identify real issues while avoiding nitpicking. + +Respond only with valid JSON in the exact format requested."; + + var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + + // Post-process the AI response to add unique IDs to errors and warnings + return AddIdsToAnalysisItems(rawAnalysis); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error analyzing application"); + return "AI analysis failed - please try again later."; + } + } + + private string AddIdsToAnalysisItems(string analysisJson) + { + try + { + using var jsonDoc = JsonDocument.Parse(analysisJson); + using var memoryStream = new System.IO.MemoryStream(); + using (var writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions { Indented = true })) + { + writer.WriteStartObject(); + + foreach (var property in jsonDoc.RootElement.EnumerateObject()) + { + if (property.Name == "errors" || property.Name == "warnings") + { + writer.WritePropertyName(property.Name); + writer.WriteStartArray(); + + foreach (var item in property.Value.EnumerateArray()) + { + writer.WriteStartObject(); + + // Add unique ID first + writer.WriteString("id", Guid.NewGuid().ToString()); + + // Copy existing properties + foreach (var itemProperty in item.EnumerateObject()) + { + itemProperty.WriteTo(writer); + } + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + else + { + property.WriteTo(writer); + } + } + + // Add dismissed_items array if not present + if (!jsonDoc.RootElement.TryGetProperty("dismissed_items", out _)) + { + writer.WritePropertyName("dismissed_items"); + writer.WriteStartArray(); + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + + return Encoding.UTF8.GetString(memoryStream.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error adding IDs to analysis items, returning original JSON"); + return analysisJson; // Return original if processing fails + } + } + + public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", NoKeyError); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} + +SCORESHEET QUESTIONS: +{scoresheetQuestions} + +Please analyze this grant application and provide appropriate answers for each scoresheet question. + +For numeric questions, provide a numeric value within the specified range. +For yes/no questions, provide either 'Yes' or 'No'. +For text questions, provide a concise, relevant response. +For select list questions, choose the most appropriate option from the provided choices. +For text area questions, provide a detailed but concise response. + +Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. + +Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: +{{ + ""question-id-1"": ""answer-value-1"", + ""question-id-2"": ""answer-value-2"" +}} +Do not return any markdown formatting, just the JSON by itself"; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. +Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. +Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. +Respond only with valid JSON in the exact format requested."; + + return await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet answers"); + return "{}"; + } + } + + public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", NoKeyError); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} + +SCORESHEET SECTION: {sectionName} +{sectionJson} + +Please analyze this grant application and provide appropriate answers for each question in the ""{sectionName}"" section only. + +For each question, provide: +1. Your answer based on the application content +2. A brief cited description (1-2 sentences) explaining your reasoning with specific references to the application content +3. A confidence score from 0-100 indicating how confident you are in your answer based on available information + +Guidelines for answers: +- For numeric questions, provide a numeric value within the specified range +- For yes/no questions, provide either 'Yes' or 'No' +- For text questions, provide a concise, relevant response +- For select list questions, respond with ONLY the number from the 'number' field (1, 2, 3, etc.) of your chosen option. NEVER return 0 - the lowest valid answer is 1. For example: if you want '(0 pts) No outcomes provided', choose the option where number=1, not 0. +- For text area questions, provide a detailed but concise response +- Base your confidence score on how clearly the application content supports your answer + +Return your response as a JSON object where each key is the question ID and the value contains the answer, citation, and confidence: +{{ + ""question-id-1"": {{ + ""answer"": ""your-answer-here"", + ""citation"": ""Brief explanation with specific reference to application content"", + ""confidence"": 85 + }}, + ""question-id-2"": {{ + ""answer"": ""3"", + ""citation"": ""Based on the project budget of $50,000 mentioned in the application, this falls into the medium budget category"", + ""confidence"": 90 + }} +}} + +IMPORTANT FOR SELECT LIST QUESTIONS: If a question has availableOptions like: +[{{""number"":1,""value"":""Low (Under $25K)""}}, {{""number"":2,""value"":""Medium ($25K-$75K)""}}, {{""number"":3,""value"":""High (Over $75K)""}}] +Then respond with ONLY the number (e.g., ""3"" for ""High (Over $75K)""), not the text value. + +Do not return any markdown formatting, just the JSON by itself"; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. +Analyze the provided application and generate appropriate answers for the scoresheet section questions based on the application content. +Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. +Always provide citations that reference specific parts of the application content to support your answers. +Be honest about your confidence level - if information is missing or unclear, reflect this in a lower confidence score. +Respond only with valid JSON in the exact format requested."; + + return await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", sectionName); + return "{}"; + } + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt new file mode 100644 index 000000000..e11dce3c9 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptCoreRules.txt @@ -0,0 +1,13 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptCoreRules + { + public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; + public const string NoInvention = "- Do not invent missing details."; + public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; + public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; + public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; + public const string ValidJsonOnly = "- Return valid JSON only."; + public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt new file mode 100644 index 000000000..701a43e74 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/PromptHeader.txt @@ -0,0 +1,14 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptHeader + { + public static string Build(string role, string task) + { + return $@"ROLE +{role} + +TASK +{task}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt new file mode 100644 index 000000000..bfe883d64 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v0/ScoresheetPrompts.txt @@ -0,0 +1,85 @@ +namespace Unity.GrantManager.AI +{ + internal static class ScoresheetPrompts + { + public static readonly string AllSystemPrompt = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, QUESTIONS, OUTPUT, and RULES, provide answers for all scoresheet questions."); + + public const string AllOutputTemplate = @"{ + """": """" +}"; + + public const string AllRules = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer per question ID in QUESTIONS. +- Do not omit any question IDs from QUESTIONS. +- Do not add keys that are not question IDs from QUESTIONS. +- The ""answer"" value type must match the question type. +- For numeric questions, return a numeric value within the allowed range. +- For yes/no questions, return exactly ""Yes"" or ""No"". +- For select list questions, return only the selected options.number as a string and never return option label text. +- For text and text area questions, return concise, evidence-based text. +- For text and text area questions, include concise source-grounded rationale from the provided input content. +- If explicit evidence is insufficient, choose the most conservative valid answer. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static readonly string SectionSystemPrompt = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + + public const string SectionOutputTemplate = @"{ + """": { + ""answer"": """", + ""rationale"": """", + ""confidence"": + } +}"; + + public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer object per question ID in SECTION.questions. +- Do not omit any question IDs from SECTION.questions. +- Do not add keys that are not question IDs from SECTION.questions. +- Use RESPONSE as the output contract and fill every placeholder value. +- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. +- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". +- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. +- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. +- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. +- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. +- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. +- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. +- Do not treat missing or non-contradictory information as evidence. +- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. +- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. +- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". +- For numeric questions, answer must be a numeric value within the allowed range. +- For numeric questions, answer must never be blank. +- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. +- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. +- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. +- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. +- Never return 0 for select list answers unless 0 exists as an explicit option number. +- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. +- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. +- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. +- Do not leave rationale empty when answer is populated. +- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". +- If any answer object is incomplete, regenerate the full JSON response before returning it. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + } +} + + + diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt new file mode 100644 index 000000000..2e5441280 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AnalysisPrompts.cs.txt @@ -0,0 +1,126 @@ +namespace Unity.GrantManager.AI +{ + internal static class AnalysisPrompts + { + public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC: + +1. ELIGIBILITY REQUIREMENTS: + - Project must align with program objectives + - Applicant must be eligible entity type + - Budget must be reasonable and well-justified + - Project timeline must be realistic + +2. COMPLETENESS CHECKS: + - All required fields completed + - Necessary supporting documents provided + - Budget breakdown detailed and accurate + - Project description clear and comprehensive + +3. FINANCIAL REVIEW: + - Requested amount is within program limits + - Budget is reasonable for scope of work + - Matching funds or in-kind contributions identified + - Cost per outcome/beneficiary is reasonable + +4. RISK ASSESSMENT: + - Applicant capacity to deliver project + - Technical feasibility of proposed work + - Environmental or regulatory compliance + - Potential for cost overruns or delays + +5. QUALITY INDICATORS: + - Clear project objectives and outcomes + - Well-defined target audience/beneficiaries + - Appropriate project methodology + - Sustainability plan for long-term impact + +EVALUATION CRITERIA: +- HIGH: Meets all requirements, well-prepared application, low risk +- MEDIUM: Meets most requirements, minor issues or missing elements +- LOW: Missing key requirements, significant concerns, high risk"; + + public const string ScoreRules = @"HIGH: Application demonstrates strong evidence across most rubric areas with few or no issues. +MEDIUM: Application has some gaps or weaknesses that require reviewer attention. +LOW: Application has significant gaps or risks across key rubric areas."; + + public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved. +WARNING: Issue that could negatively affect the application's approval. +RECOMMENDATION: Reviewer-facing improvement or follow-up consideration."; + + public const string OutputTemplate = @"{ + ""rating"": ""HIGH/MEDIUM/LOW"", + ""warnings"": [ + { + ""title"": ""Brief summary of the warning"", + ""detail"": ""Detailed warning message with full context and explanation"" + } + ], + ""errors"": [ + { + ""title"": ""Brief summary of the error"", + ""detail"": ""Detailed error message with full context and explanation"" + } + ], + ""summaries"": [ + { + ""title"": ""Brief summary of the recommendation"", + ""detail"": ""Detailed recommendation with specific actionable guidance"" + } + ], + ""dismissed"": [] +}"; + + public const string Rules = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence. +- Do not invent fields, documents, requirements, or facts. +- Treat missing or empty values as findings only when they weaken rubric evidence. +- Prefer material issues; avoid nitpicking. +- Each error/warning/recommendation must describe one concrete issue or consideration and why it matters. +- Use 3-6 words for title. +- Each detail must be 1-2 complete sentences. +- Each detail must be grounded in concrete evidence from provided inputs. +- If attachment evidence is used, reference the attachment explicitly in detail. +- Do not provide applicant-facing advice. +- Do not mention rubric section names in findings. +- If no findings exist, return empty arrays. +- rating must be HIGH, MEDIUM, or LOW." + + "\n" + PromptCoreRules.ExactOutputShape + + "\n" + PromptCoreRules.NoExtraOutputKeys + + "\n" + PromptCoreRules.ValidJsonOnly + + "\n" + PromptCoreRules.PlainJsonOnly; + + public static readonly string SystemPrompt = PromptHeader.Build( + "You are an expert grant analyst assistant for human reviewers.", + "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings."); + + public static string BuildUserPrompt( + string schemaJson, + string dataJson, + string attachmentsJson, + string rubric) + { + return $@"SCHEMA +{schemaJson} + +DATA +{dataJson} + +ATTACHMENTS +{attachmentsJson} + +RUBRIC +{rubric} + +SEVERITY +{SeverityRules} + +SCORE +{ScoreRules} + +OUTPUT +{OutputTemplate} + +RULES +{Rules}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt new file mode 100644 index 000000000..969480ea8 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/AttachmentPrompts.cs.txt @@ -0,0 +1,27 @@ +namespace Unity.GrantManager.AI +{ + internal static class AttachmentPrompts + { + public static readonly string SystemPrompt = PromptHeader.Build( + "You are a professional grant analyst for the BC Government.", + "Produce a concise reviewer-facing summary of the provided attachment context."); + + public const string OutputSection = @"OUTPUT +- Plain text only +- 1-2 complete sentences"; + + public const string RulesSection = @"RULES +- Use only the provided attachment context as evidence. +- If text content is present, summarize the actual content. +- If text content is missing or empty, provide a conservative metadata-based summary. +- Do not invent missing details. +- Keep the summary specific, concrete, and reviewer-facing. +- Return plain text only (no markdown, bullets, or JSON)."; + + public static string BuildUserPrompt(string attachmentPayloadJson) + { + return $@"ATTACHMENT +{attachmentPayloadJson}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt new file mode 100644 index 000000000..418c31ebc --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/OpenAIService.cs.txt @@ -0,0 +1,833 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Volo.Abp.DependencyInjection; + +namespace Unity.GrantManager.AI +{ + public class OpenAIService : IAIService, ITransientDependency + { + private readonly HttpClient _httpClient; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly ITextExtractionService _textExtractionService; + private const string ApplicationAnalysisPromptType = "ApplicationAnalysis"; + private const string AttachmentSummaryPromptType = "AttachmentSummary"; + private const string ScoresheetAllPromptType = "ScoresheetAll"; + private const string ScoresheetSectionPromptType = "ScoresheetSection"; + private const string NoSummaryGeneratedMessage = "No summary generated."; + private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured."; + private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable."; + private const string SummaryFailedRetryMessage = "AI analysis failed - please try again later."; + + private string? ApiKey => _configuration["Azure:OpenAI:ApiKey"]; + private string? ApiUrl => _configuration["Azure:OpenAI:ApiUrl"] ?? "https://api.openai.com/v1/chat/completions"; + private readonly string MissingApiKeyMessage = "OpenAI API key is not configured"; + + // Optional local debugging sink for prompt payload logs to a local file. + // Not intended for deployed/shared environments. + private bool IsPromptFileLoggingEnabled => _configuration.GetValue("Azure:Logging:EnablePromptFileLog") ?? false; + private const string PromptLogDirectoryName = "logs"; + private static readonly string PromptLogFileName = $"ai-prompts-{DateTime.UtcNow:yyyyMMdd-HHmmss}-{Environment.ProcessId}.log"; + + private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true }; + + public OpenAIService( + HttpClient httpClient, + IConfiguration configuration, + ILogger logger, + ITextExtractionService textExtractionService) + { + _httpClient = httpClient; + _configuration = configuration; + _logger = logger; + _textExtractionService = textExtractionService; + } + + public Task IsAvailableAsync() + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); + return Task.FromResult(false); + } + + return Task.FromResult(true); + } + + public async Task GenerateCompletionAsync(AICompletionRequest request) + { + var content = await GenerateSummaryAsync( + request?.UserPrompt ?? string.Empty, + request?.SystemPrompt, + request?.MaxTokens ?? 150); + return new AICompletionResponse { Content = content }; + } + + public async Task GenerateApplicationAnalysisAsync(ApplicationAnalysisRequest request) + { + var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); + var schemaJson = JsonSerializer.Serialize(request.Schema, JsonLogOptions); + + var attachmentsPayload = request.Attachments + .Select(a => new + { + name = string.IsNullOrWhiteSpace(a.Name) ? "attachment" : a.Name.Trim(), + summary = string.IsNullOrWhiteSpace(a.Summary) ? string.Empty : a.Summary.Trim() + }) + .Cast(); + + var analysisContent = AnalysisPrompts.BuildUserPrompt( + schemaJson, + dataJson, + JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), + request.Rubric ?? string.Empty); + + var systemPrompt = AnalysisPrompts.SystemPrompt; + await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); + var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw); + return ParseApplicationAnalysisResponse(AddIdsToAnalysisItems(raw)); + } + + public async Task GenerateSummaryAsync(string content, string? prompt = null, int maxTokens = 150) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("Error: {Message}", MissingApiKeyMessage); + return ServiceNotConfiguredMessage; + } + + _logger.LogDebug("Calling OpenAI chat completions. PromptLength: {PromptLength}, MaxTokens: {MaxTokens}", content?.Length ?? 0, maxTokens); + + try + { + var systemPrompt = prompt ?? "You are a professional grant analyst for the BC Government."; + var userPrompt = content ?? string.Empty; + + var requestBody = new + { + messages = new[] + { + new { role = "system", content = systemPrompt }, + new { role = "user", content = userPrompt } + }, + max_tokens = maxTokens, + temperature = 0.3 + }; + + var json = JsonSerializer.Serialize(requestBody); + var httpContent = new StringContent(json, Encoding.UTF8, "application/json"); + + _httpClient.DefaultRequestHeaders.Clear(); + _httpClient.DefaultRequestHeaders.Add("Authorization", ApiKey); + + var response = await _httpClient.PostAsync(ApiUrl, httpContent); + var responseContent = await response.Content.ReadAsStringAsync(); + + _logger.LogDebug( + "OpenAI chat completions response received. StatusCode: {StatusCode}, ResponseLength: {ResponseLength}", + response.StatusCode, + responseContent?.Length ?? 0); + + if (!response.IsSuccessStatusCode) + { + _logger.LogError("OpenAI API request failed: {StatusCode} - {Content}", response.StatusCode, responseContent); + return ServiceTemporarilyUnavailableMessage; + } + + if (string.IsNullOrWhiteSpace(responseContent)) + { + return NoSummaryGeneratedMessage; + } + + using var jsonDoc = JsonDocument.Parse(responseContent); + var choices = jsonDoc.RootElement.GetProperty("choices"); + if (choices.GetArrayLength() > 0) + { + var message = choices[0].GetProperty("message"); + return message.GetProperty("content").GetString() ?? NoSummaryGeneratedMessage; + } + + return NoSummaryGeneratedMessage; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating AI summary"); + return SummaryFailedRetryMessage; + } + } + + public async Task GenerateAttachmentSummaryAsync(string fileName, byte[] fileContent, string contentType) + { + try + { + var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType); + + var prompt = $@"{AttachmentPrompts.SystemPrompt} + +{AttachmentPrompts.OutputSection} + +{AttachmentPrompts.RulesSection}"; + + var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText; + if (attachmentText != null) + { + _logger.LogDebug("Extracted {TextLength} characters from {FileName}", extractedText.Length, fileName); + } + else + { + _logger.LogDebug("No text extracted from {FileName}, analyzing metadata only", fileName); + } + + var attachmentPayload = new + { + name = fileName, + contentType, + sizeBytes = fileContent.Length, + text = attachmentText + }; + var contentToAnalyze = AttachmentPrompts.BuildUserPrompt( + JsonSerializer.Serialize(attachmentPayload, JsonLogOptions)); + + await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze); + var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150); + await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput); + return modelOutput; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating attachment summary for {FileName}", fileName); + return $"AI analysis not available for this attachment ({fileName})."; + } + } + + public async Task GenerateAttachmentSummaryAsync(AttachmentSummaryRequest request) + { + var summary = await GenerateAttachmentSummaryAsync( + request?.FileName ?? string.Empty, + request?.FileContent ?? Array.Empty(), + request?.ContentType ?? "application/octet-stream"); + return new AttachmentSummaryResponse { Summary = summary }; + } + + public async Task AnalyzeApplicationAsync(string applicationContent, List attachmentSummaries, string rubric, string? formFieldConfiguration = null) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", MissingApiKeyMessage); + return ServiceNotConfiguredMessage; + } + + try + { + object schemaPayload = new { }; + if (!string.IsNullOrWhiteSpace(formFieldConfiguration)) + { + try + { + using var schemaDoc = JsonDocument.Parse(formFieldConfiguration); + schemaPayload = schemaDoc.RootElement.Clone(); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Invalid form field configuration JSON. Using empty schema payload."); + } + } + + var dataPayload = new + { + applicationContent + }; + + var attachmentsPayload = attachmentSummaries?.Count > 0 + ? attachmentSummaries + .Select((summary, index) => new + { + name = $"Attachment {index + 1}", + summary = summary + }) + .Cast() + : Enumerable.Empty(); + + var analysisContent = AnalysisPrompts.BuildUserPrompt( + JsonSerializer.Serialize(schemaPayload, JsonLogOptions), + JsonSerializer.Serialize(dataPayload, JsonLogOptions), + JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions), + rubric); + + var systemPrompt = AnalysisPrompts.SystemPrompt; + + await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent); + var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000); + await LogPromptOutputAsync(ApplicationAnalysisPromptType, rawAnalysis); + + // Post-process the AI response to add unique IDs to errors and warnings + return AddIdsToAnalysisItems(rawAnalysis); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error analyzing application"); + return SummaryFailedRetryMessage; + } + } + + private string AddIdsToAnalysisItems(string analysisJson) + { + try + { + using var jsonDoc = JsonDocument.Parse(analysisJson); + using var memoryStream = new System.IO.MemoryStream(); + using (var writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions { Indented = true })) + { + writer.WriteStartObject(); + + foreach (var property in jsonDoc.RootElement.EnumerateObject()) + { + var outputPropertyName = property.Name; + + if (outputPropertyName == AIJsonKeys.Errors || outputPropertyName == AIJsonKeys.Warnings) + { + writer.WritePropertyName(outputPropertyName); + writer.WriteStartArray(); + + foreach (var item in property.Value.EnumerateArray()) + { + writer.WriteStartObject(); + + // Add unique ID first + writer.WriteString("id", Guid.NewGuid().ToString()); + + // Copy existing properties + foreach (var itemProperty in item.EnumerateObject()) + { + itemProperty.WriteTo(writer); + } + + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + else + { + if (outputPropertyName != property.Name) + { + writer.WritePropertyName(outputPropertyName); + property.Value.WriteTo(writer); + continue; + } + + property.WriteTo(writer); + } + } + + // Add dismissed array if not present. + if (!jsonDoc.RootElement.TryGetProperty(AIJsonKeys.Dismissed, out _)) + { + writer.WritePropertyName(AIJsonKeys.Dismissed); + writer.WriteStartArray(); + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } + + return Encoding.UTF8.GetString(memoryStream.ToArray()); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error adding IDs to analysis items, returning original JSON"); + return analysisJson; // Return original if processing fails + } + } + + public async Task GenerateScoresheetAnswersAsync(string applicationContent, List attachmentSummaries, string scoresheetQuestions) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", MissingApiKeyMessage); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + var analysisContent = $@"APPLICATION CONTENT: +{applicationContent} + +ATTACHMENT SUMMARIES: +- {attachmentSummariesText} + +SCORESHEET QUESTIONS: +{scoresheetQuestions} + +Please analyze this grant application and provide appropriate answers for each scoresheet question. + +For numeric questions, provide a numeric value within the specified range. +For yes/no questions, provide either 'Yes' or 'No'. +For text questions, provide a concise, relevant response. +For select list questions, choose the most appropriate option from the provided choices. +For text area questions, provide a detailed but concise response. + +Base your answers on the application content and attachment summaries provided. Be objective and fair in your assessment. + +Return your response as a JSON object where each key is the question ID and the value is the appropriate answer: +{{ + ""question-id-1"": ""answer-value-1"", + ""question-id-2"": ""answer-value-2"" +}} +Do not return any markdown formatting, just the JSON by itself"; + + var systemPrompt = @"You are an expert grant application reviewer for the BC Government. +Analyze the provided application and generate appropriate answers for the scoresheet questions based on the application content. +Be thorough, objective, and fair in your assessment. Base your answers strictly on the provided application content. +Respond only with valid JSON in the exact format requested."; + + await LogPromptInputAsync(ScoresheetAllPromptType, systemPrompt, analysisContent); + var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + await LogPromptOutputAsync(ScoresheetAllPromptType, modelOutput); + return modelOutput; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet answers"); + return "{}"; + } + } + + public async Task GenerateScoresheetSectionAnswersAsync(string applicationContent, List attachmentSummaries, string sectionJson, string sectionName) + { + if (string.IsNullOrEmpty(ApiKey)) + { + _logger.LogWarning("{Message}", MissingApiKeyMessage); + return "{}"; + } + + try + { + var attachmentSummariesText = attachmentSummaries?.Count > 0 + ? string.Join("\n- ", attachmentSummaries.Select((s, i) => $"Attachment {i + 1}: {s}")) + : "No attachments provided."; + + object sectionQuestionsPayload = sectionJson; + if (!string.IsNullOrWhiteSpace(sectionJson)) + { + try + { + using var sectionDoc = JsonDocument.Parse(sectionJson); + sectionQuestionsPayload = sectionDoc.RootElement.Clone(); + } + catch (JsonException) + { + // Keep raw string payload when JSON parsing fails. + } + } + + var sectionPayload = new + { + name = sectionName, + questions = sectionQuestionsPayload + }; + var sectionPayloadJson = JsonSerializer.Serialize(sectionPayload, JsonLogOptions); + var responseTemplate = BuildScoresheetSectionResponseTemplate(sectionPayloadJson); + + var analysisContent = ScoresheetPrompts.BuildSectionUserPrompt( + applicationContent, + attachmentSummariesText, + sectionPayloadJson, + responseTemplate); + + var systemPrompt = ScoresheetPrompts.SectionSystemPrompt; + + await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent); + var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000); + await LogPromptOutputAsync(ScoresheetSectionPromptType, modelOutput); + return modelOutput; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating scoresheet section answers for section {SectionName}", sectionName); + return "{}"; + } + } + + public async Task GenerateScoresheetSectionAnswersAsync(ScoresheetSectionRequest request) + { + var dataJson = JsonSerializer.Serialize(request.Data, JsonLogOptions); + var sectionJson = JsonSerializer.Serialize(request.SectionSchema, JsonLogOptions); + + var attachmentSummaries = request.Attachments + .Select(a => $"{a.Name}: {a.Summary}") + .ToList(); + + var raw = await GenerateScoresheetSectionAnswersAsync( + dataJson, + attachmentSummaries, + sectionJson, + request.SectionName); + return ParseScoresheetSectionResponse(raw); + } + + private static ApplicationAnalysisResponse ParseApplicationAnalysisResponse(string raw) + { + var response = new ApplicationAnalysisResponse(); + + if (!TryParseJsonObjectFromResponse(raw, out var root)) + { + return response; + } + + if (TryGetStringProperty(root, AIJsonKeys.Rating, out var rating)) + { + response.Rating = rating; + } + + if (root.TryGetProperty("errors", out var errors) && errors.ValueKind == JsonValueKind.Array) + { + response.Errors = ParseFindings(errors); + } + + if (root.TryGetProperty("warnings", out var warnings) && warnings.ValueKind == JsonValueKind.Array) + { + response.Warnings = ParseFindings(warnings); + } + + if (root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) && summaries.ValueKind == JsonValueKind.Array) + { + response.Summaries = ParseFindings(summaries); + } + + if (root.TryGetProperty(AIJsonKeys.Dismissed, out var dismissed) && dismissed.ValueKind == JsonValueKind.Array) + { + response.Dismissed = dismissed + .EnumerateArray() + .Select(GetStringValueOrNull) + .Where(item => !string.IsNullOrWhiteSpace(item)) + .Cast() + .ToList(); + } + + return response; + } + + private static bool TryGetStringProperty(JsonElement root, string propertyName, out string? value) + { + value = null; + if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) + { + return false; + } + + value = property.GetString(); + return !string.IsNullOrWhiteSpace(value); + } + + private static string? GetStringValueOrNull(JsonElement element) + { + if (element.ValueKind == JsonValueKind.String) + { + return element.GetString(); + } + + return null; + } + + private static List ParseFindings(JsonElement array) + { + var findings = new List(); + foreach (var item in array.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var id = item.TryGetProperty(AIJsonKeys.Id, out var idProp) && idProp.ValueKind == JsonValueKind.String + ? idProp.GetString() + : null; + string? title = null; + if (item.TryGetProperty(AIJsonKeys.Title, out var titleProp) && titleProp.ValueKind == JsonValueKind.String) + { + title = titleProp.GetString(); + } + else if (item.TryGetProperty("category", out var legacyTitleProp) && + legacyTitleProp.ValueKind == JsonValueKind.String) + { + title = legacyTitleProp.GetString(); + } + + string? detail = null; + if (item.TryGetProperty(AIJsonKeys.Detail, out var detailProp) && detailProp.ValueKind == JsonValueKind.String) + { + detail = detailProp.GetString(); + } + else if (item.TryGetProperty("message", out var legacyDetailProp) && + legacyDetailProp.ValueKind == JsonValueKind.String) + { + detail = legacyDetailProp.GetString(); + } + + findings.Add(new ApplicationAnalysisFinding + { + Id = id, + Title = title, + Detail = detail + }); + } + + return findings; + } + + private static ScoresheetSectionResponse ParseScoresheetSectionResponse(string raw) + { + var response = new ScoresheetSectionResponse(); + if (!TryParseJsonObjectFromResponse(raw, out var root)) + { + return response; + } + + foreach (var property in root.EnumerateObject()) + { + if (property.Value.ValueKind != JsonValueKind.Object) + { + continue; + } + + var answer = property.Value.TryGetProperty("answer", out var answerProp) + ? answerProp.Clone() + : default; + var rationale = property.Value.TryGetProperty("rationale", out var rationaleProp) && + rationaleProp.ValueKind == JsonValueKind.String + ? rationaleProp.GetString() ?? string.Empty + : string.Empty; + var confidence = property.Value.TryGetProperty("confidence", out var confidenceProp) && + confidenceProp.ValueKind == JsonValueKind.Number && + confidenceProp.TryGetInt32(out var parsedConfidence) + ? NormalizeConfidence(parsedConfidence) + : 0; + + response.Answers[property.Name] = new ScoresheetSectionAnswer + { + Answer = answer, + Rationale = rationale, + Confidence = confidence + }; + } + + return response; + } + + private static int NormalizeConfidence(int confidence) + { + var clamped = Math.Clamp(confidence, 0, 100); + var rounded = (int)Math.Round(clamped / 5.0, MidpointRounding.AwayFromZero) * 5; + return Math.Clamp(rounded, 0, 100); + } + + private static string BuildScoresheetSectionResponseTemplate(string sectionPayloadJson) + { + try + { + using var doc = JsonDocument.Parse(sectionPayloadJson); + if (!doc.RootElement.TryGetProperty("questions", out var questions) || questions.ValueKind != JsonValueKind.Array) + { + return ScoresheetPrompts.SectionOutputTemplate; + } + + var template = new Dictionary(); + foreach (var question in questions.EnumerateArray()) + { + if (!question.TryGetProperty("id", out var idProp) || idProp.ValueKind != JsonValueKind.String) + { + continue; + } + + var questionId = idProp.GetString(); + if (string.IsNullOrWhiteSpace(questionId)) + { + continue; + } + + template[questionId] = new + { + answer = string.Empty, + rationale = string.Empty, + confidence = 0 + }; + } + + if (template.Count == 0) + { + return ScoresheetPrompts.SectionOutputTemplate; + } + + return JsonSerializer.Serialize(template, JsonLogOptions); + } + catch (JsonException) + { + return ScoresheetPrompts.SectionOutputTemplate; + } + } + + private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt) + { + var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt); + _logger.LogInformation("AI {PromptType} input payload: {PromptInput}", promptType, formattedInput); + await WritePromptLogFileAsync(promptType, "INPUT", formattedInput); + } + + private async Task LogPromptOutputAsync(string promptType, string output) + { + var formattedOutput = FormatPromptOutputForLog(output); + _logger.LogInformation("AI {PromptType} model output payload: {ModelOutput}", promptType, formattedOutput); + await WritePromptLogFileAsync(promptType, "OUTPUT", formattedOutput); + } + + private async Task WritePromptLogFileAsync(string promptType, string payloadType, string payload) + { + if (!CanWritePromptFileLog()) + { + return; + } + + try + { + var now = DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss zzz"); + var logDirectory = Path.Combine(AppContext.BaseDirectory, PromptLogDirectoryName); + Directory.CreateDirectory(logDirectory); + + var logPath = Path.Combine(logDirectory, PromptLogFileName); + var entry = $"{now} [{promptType}] {payloadType}\n{payload}\n\n"; + await File.AppendAllTextAsync(logPath, entry); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write AI prompt log file."); + } + } + + private bool CanWritePromptFileLog() + { + return IsPromptFileLoggingEnabled; + } + + private static string FormatPromptInputForLog(string? systemPrompt, string userPrompt) + { + var normalizedSystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? string.Empty : systemPrompt.Trim(); + var normalizedUserPrompt = string.IsNullOrWhiteSpace(userPrompt) ? string.Empty : userPrompt.Trim(); + return $"SYSTEM_PROMPT\n{normalizedSystemPrompt}\n\nUSER_PROMPT\n{normalizedUserPrompt}"; + } + + private static string FormatPromptOutputForLog(string output) + { + if (string.IsNullOrWhiteSpace(output)) + { + return string.Empty; + } + + if (TryParseJsonObjectFromResponse(output, out var jsonObject)) + { + return JsonSerializer.Serialize(jsonObject, JsonLogOptions); + } + + return output.Trim(); + } + + private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) + { + objectElement = default; + var cleaned = CleanJsonResponse(response); + if (string.IsNullOrWhiteSpace(cleaned)) + { + return false; + } + + try + { + using var doc = JsonDocument.Parse(cleaned); + if (doc.RootElement.ValueKind != JsonValueKind.Object) + { + return false; + } + + objectElement = doc.RootElement.Clone(); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static string CleanJsonResponse(string response) + { + if (string.IsNullOrWhiteSpace(response)) + { + return string.Empty; + } + + var cleaned = response.Trim(); + + if (cleaned.StartsWith("```json", StringComparison.OrdinalIgnoreCase) || cleaned.StartsWith("```")) + { + var startIndex = cleaned.IndexOf('\n'); + if (startIndex >= 0) + { + // Multi-line fenced code block: remove everything up to and including the first newline. + cleaned = cleaned[(startIndex + 1)..]; + } + else + { + // Single-line fenced JSON, e.g. ```json { ... } ``` or ```{ ... } ```. + // Strip everything before the first likely JSON payload token. + var jsonStart = FindFirstJsonTokenIndex(cleaned); + + if (jsonStart > 0) + { + cleaned = cleaned[jsonStart..]; + } + } + } + + if (cleaned.EndsWith("```", StringComparison.Ordinal)) + { + var lastIndex = cleaned.LastIndexOf("```", StringComparison.Ordinal); + if (lastIndex > 0) + { + cleaned = cleaned[..lastIndex]; + } + } + + return cleaned.Trim(); + } + + private static int FindFirstJsonTokenIndex(string value) + { + var objectStart = value.IndexOf('{'); + var arrayStart = value.IndexOf('['); + + if (objectStart >= 0 && arrayStart >= 0) + { + return Math.Min(objectStart, arrayStart); + } + + if (objectStart >= 0) + { + return objectStart; + } + + return arrayStart; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt new file mode 100644 index 000000000..e11dce3c9 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptCoreRules.cs.txt @@ -0,0 +1,13 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptCoreRules + { + public const string UseProvidedEvidence = "- Use only provided input sections as evidence."; + public const string NoInvention = "- Do not invent missing details."; + public const string MinimumNarrativeWords = "- Any narrative text response must be at least 12 words."; + public const string ExactOutputShape = "- Return values exactly as specified in OUTPUT."; + public const string NoExtraOutputKeys = "- Do not return keys outside OUTPUT."; + public const string ValidJsonOnly = "- Return valid JSON only."; + public const string PlainJsonOnly = "- Return plain JSON only (no markdown)."; + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt new file mode 100644 index 000000000..701a43e74 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/PromptHeader.cs.txt @@ -0,0 +1,14 @@ +namespace Unity.GrantManager.AI +{ + internal static class PromptHeader + { + public static string Build(string role, string task) + { + return $@"ROLE +{role} + +TASK +{task}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt new file mode 100644 index 000000000..2db4de742 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/v1/ScoresheetPrompts.cs.txt @@ -0,0 +1,80 @@ +namespace Unity.GrantManager.AI +{ + internal static class ScoresheetPrompts + { + public static readonly string SectionSystemPrompt = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + + public const string SectionOutputTemplate = @"{ + """": { + ""answer"": """", + ""rationale"": """", + ""confidence"": + } +}"; + + public const string SectionRules = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer object per question ID in SECTION.questions. +- Do not omit any question IDs from SECTION.questions. +- Do not add keys that are not question IDs from SECTION.questions. +- Use RESPONSE as the output contract and fill every placeholder value. +- Follow this process in order: (1) copy RESPONSE, (2) iterate SECTION.questions in order, (3) fill answer+rationale+confidence for each matching question ID, (4) run final completeness check. +- Each answer object must include: ""answer"", ""rationale"", and ""confidence"". +- Never omit ""answer"", ""rationale"", or ""confidence"" for any question type. +- The ""answer"" value type must match question type: Number => numeric; YesNo/SelectList/Text/TextArea => string. +- The ""rationale"" field must be 1-2 complete sentences and grounded in concrete DATA/ATTACHMENTS evidence. +- In ""rationale"", cite concrete source evidence from the provided input content; do not cite prompt section headers. +- For every question, rationale must justify both the selected answer and the selected confidence level based on evidence strength. +- If explicit evidence is insufficient, choose the most conservative valid answer and state uncertainty in rationale. +- Do not treat missing or non-contradictory information as evidence. +- The ""confidence"" field must be an integer from 0 to 100 in increments of 5 and represents confidence in the selected answer. +- Set confidence by certainty of the selected answer based on available evidence, regardless of which option is selected. +- For yes/no questions, the ""answer"" field must be exactly ""Yes"" or ""No"". +- For numeric questions, answer must be a numeric value within the allowed range. +- For numeric questions, answer must never be blank. +- If evidence is insufficient for a numeric question, return the minimum allowed numeric value and explain uncertainty in rationale. +- If a required value is explicitly missing in DATA/ATTACHMENTS, set confidence high (80-100) when selecting the conservative minimum. +- For select list questions, return only the selected options.number as a string (the option index shown in options), never label text or points. +- For select list questions, the ""answer"" value must be one of question.allowed_answers exactly. +- Never return 0 for select list answers unless 0 exists as an explicit option number. +- For text and text area questions, answer must be concise, evidence-based, non-empty, and avoid boilerplate placeholders. +- For text and text area questions, answer is the reviewer comment, and rationale must explain the evidence basis and certainty for that comment. +- For comment fields, summarize key evidence-based conclusions from the other questions in SECTION, including uncertainty where applicable. +- Do not leave rationale empty when answer is populated. +- Final self-check before responding: every question ID in RESPONSE must have a non-empty ""answer"", non-empty ""rationale"", and ""confidence"". +- If any answer object is incomplete, regenerate the full JSON response before returning it. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + + public static string BuildSectionUserPrompt( + string applicationContent, + string attachmentSummariesText, + string sectionPayloadJson, + string responseTemplateJson) + { + return $@"DATA +{applicationContent} + +ATTACHMENTS +- {attachmentSummariesText} + +SECTION +{sectionPayloadJson} + +RESPONSE +{responseTemplateJson} + +OUTPUT +{SectionOutputTemplate} + +RULES +{SectionRules}"; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs index 2db4de742..306e43e36 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/ScoresheetPrompts.cs @@ -2,10 +2,41 @@ namespace Unity.GrantManager.AI { internal static class ScoresheetPrompts { + public static readonly string AllSystemPromptV0 = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, QUESTIONS, OUTPUT, and RULES, provide answers for all scoresheet questions."); + + public const string AllOutputTemplateV0 = @"{ + """": """" +}"; + + public const string AllRulesV0 = "- Use only DATA and ATTACHMENTS as evidence.\n" + + "- Do not invent missing application details.\n" + + @"- Return exactly one answer per question ID in QUESTIONS. +- Do not omit any question IDs from QUESTIONS. +- Do not add keys that are not question IDs from QUESTIONS. +- The ""answer"" value type must match the question type. +- For numeric questions, return a numeric value within the allowed range. +- For yes/no questions, return exactly ""Yes"" or ""No"". +- For select list questions, return only the selected options.number as a string and never return option label text. +- For text and text area questions, return concise, evidence-based text. +- For text and text area questions, include concise source-grounded rationale from the provided input content. +- If explicit evidence is insufficient, choose the most conservative valid answer. +" + + PromptCoreRules.MinimumNarrativeWords + "\n" + + PromptCoreRules.ExactOutputShape + "\n" + + PromptCoreRules.NoExtraOutputKeys + "\n" + + PromptCoreRules.ValidJsonOnly + "\n" + + PromptCoreRules.PlainJsonOnly; + public static readonly string SectionSystemPrompt = PromptHeader.Build( "You are an expert grant application reviewer for the BC Government.", "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + public static readonly string SectionSystemPromptV0 = PromptHeader.Build( + "You are an expert grant application reviewer for the BC Government.", + "Using DATA, ATTACHMENTS, SECTION, RESPONSE, OUTPUT, and RULES, answer only the questions in SECTION."); + public const string SectionOutputTemplate = @"{ """": { ""answer"": """", @@ -52,11 +83,23 @@ internal static class ScoresheetPrompts + PromptCoreRules.ValidJsonOnly + "\n" + PromptCoreRules.PlainJsonOnly; + public static string GetSectionSystemPrompt(bool useV0) => useV0 ? SectionSystemPromptV0 : SectionSystemPrompt; + public static string BuildSectionUserPrompt( string applicationContent, string attachmentSummariesText, string sectionPayloadJson, string responseTemplateJson) + { + return BuildSectionUserPrompt(applicationContent, attachmentSummariesText, sectionPayloadJson, responseTemplateJson, useV0: false); + } + + public static string BuildSectionUserPrompt( + string applicationContent, + string attachmentSummariesText, + string sectionPayloadJson, + string responseTemplateJson, + bool useV0) { return $@"DATA {applicationContent} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs index 9aad041a5..e2353ca79 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/TextExtractionService.cs @@ -23,10 +23,22 @@ public partial class TextExtractionService : ITextExtractionService, ITransientD private const int MaxDocxTableRows = 2000; private const int MaxDocxTableCellsPerRow = 50; private readonly ILogger _logger; + private readonly Dictionary> _extractorsByExtension; public TextExtractionService(ILogger logger) { _logger = logger; + _extractorsByExtension = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + [".txt"] = (_, content) => ExtractTextFromTextFile(content), + [".csv"] = (_, content) => ExtractTextFromTextFile(content), + [".json"] = (_, content) => ExtractTextFromTextFile(content), + [".xml"] = (_, content) => ExtractTextFromTextFile(content), + [".pdf"] = ExtractTextFromPdfFile, + [".docx"] = (name, content) => ExtractTextFromWordDocx(name, content), + [".xls"] = ExtractTextFromExcelFile, + [".xlsx"] = ExtractTextFromExcelFile + }; } public Task ExtractTextAsync(string fileName, byte[] fileContent, string contentType) @@ -42,46 +54,41 @@ public Task ExtractTextAsync(string fileName, byte[] fileContent, string var normalizedContentType = contentType?.ToLowerInvariant() ?? string.Empty; var extension = Path.GetExtension(fileName)?.ToLowerInvariant() ?? string.Empty; - string rawText; + if (extension == ".doc") + { + _logger.LogDebug("Legacy .doc extraction is not supported for {FileName}", fileName); + return Task.FromResult(string.Empty); + } + + if (_extractorsByExtension.TryGetValue(extension, out var extractor)) + { + var rawText = extractor(fileName, fileContent); + return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); + } - if (normalizedContentType.Contains("text/") || - extension == ".txt" || - extension == ".csv" || - extension == ".json" || - extension == ".xml") + if (normalizedContentType.Contains("text/")) { - rawText = ExtractTextFromTextFile(fileContent); + var rawText = ExtractTextFromTextFile(fileContent); return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } - if (normalizedContentType.Contains("pdf") || extension == ".pdf") + if (normalizedContentType.Contains("pdf")) { - rawText = ExtractTextFromPdfFile(fileName, fileContent); + var rawText = ExtractTextFromPdfFile(fileName, fileContent); return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } if (normalizedContentType.Contains("word") || normalizedContentType.Contains("msword") || - normalizedContentType.Contains("officedocument.wordprocessingml") || - extension == ".doc" || - extension == ".docx") + normalizedContentType.Contains("officedocument.wordprocessingml")) { - if (extension == ".docx" || normalizedContentType.Contains("officedocument.wordprocessingml")) - { - rawText = ExtractTextFromWordDocx(fileName, fileContent); - return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); - } - - _logger.LogDebug("Legacy .doc extraction is not supported for {FileName}", fileName); - return Task.FromResult(string.Empty); + var rawText = ExtractTextFromWordDocx(fileName, fileContent); + return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } - if (normalizedContentType.Contains("excel") || - normalizedContentType.Contains("spreadsheet") || - extension == ".xls" || - extension == ".xlsx") + if (normalizedContentType.Contains("excel") || normalizedContentType.Contains("spreadsheet")) { - rawText = ExtractTextFromExcelFile(fileName, fileContent); + var rawText = ExtractTextFromExcelFile(fileName, fileContent); return Task.FromResult(NormalizeAndLimitText(rawText, fileName)); } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs index 075d56866..f6c4943e8 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/GrantApplicationAppService.cs @@ -974,11 +974,11 @@ public async Task> GetActions(Guid applicati // NOTE: Authorization is applied on the AppService layer and is false by default // AUTHORIZATION HANDLING - actionDtos.ForEach(async item => - { - item.IsPermitted = item.IsPermitted && (await AuthorizationService.IsGrantedAsync(application, GetActionAuthorizationRequirement(item.ApplicationAction))); - item.IsAuthorized = true; - }); + foreach (var item in actionDtos) + { + item.IsPermitted = item.IsPermitted && (await AuthorizationService.IsGrantedAsync(application, GetActionAuthorizationRequirement(item.ApplicationAction))); + item.IsAuthorized = true; + } return new ListResultDto(actionDtos); }