Feature/ab#32009 prompt version baselines

applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/OpenAIService.cs

@@ -22,6 +22,8 @@ public class OpenAIService : IAIService, ITransientDependency
         private const string AttachmentSummaryPromptType = "AttachmentSummary";
         private const string ScoresheetAllPromptType = "ScoresheetAll";
         private const string ScoresheetSectionPromptType = "ScoresheetSection";
+        private const string PromptVersionV0 = "v0";
+        private const string PromptVersionV1 = "v1";
         private const string NoSummaryGeneratedMessage = "No summary generated.";
         private const string ServiceNotConfiguredMessage = "AI analysis not available - service not configured.";
         private const string ServiceTemporarilyUnavailableMessage = "AI analysis failed - service temporarily unavailable.";
@@ -39,6 +41,9 @@ public class OpenAIService : IAIService, ITransientDependency
 
         private static readonly JsonSerializerOptions JsonLogOptions = new() { WriteIndented = true };
 
+        private string SelectedPromptVersion => NormalizePromptVersion(_configuration["Azure:OpenAI:PromptVersion"]);
+        private bool UseV0Prompts => string.Equals(SelectedPromptVersion, PromptVersionV0, StringComparison.OrdinalIgnoreCase);
+
         public OpenAIService(
             HttpClient httpClient,
             IConfiguration configuration,
@@ -84,13 +89,10 @@ public async Task<ApplicationAnalysisResponse> GenerateApplicationAnalysisAsync(
                 })
                 .Cast<object>();
 
-            var analysisContent = AnalysisPrompts.BuildUserPrompt(
-                schemaJson,
-                dataJson,
-                JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions),
-                request.Rubric ?? string.Empty);
-
-            var systemPrompt = AnalysisPrompts.SystemPrompt;
+            var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions);
+            var rubric = request.Rubric ?? AnalysisPrompts.GetRubric(UseV0Prompts);
+            var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, UseV0Prompts);
+            var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts);
             await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent);
             var raw = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000);
             await LogPromptOutputAsync(ApplicationAnalysisPromptType, raw);
@@ -171,11 +173,11 @@ public async Task<string> GenerateAttachmentSummaryAsync(string fileName, byte[]
             {
                 var extractedText = await _textExtractionService.ExtractTextAsync(fileName, fileContent, contentType);
 
-                var prompt = $@"{AttachmentPrompts.SystemPrompt}
+                var prompt = $@"{AttachmentPrompts.GetSystemPrompt(UseV0Prompts)}
 
-{AttachmentPrompts.OutputSection}
+{AttachmentPrompts.GetOutputSection(UseV0Prompts)}
 
-{AttachmentPrompts.RulesSection}";
+{AttachmentPrompts.GetRulesSection(UseV0Prompts)}";
 
                 var attachmentText = string.IsNullOrWhiteSpace(extractedText) ? null : extractedText;
                 if (attachmentText != null)
@@ -194,13 +196,13 @@ public async Task<string> GenerateAttachmentSummaryAsync(string fileName, byte[]
                     sizeBytes = fileContent.Length,
                     text = attachmentText
                 };
-                var contentToAnalyze = AttachmentPrompts.BuildUserPrompt(
-                    JsonSerializer.Serialize(attachmentPayload, JsonLogOptions));
+                var payloadJson = JsonSerializer.Serialize(attachmentPayload, JsonLogOptions);
+                var contentToAnalyze = AttachmentPrompts.BuildUserPrompt(payloadJson, UseV0Prompts);
 
                 await LogPromptInputAsync(AttachmentSummaryPromptType, prompt, contentToAnalyze);
                 var modelOutput = await GenerateSummaryAsync(contentToAnalyze, prompt, 150);
                 await LogPromptOutputAsync(AttachmentSummaryPromptType, modelOutput);
-                return modelOutput;
+                return UseV0Prompts ? ExtractSummaryFromJson(modelOutput) : modelOutput;
             }
             catch (Exception ex)
             {
@@ -257,13 +259,14 @@ public async Task<string> AnalyzeApplicationAsync(string applicationContent, Lis
                         .Cast<object>()
                     : Enumerable.Empty<object>();
 
-                var analysisContent = AnalysisPrompts.BuildUserPrompt(
-                    JsonSerializer.Serialize(schemaPayload, JsonLogOptions),
-                    JsonSerializer.Serialize(dataPayload, JsonLogOptions),
-                    JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions),
-                    rubric);
-
-                var systemPrompt = AnalysisPrompts.SystemPrompt;
+                var schemaJson = JsonSerializer.Serialize(schemaPayload, JsonLogOptions);
+                var dataJson = JsonSerializer.Serialize(dataPayload, JsonLogOptions);
+                var attachmentsJson = JsonSerializer.Serialize(attachmentsPayload, JsonLogOptions);
+                var fallbackRubric = string.IsNullOrWhiteSpace(rubric)
+                    ? AnalysisPrompts.GetRubric(UseV0Prompts)
+                    : rubric;
+                var analysisContent = AnalysisPrompts.BuildUserPrompt(schemaJson, dataJson, attachmentsJson, fallbackRubric, UseV0Prompts);
+                var systemPrompt = AnalysisPrompts.GetSystemPrompt(UseV0Prompts);
 
                 await LogPromptInputAsync(ApplicationAnalysisPromptType, systemPrompt, analysisContent);
                 var rawAnalysis = await GenerateSummaryAsync(analysisContent, systemPrompt, 1000);
@@ -446,9 +449,10 @@ public async Task<string> GenerateScoresheetSectionAnswersAsync(string applicati
                     applicationContent,
                     attachmentSummariesText,
                     sectionPayloadJson,
-                    responseTemplate);
+                    responseTemplate,
+                    UseV0Prompts);
 
-                var systemPrompt = ScoresheetPrompts.SectionSystemPrompt;
+                var systemPrompt = ScoresheetPrompts.GetSectionSystemPrompt(UseV0Prompts);
 
                 await LogPromptInputAsync(ScoresheetSectionPromptType, systemPrompt, analysisContent);
                 var modelOutput = await GenerateSummaryAsync(analysisContent, systemPrompt, 2000);
@@ -683,14 +687,14 @@ private static string BuildScoresheetSectionResponseTemplate(string sectionPaylo
         private async Task LogPromptInputAsync(string promptType, string? systemPrompt, string userPrompt)
         {
             var formattedInput = FormatPromptInputForLog(systemPrompt, userPrompt);
-            _logger.LogInformation("AI {PromptType} input payload: {PromptInput}", promptType, formattedInput);
+            _logger.LogInformation("AI {PromptType} ({PromptVersion}) input payload: {PromptInput}", promptType, SelectedPromptVersion, formattedInput);
             await WritePromptLogFileAsync(promptType, "INPUT", formattedInput);
         }
 
         private async Task LogPromptOutputAsync(string promptType, string output)
         {
             var formattedOutput = FormatPromptOutputForLog(output);
-            _logger.LogInformation("AI {PromptType} model output payload: {ModelOutput}", promptType, formattedOutput);
+            _logger.LogInformation("AI {PromptType} ({PromptVersion}) model output payload: {ModelOutput}", promptType, SelectedPromptVersion, formattedOutput);
             await WritePromptLogFileAsync(promptType, "OUTPUT", formattedOutput);
         }
 
@@ -829,5 +833,31 @@ private static int FindFirstJsonTokenIndex(string value)
 
             return arrayStart;
         }
+
+        private static string NormalizePromptVersion(string? version)
+        {
+            if (string.Equals(version, PromptVersionV0, StringComparison.OrdinalIgnoreCase))
+            {
+                return PromptVersionV0;
+            }
+
+            return PromptVersionV1;
+        }
+
+        private static string ExtractSummaryFromJson(string output)
+        {
+            if (!TryParseJsonObjectFromResponse(output, out var jsonObject))
+            {
+                return output?.Trim() ?? string.Empty;
+            }
+
+            if (jsonObject.TryGetProperty(AIJsonKeys.Summary, out var summaryProp) &&
+                summaryProp.ValueKind == JsonValueKind.String)
+            {
+                return summaryProp.GetString() ?? string.Empty;
+            }
+
+            return output?.Trim() ?? string.Empty;
+        }
     }
 }

applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AnalysisPrompts.cs

@@ -2,7 +2,13 @@ namespace Unity.GrantManager.AI
 {
     internal static class AnalysisPrompts
     {
-        public const string DefaultRubric = @"BC GOVERNMENT GRANT EVALUATION RUBRIC:
+        public const string DefaultRubric = @"ELIGIBILITY REQUIREMENTS: Project aligns with program objectives; Applicant is an eligible entity; Budget is reasonable and justified; Timeline is realistic.
+COMPLETENESS CHECKS: Required information is present; Supporting materials are provided where applicable; Description is clear.
+FINANCIAL REVIEW: Requested amount is within limits; Budget matches scope; Matching funds or contributions are identified.
+RISK ASSESSMENT: Applicant capacity; Feasibility; Compliance considerations; Delivery risks.
+QUALITY INDICATORS: Clear objectives; Defined beneficiaries; Appropriate approach; Long-term sustainability.";
+
+        public const string DefaultRubricV0 = @"BC GOVERNMENT GRANT EVALUATION RUBRIC:
 
 1. ELIGIBILITY REQUIREMENTS:
    - Project must align with program objectives
@@ -43,42 +49,81 @@ internal static class AnalysisPrompts
 MEDIUM: Application has some gaps or weaknesses that require reviewer attention.
 LOW: Application has significant gaps or risks across key rubric areas.";
 
+        public const string OutputTemplate = @"{
+  ""rating"": ""<HIGH|MEDIUM|LOW>"",
+  ""errors"": [
+    {
+      ""title"": ""<string>"",
+      ""detail"": ""<string>""
+    }
+  ],
+  ""warnings"": [
+    {
+      ""title"": ""<string>"",
+      ""detail"": ""<string>""
+    }
+  ],
+  ""summaries"": [
+    {
+      ""title"": ""<string>"",
+      ""detail"": ""<string>""
+    }
+  ]
+}";
+
+        public const string Rules = PromptCoreRules.UseProvidedEvidence + "\n"
+            + "- Do not invent fields, documents, requirements, or facts.\n"
+            + @"- Treat missing or empty values as findings only when they weaken rubric evidence.
+- Prefer material issues; avoid nitpicking.
+- Use 3-6 words for title.
+- Each detail must be 1-2 complete sentences.
+- Each detail must cite concrete evidence from DATA or ATTACHMENTS.
+- If ATTACHMENTS evidence is used, cite the attachment by name in detail.
+- If no findings exist, return empty arrays.
+- Rating must be HIGH, MEDIUM, or LOW.
+"
+            + PromptCoreRules.MinimumNarrativeWords + "\n"
+            + PromptCoreRules.ExactOutputShape + "\n"
+            + PromptCoreRules.NoExtraOutputKeys + "\n"
+            + PromptCoreRules.ValidJsonOnly + "\n"
+            + PromptCoreRules.PlainJsonOnly;
+
         public const string SeverityRules = @"ERROR: Issue that would likely prevent the application from being approved.
 WARNING: Issue that could negatively affect the application's approval.
 RECOMMENDATION: Reviewer-facing improvement or follow-up consideration.";
 
-        public const string OutputTemplate = @"{
+        public const string OutputTemplateV0 = @"{
   ""rating"": ""HIGH/MEDIUM/LOW"",
   ""warnings"": [
     {
-      ""title"": ""Brief summary of the warning"",
-      ""detail"": ""Detailed warning message with full context and explanation""
+      ""category"": ""Brief summary of the warning"",
+      ""message"": ""Detailed warning message with full context and explanation""
     }
   ],
   ""errors"": [
     {
-      ""title"": ""Brief summary of the error"",
-      ""detail"": ""Detailed error message with full context and explanation""
+      ""category"": ""Brief summary of the error"",
+      ""message"": ""Detailed error message with full context and explanation""
     }
   ],
   ""summaries"": [
     {
-      ""title"": ""Brief summary of the recommendation"",
-      ""detail"": ""Detailed recommendation with specific actionable guidance""
+      ""category"": ""Brief summary of the recommendation"",
+      ""message"": ""Detailed recommendation with specific actionable guidance""
     }
   ],
   ""dismissed"": []
 }";
 
-        public const string Rules = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence.
+        public const string RulesV0 = @"- Use only SCHEMA, DATA, ATTACHMENTS, and RUBRIC as evidence.
 - Do not invent fields, documents, requirements, or facts.
 - Treat missing or empty values as findings only when they weaken rubric evidence.
 - Prefer material issues; avoid nitpicking.
 - Each error/warning/recommendation must describe one concrete issue or consideration and why it matters.
-- Use 3-6 words for title.
-- Each detail must be 1-2 complete sentences.
-- Each detail must be grounded in concrete evidence from provided inputs.
-- If attachment evidence is used, reference the attachment explicitly in detail.
+- Use 3-6 words for category.
+- Each message must be 1-2 complete sentences.
+- Each message must be grounded in concrete evidence from provided inputs.
+- If attachment evidence is used, reference the attachment explicitly in the message.
 - Do not provide applicant-facing advice.
 - Do not mention rubric section names in findings.
 - If no findings exist, return empty arrays.
@@ -89,15 +134,39 @@ internal static class AnalysisPrompts
             + "\n" + PromptCoreRules.PlainJsonOnly;
 
         public static readonly string SystemPrompt = PromptHeader.Build(
+            "You are an expert grant analyst assistant for human reviewers.",
+            "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SCORE, OUTPUT, and RULES, return review findings.");
+
+        public static readonly string SystemPromptV0 = PromptHeader.Build(
             "You are an expert grant analyst assistant for human reviewers.",
             "Using SCHEMA, DATA, ATTACHMENTS, RUBRIC, SEVERITY, SCORE, OUTPUT, and RULES, return review findings.");
 
+        public static string GetRubric(bool useV0) => useV0 ? DefaultRubricV0 : DefaultRubric;
+        public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt;
+
         public static string BuildUserPrompt(
             string schemaJson,
             string dataJson,
             string attachmentsJson,
             string rubric)
         {
+            return BuildUserPrompt(schemaJson, dataJson, attachmentsJson, rubric, useV0: false);
+        }
+
+        public static string BuildUserPrompt(
+            string schemaJson,
+            string dataJson,
+            string attachmentsJson,
+            string rubric,
+            bool useV0)
+        {
+            var output = useV0 ? OutputTemplateV0 : OutputTemplate;
+            var rules = useV0 ? RulesV0 : Rules;
+            var severitySection = useV0 ? $@"SEVERITY
+{SeverityRules}
+
+" : string.Empty;
+
             return $@"SCHEMA
 {schemaJson}
 
@@ -110,17 +179,14 @@ public static string BuildUserPrompt(
 RUBRIC
 {rubric}
 
-SEVERITY
-{SeverityRules}
-
-SCORE
+{severitySection}SCORE
 {ScoreRules}
 
 OUTPUT
-{OutputTemplate}
+{output}
 
 RULES
-{Rules}";
+{rules}";
         }
     }
 }

applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/AttachmentPrompts.cs

@@ -6,19 +6,51 @@ internal static class AttachmentPrompts
             "You are a professional grant analyst for the BC Government.",
             "Produce a concise reviewer-facing summary of the provided attachment context.");
 
+        public static readonly string SystemPromptV0 = PromptHeader.Build(
+            "You are a professional grant analyst for the BC Government.",
+            "Produce a concise reviewer-facing summary of the provided attachment context.");
+
         public const string OutputSection = @"OUTPUT
+{
+  ""summary"": ""<string>""
+}";
+
+        public const string RulesSection = "- Use only ATTACHMENT as evidence.\n"
+            + "- If ATTACHMENT.text is present, summarize actual content.\n"
+            + "- If ATTACHMENT.text is null or empty, provide a conservative file-level summary.\n"
+            + PromptCoreRules.NoInvention + "\n"
+            + @"- Write 1-2 complete sentences.
+- Summary must be grounded in concrete ATTACHMENT evidence.
+- Return exactly one object with only the key: summary.
+"
+            + PromptCoreRules.MinimumNarrativeWords + "\n"
+            + PromptCoreRules.ExactOutputShape + "\n"
+            + PromptCoreRules.NoExtraOutputKeys + "\n"
+            + PromptCoreRules.ValidJsonOnly + "\n"
+            + PromptCoreRules.PlainJsonOnly;
+
+        public const string OutputSectionV0 = @"OUTPUT
 - Plain text only
 - 1-2 complete sentences";
 
-        public const string RulesSection = @"RULES
+        public const string RulesSectionV0 = @"RULES
 - Use only the provided attachment context as evidence.
 - If text content is present, summarize the actual content.
 - If text content is missing or empty, provide a conservative metadata-based summary.
 - Do not invent missing details.
 - Keep the summary specific, concrete, and reviewer-facing.
 - Return plain text only (no markdown, bullets, or JSON).";
 
+        public static string GetSystemPrompt(bool useV0) => useV0 ? SystemPromptV0 : SystemPrompt;
+        public static string GetOutputSection(bool useV0) => useV0 ? OutputSectionV0 : OutputSection;
+        public static string GetRulesSection(bool useV0) => useV0 ? RulesSectionV0 : RulesSection;
+
         public static string BuildUserPrompt(string attachmentPayloadJson)
+        {
+            return BuildUserPrompt(attachmentPayloadJson, useV0: false);
+        }
+
+        public static string BuildUserPrompt(string attachmentPayloadJson, bool useV0)
         {
             return $@"ATTACHMENT
 {attachmentPayloadJson}";

applications/Unity.GrantManager/src/Unity.GrantManager.Application/AI/Prompts/Baselines/README.md

@@ -0,0 +1,8 @@
+# Prompt Baselines
+
+- `v0`: legacy prompt/service snapshots used as historical baseline.
+- `v1`: current runtime prompt/service snapshots.
+
+Versioning convention:
+- Folder name is the baseline version.
+- Filenames inside each folder are normalized and do not include version suffixes.