diff --git a/applications/Unity.GrantManager/docker-compose.yml b/applications/Unity.GrantManager/docker-compose.yml index a3015c37e..720f9574d 100644 --- a/applications/Unity.GrantManager/docker-compose.yml +++ b/applications/Unity.GrantManager/docker-compose.yml @@ -146,6 +146,27 @@ services: networks: - common-network + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - ./scripts/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./scripts/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro + depends_on: + - unity-grantmanager-web + networks: + - common-network + + alertmanager: + image: prom/alertmanager:latest + ports: + - "9093:9093" + volumes: + - ./scripts/prometheus/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + networks: + - common-network + volumes: postgres_data: redis_volume_data: diff --git a/applications/Unity.GrantManager/scripts/openshift/alertmanager-config.yaml b/applications/Unity.GrantManager/scripts/openshift/alertmanager-config.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/applications/Unity.GrantManager/scripts/openshift/prometheus-rule.yaml b/applications/Unity.GrantManager/scripts/openshift/prometheus-rule.yaml new file mode 100644 index 000000000..d4096abc2 --- /dev/null +++ b/applications/Unity.GrantManager/scripts/openshift/prometheus-rule.yaml @@ -0,0 +1,44 @@ +# PrometheusRule CRD — loaded by the OpenShift cluster Prometheus Operator +# Deploy with: oc apply -f scripts/openshift/prometheus-rule.yaml -n d18498- +# +# Replaces: scripts/prometheus/alert-rules.yml (docker-compose local only) +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: unity-grantmanager-exceptions + labels: + # These labels must match the Prometheus Operator's ruleSelector in your namespace. + # On BC Gov Silver cluster the label below is standard. + role: alert-rules +spec: + groups: + - name: unity-grantmanager-exceptions + rules: + # Fire if any exception type exceeds 5 occurrences in a 5-minute window + - alert: HighExceptionRate + expr: | + increase(application_exceptions_total[5m]) > 5 + for: 1m + labels: + severity: critical + annotations: + summary: "High exception rate in Unity GrantManager" + description: > + Exception type {{ $labels.type }} has fired {{ $value | humanize }} times + in the last 5 minutes (namespace: {{ $labels.namespace }}). + + # Fire if a new exception type appears (catches regressions after deploys) + - alert: NewExceptionType + expr: | + increase(application_exceptions_total[10m]) > 0 + unless ( + increase(application_exceptions_total[10m] offset 10m) > 0 + ) + for: 0m + labels: + severity: warning + annotations: + summary: "New exception type detected in Unity GrantManager" + description: > + A new exception type {{ $labels.type }} appeared for the first time + in the last 10 minutes (namespace: {{ $labels.namespace }}). diff --git a/applications/Unity.GrantManager/scripts/openshift/service-monitor.yaml b/applications/Unity.GrantManager/scripts/openshift/service-monitor.yaml new file mode 100644 index 000000000..f5428da67 --- /dev/null +++ b/applications/Unity.GrantManager/scripts/openshift/service-monitor.yaml @@ -0,0 +1,23 @@ +# ServiceMonitor CRD — tells the Prometheus Operator how to scrape /metrics from the app +# Deploy with: oc apply -f scripts/openshift/service-monitor.yaml -n d18498- +# +# Replaces: scrape_configs in scripts/prometheus/prometheus.yml (docker-compose local only) +# +# Prerequisites: +# The app Service must exist and expose port 8080 (or 80). +# Adjust 'port' below to match your Service's named port. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: unity-grantmanager + labels: + app: unity-grantmanager +spec: + selector: + matchLabels: + app: unity-grantmanager # must match labels on your OpenShift Service + endpoints: + - port: http # named port on the Service pointing to 8080 + path: /metrics + interval: 15s + scheme: http diff --git a/applications/Unity.GrantManager/scripts/prometheus/alert-rules.yml b/applications/Unity.GrantManager/scripts/prometheus/alert-rules.yml new file mode 100644 index 000000000..c8ec1ad6e --- /dev/null +++ b/applications/Unity.GrantManager/scripts/prometheus/alert-rules.yml @@ -0,0 +1,29 @@ +groups: + - name: unity-grantmanager-exceptions + rules: + # Fire if any exception type exceeds 5 occurrences in a 5-minute window + - alert: HighExceptionRate + expr: | + increase(application_exceptions_total[5m]) > 5 + for: 1m + labels: + severity: critical + annotations: + summary: "High exception rate in Unity GrantManager" + description: > + Exception type {{ $labels.type }} has fired {{ $value | humanize }} times + in the last 5 minutes (job: {{ $labels.job }}, instance: {{ $labels.instance }}). + - alert: NewExceptionType + expr: | + increase(application_exceptions_total[10m]) > 0 + unless ( + increase(application_exceptions_total[10m] offset 10m) > 0 + ) + for: 0m + labels: + severity: warning + annotations: + summary: "New exception type detected in Unity GrantManager" + description: > + A new exception type {{ $labels.type }} appeared for the first time + in the last 10 minutes. diff --git a/applications/Unity.GrantManager/scripts/prometheus/alertmanager.yml b/applications/Unity.GrantManager/scripts/prometheus/alertmanager.yml new file mode 100644 index 000000000..f313fbf3b --- /dev/null +++ b/applications/Unity.GrantManager/scripts/prometheus/alertmanager.yml @@ -0,0 +1,15 @@ +global: + resolve_timeout: 5m + +route: + group_by: ["alertname", "type"] + group_wait: 10s + group_interval: 5m + repeat_interval: 1h + receiver: unity-webhook + +receivers: + - name: unity-webhook + webhook_configs: + - url: "http://unity-grantmanager-web:8080/api/monitoring/alert" + send_resolved: false diff --git a/applications/Unity.GrantManager/scripts/prometheus/prometheus.yml b/applications/Unity.GrantManager/scripts/prometheus/prometheus.yml new file mode 100644 index 000000000..4bc80b2be --- /dev/null +++ b/applications/Unity.GrantManager/scripts/prometheus/prometheus.yml @@ -0,0 +1,17 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + +rule_files: + - /etc/prometheus/alert-rules.yml + +scrape_configs: + - job_name: unity-grantmanager + static_configs: + - targets: ["unity-grantmanager-web:8080"] + metrics_path: /metrics diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Controllers/Monitoring/AlertPayload.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Controllers/Monitoring/AlertPayload.cs new file mode 100644 index 000000000..75a9c2bfa --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Controllers/Monitoring/AlertPayload.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Unity.GrantManager.Web.Controllers.Monitoring; + +public class AlertManagerPayload +{ + [JsonPropertyName("receiver")] + public string Receiver { get; set; } = string.Empty; + + [JsonPropertyName("status")] + public string Status { get; set; } = string.Empty; + + private List _alerts = []; + + [JsonPropertyName("alerts")] + public List Alerts + { + get => _alerts; + set => _alerts = value ?? []; + } +} + +public class AlertItem +{ + [JsonPropertyName("status")] + public string Status { get; set; } = string.Empty; + + private Dictionary _labels = []; + private Dictionary _annotations = []; + + [JsonPropertyName("labels")] + public Dictionary Labels + { + get => _labels; + set => _labels = value ?? []; + } + + [JsonPropertyName("annotations")] + public Dictionary Annotations + { + get => _annotations; + set => _annotations = value ?? []; + } + + [JsonPropertyName("startsAt")] + public DateTimeOffset StartsAt { get; set; } + + [JsonPropertyName("generatorURL")] + public string GeneratorURL { get; set; } = string.Empty; + + [JsonPropertyName("fingerprint")] + public string Fingerprint { get; set; } = string.Empty; +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Controllers/Monitoring/AlertWebhookController.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Controllers/Monitoring/AlertWebhookController.cs new file mode 100644 index 000000000..ea32f890c --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Controllers/Monitoring/AlertWebhookController.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Unity.GrantManager.Notifications; +using Unity.Notifications.TeamsNotifications; +using Volo.Abp.AspNetCore.Mvc; + +namespace Unity.GrantManager.Web.Controllers.Monitoring; + +[ApiController] +[Route("api/monitoring")] +[AllowAnonymous] +[IgnoreAntiforgeryToken] +public class AlertWebhookController( + INotificationsAppService notificationsAppService, + ILogger logger) : AbpController +{ + /// + /// Receives Alertmanager webhook payloads and forwards a concise summary to Teams. + /// + [HttpPost("alert")] + public async Task ProcessAlert([FromBody] AlertManagerPayload? payload) + { + if (payload is null || !ModelState.IsValid || payload.Alerts.Count == 0) + { + return BadRequest(); + } + + try + { + var firing = payload.Alerts + .Where(a => a is not null && string.Equals(a.Status, "firing", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (firing.Count == 0) + { + return Ok(); + } + + // Pick the most severe alert as the headline (critical > error > warning > info > unknown) + var lead = firing + .OrderBy(a => SeverityOrder(a.Labels.GetValueOrDefault("severity", "unknown"))) + .First(); + string alertName = lead.Labels.GetValueOrDefault("alertname", "Unknown Alert"); + string severity = lead.Labels.GetValueOrDefault("severity", "unknown"); + string summary = lead.Annotations.GetValueOrDefault("summary", alertName); + string description = lead.Annotations.GetValueOrDefault("description", string.Empty); + string @namespace = lead.Labels.GetValueOrDefault("kubernetes_namespace_name", + lead.Labels.GetValueOrDefault("namespace", string.Empty)); + string endpoint = lead.Labels.GetValueOrDefault("handler", + lead.Labels.GetValueOrDefault("endpoint", string.Empty)); + string? envInfo = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); + + string activityTitle = $"[{severity.ToUpperInvariant()}] {summary}"; + string activitySubtitle = $"Environment: {envInfo} | Namespace: {@namespace}"; + + var facts = new List(); + + if (!string.IsNullOrEmpty(description)) + { + facts.Add(new Fact { Name = "Description", Value = description }); + } + + if (firing.Count > 1) + { + facts.Add(new Fact { Name = "Firing alerts", Value = firing.Count.ToString() }); + } + + if (!string.IsNullOrEmpty(endpoint)) + { + facts.Add(new Fact { Name = "Affected endpoint", Value = endpoint }); + } + + facts.Add(new Fact { Name = "First seen", Value = lead.StartsAt.ToString("u") }); + + if (!string.IsNullOrEmpty(lead.GeneratorURL)) + { + facts.Add(new Fact { Name = "Source", Value = lead.GeneratorURL }); + } + + await notificationsAppService.PostToTeamsAsync(activityTitle, activitySubtitle, facts); + + return Ok(); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to forward alert {AlertName} to Teams", + payload.Alerts.FirstOrDefault()?.Labels?.GetValueOrDefault("alertname")); + return StatusCode(500); + } + } + + private static int SeverityOrder(string? severity) => severity?.ToLowerInvariant() switch + { + "critical" => 0, + "error" => 1, + "warning" => 2, + "info" => 3, + _ => 4 + }; +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/GrantManagerWebModule.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/GrantManagerWebModule.cs index e6f9d5eb1..3a6f891df 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/GrantManagerWebModule.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/GrantManagerWebModule.cs @@ -5,6 +5,7 @@ using Microsoft.AspNetCore.CookiePolicy; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Localization; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -77,6 +78,7 @@ using Unity.Reporting.Web; using Unity.AI.Web; using Unity.GrantManager.Web.Views.Settings; +using Prometheus; namespace Unity.GrantManager.Web; @@ -149,6 +151,46 @@ public override void ConfigureServices(ServiceConfigurationContext context) ConfigureDataProtection(context, configuration); ConfigureMiniProfiler(context, configuration); + // Trust forwarded client IP headers only from explicitly configured ingress/router addresses. + // This ensures RemoteIpAddress reflects the real client IP only when the request came + // through a known proxy, so IP-based checks such as the /metrics policy cannot be spoofed + // by arbitrary internal callers. + var knownForwardedHeaderProxies = configuration + .GetSection("ForwardedHeaders:KnownProxies") + .Get() ?? Array.Empty(); + var knownForwardedHeaderNetworks = configuration + .GetSection("ForwardedHeaders:KnownNetworks") + .Get() ?? Array.Empty(); + + context.Services.Configure(options => + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedProto; + options.ForwardLimit = 1; + options.KnownProxies.Clear(); + options.KnownIPNetworks.Clear(); + + foreach (var proxy in knownForwardedHeaderProxies) + { + if (!string.IsNullOrWhiteSpace(proxy)) + { + options.KnownProxies.Add(System.Net.IPAddress.Parse(proxy)); + } + } + + foreach (var network in knownForwardedHeaderNetworks) + { + if (!string.IsNullOrWhiteSpace(network)) + { + options.KnownIPNetworks.Add(System.Net.IPNetwork.Parse(network)); + } + } + + if (options.KnownProxies.Count > 0 || options.KnownIPNetworks.Count > 0) + { + options.ForwardedHeaders |= ForwardedHeaders.XForwardedFor; + } + }); + Configure(options => { options.TokenCookie.Expiration = TimeSpan.FromDays(365); @@ -276,6 +318,8 @@ private static void ConfgureFormsApiAuhentication(ServiceConfigurationContext co private static void ConfigurePolicies(ServiceConfigurationContext context) { + context.Services.AddScoped(); + context.Services.AddSingleton(); PolicyRegistrant.Register(context); } @@ -555,6 +599,10 @@ public override void OnApplicationInitialization(ApplicationInitializationContex IdentityModelEventSource.ShowPII = true; } + // Rewrite RemoteIpAddress from X-Forwarded-For before any IP-based checks run. + // Trusted networks are configured in ConfigureServices above. + app.UseForwardedHeaders(); + app.UseAbpRequestLocalization(); if (env.IsProduction() || env.IsStaging()) @@ -587,7 +635,9 @@ public override void OnApplicationInitialization(ApplicationInitializationContex app.UseCorrelationId(); app.UseStaticFiles(); + app.UseMiddleware(); app.UseRouting(); + app.UseHttpMetrics(); app.UseAuthentication(); if (MultiTenancyConsts.IsEnabled) @@ -608,7 +658,10 @@ public override void OnApplicationInitialization(ApplicationInitializationContex }); app.UseAuditing(); app.UseAbpSerilogEnrichers(); - app.UseConfiguredEndpoints(); + app.UseConfiguredEndpoints(endpoints => + { + endpoints.MapMetrics().RequireAuthorization(Unity.GrantManager.Web.Identity.Policy.PolicyRegistrant.MetricsAccessPolicy); + }); var supportedCultures = new[] { diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/InternalNetworkRequirement.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/InternalNetworkRequirement.cs new file mode 100644 index 000000000..ade7147bc --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/InternalNetworkRequirement.cs @@ -0,0 +1,69 @@ +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; + +namespace Unity.GrantManager.Web.Identity.Policy; + +/// +/// Allows access to /metrics only from loopback or RFC-1918 private addresses. +/// This permits Prometheus to scrape pod-to-pod within the OpenShift cluster +/// while blocking external callers. +/// +public class InternalNetworkRequirement : IAuthorizationRequirement { } + +public class InternalNetworkHandler(IHttpContextAccessor httpContextAccessor) + : AuthorizationHandler +{ + protected override Task HandleRequirementAsync( + AuthorizationHandlerContext context, + InternalNetworkRequirement requirement) + { + var remoteIp = httpContextAccessor.HttpContext?.Connection.RemoteIpAddress; + + if (remoteIp is null) + { + context.Fail(); + return Task.CompletedTask; + } + + // Map IPv4-in-IPv6 (::ffff:x.x.x.x) back to IPv4 for range checks + if (remoteIp.IsIPv4MappedToIPv6) + { + remoteIp = remoteIp.MapToIPv4(); + } + + if (IsAllowed(remoteIp)) + { + context.Succeed(requirement); + } + else + { + context.Fail(); + } + + return Task.CompletedTask; + } + + private static bool IsAllowed(IPAddress ip) + { + if (IPAddress.IsLoopback(ip)) return true; + + if (ip.AddressFamily == AddressFamily.InterNetwork) + { + byte[] bytes = ip.GetAddressBytes(); + + // 10.0.0.0/8 + if (bytes[0] == 10) return true; + + // 172.16.0.0/12 + if (bytes[0] == 172 && bytes[1] >= 16 && bytes[1] <= 31) return true; + + // 192.168.0.0/16 + if (bytes[0] == 192 && bytes[1] == 168) return true; + } + + return false; + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/PolicyRegistrant.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/PolicyRegistrant.cs index 10e9446c7..ea05317e4 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/PolicyRegistrant.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Identity/PolicyRegistrant.cs @@ -12,12 +12,17 @@ namespace Unity.GrantManager.Web.Identity.Policy; internal static class PolicyRegistrant { internal const string PermissionConstant = "Permission"; + internal const string MetricsAccessPolicy = "MetricsAccess"; internal static void Register(ServiceConfigurationContext context) { // Using AddAuthorizationBuilder to register authorization services and construct policies var authorizationBuilder = context.Services.AddAuthorizationBuilder(); + // Metrics endpoint — allow only loopback / RFC-1918 (cluster-internal) callers + authorizationBuilder.AddPolicy(MetricsAccessPolicy, + policy => policy.AddRequirements(new InternalNetworkRequirement())); + // Identity Role Policies authorizationBuilder.AddPolicy(IdentityPermissions.Roles.Default, policy => policy.RequireClaim(PermissionConstant, IdentityPermissions.Roles.Default)); diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ErrorCountingLoggerSink.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ErrorCountingLoggerSink.cs new file mode 100644 index 000000000..e7380fb09 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ErrorCountingLoggerSink.cs @@ -0,0 +1,32 @@ +using Prometheus; +using Serilog.Core; +using Serilog.Events; + +namespace Unity.GrantManager.Web.Middleware; + +/// +/// Shared Prometheus counter for application-level errors. +/// Labelled by log level ("error" / "fatal") and exception type (empty when no exception). +/// Implemented as a Serilog ILogEventSink so it works alongside UseSerilog(). +/// Register via: .WriteTo.Sink(new ErrorCountingLoggerSink()) +/// +public sealed class ErrorCountingLoggerSink : ILogEventSink +{ + internal static readonly Counter ErrorCounter = + Metrics.CreateCounter( + "application_errors_total", + "Total application errors captured via Serilog", + new CounterConfiguration + { + LabelNames = ["level", "exception"] + }); + + public void Emit(LogEvent logEvent) + { + if (logEvent.Level < LogEventLevel.Error) return; + + string level = logEvent.Level.ToString().ToLowerInvariant(); + string exceptionType = logEvent.Exception?.GetType().Name ?? string.Empty; + ErrorCounter.WithLabels(level, exceptionType).Inc(); + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ExceptionCounterMiddleware.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ExceptionCounterMiddleware.cs new file mode 100644 index 000000000..1e96b723d --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ExceptionCounterMiddleware.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Prometheus; +using Unity.GrantManager.Notifications; +using Unity.Notifications.TeamsNotifications; +using Volo.Abp.Uow; + +namespace Unity.GrantManager.Web.Middleware; + +public class ExceptionCounterMiddleware( + RequestDelegate next, + ExceptionNotificationThrottle throttle, + ILogger logger) +{ + // Notify only in these environments; add "Staging" if desired + private static readonly HashSet NotifyEnvironments = + new(StringComparer.OrdinalIgnoreCase) { "Production", "Test", "Development" }; + + private static readonly Counter ExceptionCounter = + Metrics.CreateCounter( + "application_exceptions_total", + "Total number of application exceptions", + new CounterConfiguration + { + LabelNames = ["type"] + }); + + // Git SHA baked in at build time via -p:SourceRevisionId= in the Dockerfile. + // Format is "+" e.g. "1.0.0+a3f8c21"; we extract just the SHA. + private static readonly string CommitSha = ParseCommitSha( + typeof(ExceptionCounterMiddleware).Assembly + .GetCustomAttribute()? + .InformationalVersion); + + private static string ParseCommitSha(string? informationalVersion) + { + if (string.IsNullOrWhiteSpace(informationalVersion)) return "unknown"; + var plusIndex = informationalVersion.IndexOf('+'); + return plusIndex >= 0 ? informationalVersion[(plusIndex + 1)..] : informationalVersion; + } + + public async Task InvokeAsync(HttpContext context) + { + try + { + await next(context); + } + catch (Exception ex) + { + ExceptionCounter.WithLabels(ex.GetType().Name).Inc(); + ErrorCountingLoggerSink.ErrorCounter.WithLabels("fatal", ex.GetType().Name).Inc(); + + QueueTeamsNotification(context, ex); + + throw; + } + } + + private void QueueTeamsNotification(HttpContext context, Exception ex) + { + string? env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); + + if (!NotifyEnvironments.Contains(env ?? string.Empty)) + { + return; + } + + if (!throttle.ShouldNotify(ex.GetType().Name)) + { + return; + } + + // Capture values from the request context before it is disposed + string endpoint = $"{context.Request.Method} {context.Request.Path}"; + string exTypeName = ex.GetType().FullName ?? ex.GetType().Name; + string exMessage = ex.Message; + string innerMessage = ex.InnerException?.Message ?? string.Empty; + string stackTrace = ex.StackTrace ?? "(no stack trace)"; + if (stackTrace.Length > 1500) + { + stackTrace = stackTrace[..1500] + "\n... (truncated)"; + } + + // Resolve a scoped INotificationsAppService from a fresh DI scope so + // we can safely use it after the request scope has ended + var scopeFactory = context.RequestServices.GetRequiredService(); + + _ = Task.Run(async () => + { + try + { + await using var scope = scopeFactory.CreateAsyncScope(); + var uowManager = scope.ServiceProvider.GetRequiredService(); + var notifications = scope.ServiceProvider.GetRequiredService(); + + using var uow = uowManager.Begin(requiresNew: true, isTransactional: false); + + string activityTitle = $"[CRITICAL] {ex.GetType().Name}"; + string activitySubtitle = $"Environment: {env} | {endpoint}"; + + var facts = new List + { + new() { Name = "Exception", Value = exTypeName }, + new() { Name = "Message", Value = exMessage }, + new() { Name = "Endpoint", Value = endpoint }, + new() { Name = "Stack Trace", Value = stackTrace }, + new() { Name = "Commit", Value = CommitSha }, + }; + + if (!string.IsNullOrEmpty(innerMessage)) + { + facts.Add(new Fact { Name = "Inner Exception", Value = innerMessage }); + } + + await notifications.PostToTeamsAsync(activityTitle, activitySubtitle, facts); + await uow.CompleteAsync(); + } + catch (Exception notifyEx) + { + logger.LogWarning(notifyEx, "Failed to send Teams exception notification"); + } + }); + } +} + diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ExceptionNotificationThrottle.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ExceptionNotificationThrottle.cs new file mode 100644 index 000000000..c41dc6f91 --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Middleware/ExceptionNotificationThrottle.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; + +namespace Unity.GrantManager.Web.Middleware; + +/// +/// Singleton that tracks per-exception-type cooldowns and a global rate limit +/// to prevent Teams notification storms during an outage. +/// +public sealed class ExceptionNotificationThrottle +{ + // Only send one notification per exception type per cooldown window + private static readonly TimeSpan PerTypeCooldown = TimeSpan.FromMinutes(5); + + // Global cap: at most N notifications per rolling minute across all types + private const int GlobalMaxPerMinute = 5; + + // All state is accessed exclusively under _lock — no concurrent collections needed + private readonly object _lock = new(); + private readonly Dictionary _lastSent = new(); + private int _sentThisMinute; + private DateTimeOffset _windowStart = DateTimeOffset.UtcNow; + + /// + /// Returns true if a Teams notification should be sent for this exception type. + /// Thread-safe. + /// + public bool ShouldNotify(string exceptionTypeName) + { + var now = DateTimeOffset.UtcNow; + + lock (_lock) + { + // Reset the global window if a full minute has elapsed + if (now - _windowStart >= TimeSpan.FromMinutes(1)) + { + _sentThisMinute = 0; + _windowStart = now; + } + + // Per-type cooldown check — inside the lock to prevent concurrent + // callers with the same exception type both passing the check + if (_lastSent.TryGetValue(exceptionTypeName, out var last) && + now - last < PerTypeCooldown) + { + return false; + } + + if (_sentThisMinute >= GlobalMaxPerMinute) + { + return false; + } + + _sentThisMinute++; + _lastSent[exceptionTypeName] = now; + return true; + } + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Program.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Program.cs index 2f3f11cbd..f4e92e7a2 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Program.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Program.cs @@ -5,6 +5,7 @@ using Serilog; using System; using System.Threading.Tasks; +using Unity.GrantManager.Web.Middleware; namespace Unity.GrantManager.Web; @@ -23,7 +24,9 @@ public async static Task Main(string[] args) builder.Host.AddAppSettingsSecretsJson() .UseAutofac() .UseSerilog((hostingContext, loggerConfiguration) => - loggerConfiguration.ReadFrom.Configuration(hostingContext.Configuration)); + loggerConfiguration + .ReadFrom.Configuration(hostingContext.Configuration) + .WriteTo.Sink(new ErrorCountingLoggerSink())); await builder.AddApplicationAsync(); var app = builder.Build(); diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Unity.GrantManager.Web.csproj b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Unity.GrantManager.Web.csproj index f44d2fc83..cc87311b1 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Unity.GrantManager.Web.csproj +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Unity.GrantManager.Web.csproj @@ -76,6 +76,9 @@ + + + diff --git a/applications/Unity.GrantManager/test/Unity.GrantManager.Web.Tests/Identity/InternalNetworkHandlerTests.cs b/applications/Unity.GrantManager/test/Unity.GrantManager.Web.Tests/Identity/InternalNetworkHandlerTests.cs new file mode 100644 index 000000000..0351767ab --- /dev/null +++ b/applications/Unity.GrantManager/test/Unity.GrantManager.Web.Tests/Identity/InternalNetworkHandlerTests.cs @@ -0,0 +1,118 @@ +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using NSubstitute; +using Shouldly; +using Unity.GrantManager.Web.Identity.Policy; +using Xunit; + +namespace Unity.GrantManager.Identity; + +public class InternalNetworkHandlerTests +{ + private static Task BuildContextAsync(IPAddress remoteIp) + { + var httpContext = new DefaultHttpContext(); + httpContext.Connection.RemoteIpAddress = remoteIp; + + var httpContextAccessor = Substitute.For(); + httpContextAccessor.HttpContext.Returns(httpContext); + + var requirement = new InternalNetworkRequirement(); + var authContext = new AuthorizationHandlerContext( + [requirement], + new ClaimsPrincipal(), + null); + + var handler = new InternalNetworkHandler(httpContextAccessor); + return handler.HandleAsync(authContext).ContinueWith(_ => authContext); + } + + [Theory] + [InlineData("127.0.0.1")] // IPv4 loopback + [InlineData("::1")] // IPv6 loopback + [InlineData("10.0.0.1")] // 10/8 start + [InlineData("10.255.255.255")] // 10/8 end + [InlineData("172.16.0.1")] // 172.16/12 start + [InlineData("172.31.255.255")] // 172.16/12 end + [InlineData("192.168.0.1")] // 192.168/16 start + [InlineData("192.168.255.255")] // 192.168/16 end + public async Task Allows_InternalAddresses(string ip) + { + var ctx = await BuildContextAsync(IPAddress.Parse(ip)); + ctx.HasSucceeded.ShouldBeTrue($"{ip} should be allowed"); + } + + [Theory] + [InlineData("8.8.8.8")] // public internet + [InlineData("172.15.255.255")] // just below 172.16/12 + [InlineData("172.32.0.0")] // just above 172.16/12 + [InlineData("192.167.255.255")] // just below 192.168/16 + [InlineData("192.169.0.0")] // just above 192.168/16 + [InlineData("11.0.0.0")] // not 10/8 + [InlineData("203.0.113.1")] // TEST-NET-3 (documentation range) + public async Task Denies_ExternalAddresses(string ip) + { + var ctx = await BuildContextAsync(IPAddress.Parse(ip)); + ctx.HasSucceeded.ShouldBeFalse($"{ip} should be denied"); + } + + [Fact] + public async Task Allows_IPv4MappedToIPv6_Loopback() + { + // ::ffff:127.0.0.1 — loopback mapped into IPv6 + var ip = IPAddress.Parse("::ffff:127.0.0.1"); + var ctx = await BuildContextAsync(ip); + ctx.HasSucceeded.ShouldBeTrue("IPv4-mapped loopback should be allowed"); + } + + [Fact] + public async Task Allows_IPv4MappedToIPv6_PrivateRange() + { + // ::ffff:10.0.0.1 — private range mapped into IPv6 + var ip = IPAddress.Parse("::ffff:10.0.0.1"); + var ctx = await BuildContextAsync(ip); + ctx.HasSucceeded.ShouldBeTrue("IPv4-mapped private address should be allowed"); + } + + [Fact] + public async Task Denies_NullRemoteIp() + { + var httpContext = new DefaultHttpContext(); + // RemoteIpAddress is null by default on DefaultHttpContext + + var httpContextAccessor = Substitute.For(); + httpContextAccessor.HttpContext.Returns(httpContext); + + var requirement = new InternalNetworkRequirement(); + var authContext = new AuthorizationHandlerContext( + [requirement], + new ClaimsPrincipal(), + null); + + var handler = new InternalNetworkHandler(httpContextAccessor); + await handler.HandleAsync(authContext); + + authContext.HasSucceeded.ShouldBeFalse("null remote IP should be denied"); + } + + [Fact] + public async Task Denies_NullHttpContext() + { + var httpContextAccessor = Substitute.For(); + httpContextAccessor.HttpContext.Returns((HttpContext?)null); + + var requirement = new InternalNetworkRequirement(); + var authContext = new AuthorizationHandlerContext( + [requirement], + new ClaimsPrincipal(), + null); + + var handler = new InternalNetworkHandler(httpContextAccessor); + await handler.HandleAsync(authContext); + + authContext.HasSucceeded.ShouldBeFalse("null HttpContext should be denied"); + } +}