diff --git a/AGENTS.md b/AGENTS.md index 74c5f6a..831b6f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ control plane, or runtime AI filter. - `src/Bower.Analytics`: telemetry quality and coverage scoring. - `src/Bower.Source.Ama`: Azure Monitor Agent companion discovery and custom log mapping. - `src/Bower.Dcr`: Data Collection Rule optimiser and health assessment. +- `src/Bower.Agent.Aws`: native EC2 host telemetry agent with IMDS enrichment. - `schemas`, `policies`, `deploy`, `docs`, `tests`: versioned product assets. Inspect nearest `AGENTS.md` before editing. diff --git a/Bower.sln b/Bower.sln index 9076a20..767bba7 100644 --- a/Bower.sln +++ b/Bower.sln @@ -49,6 +49,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Source.Ama", "src\Bow EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Dcr", "src\Bower.Dcr\Bower.Dcr.csproj", "{C6FDDD4F-022B-4431-85DF-839A7DEF1263}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Agent.Aws", "src\Bower.Agent.Aws\Bower.Agent.Aws.csproj", "{A240528B-2792-4CEB-8A84-1CB773CA4BEC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -323,6 +325,18 @@ Global {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x64.Build.0 = Release|Any CPU {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x86.ActiveCfg = Release|Any CPU {C6FDDD4F-022B-4431-85DF-839A7DEF1263}.Release|x86.Build.0 = Release|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Debug|x64.ActiveCfg = Debug|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Debug|x64.Build.0 = Debug|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Debug|x86.ActiveCfg = Debug|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Debug|x86.Build.0 = Debug|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Release|Any CPU.Build.0 = Release|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Release|x64.ActiveCfg = Release|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Release|x64.Build.0 = Release|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Release|x86.ActiveCfg = Release|Any CPU + {A240528B-2792-4CEB-8A84-1CB773CA4BEC}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -337,5 +351,6 @@ Global {F1BFA9A3-4762-47C4-B97D-578A105A8D6C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {A876B3F7-3209-4E9E-8993-2D9FF55637F2} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {C6FDDD4F-022B-4431-85DF-839A7DEF1263} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A240528B-2792-4CEB-8A84-1CB773CA4BEC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/src/Bower.Agent.Aws/AGENTS.md b/src/Bower.Agent.Aws/AGENTS.md new file mode 100644 index 0000000..03f05f2 --- /dev/null +++ b/src/Bower.Agent.Aws/AGENTS.md @@ -0,0 +1,5 @@ +# AWS EC2 agent instructions + +Host collectors stay read-only and path-bounded. Metadata enrichment uses IMDSv2 +when available but unit tests must inject metadata. Never log raw host payloads +or credentials. diff --git a/src/Bower.Agent.Aws/Bower.Agent.Aws.csproj b/src/Bower.Agent.Aws/Bower.Agent.Aws.csproj new file mode 100644 index 0000000..03024fd --- /dev/null +++ b/src/Bower.Agent.Aws/Bower.Agent.Aws.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/Bower.Agent.Aws/Ec2Agent.cs b/src/Bower.Agent.Aws/Ec2Agent.cs new file mode 100644 index 0000000..67581b4 --- /dev/null +++ b/src/Bower.Agent.Aws/Ec2Agent.cs @@ -0,0 +1,391 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Bower.Contracts; + +namespace Bower.Agent.Aws; + +public sealed record Ec2InstanceMetadata( + string InstanceId, + string AccountId, + string Region, + string AvailabilityZone, + string? VpcId, + string? SubnetId, + string? AmiId, + IReadOnlyList SecurityGroups, + IReadOnlyDictionary Tags, + string? AutoScalingGroup, + string? EcsCluster, + string? EksCluster); + +public enum HostLogKind +{ + WindowsEvent, + Sysmon, + PowerShell, + Syslog, + Journald, + Auditd, + Auth, + Docker, + Custom +} + +public sealed record HostLogSource( + string Id, + HostLogKind Kind, + string Path, + bool Enabled = true); + +public sealed record Ec2AgentOptions +{ + public required string AgentId { get; init; } + + public string Environment { get; init; } = "production"; + + public string ApplicationName { get; init; } = "bower-ec2-agent"; + + public IReadOnlyList Sources { get; init; } = []; + + public int MaximumLineBytes { get; init; } = 65_536; + + public int MaximumBatchLines { get; init; } = 1_000; + + public void Validate() + { + ArgumentException.ThrowIfNullOrWhiteSpace(AgentId); + if (AgentId.Length > 128) + { + throw new ArgumentException("Agent id cannot exceed 128 characters."); + } + + if (MaximumLineBytes is < 256 or > 1_048_576) + { + throw new ArgumentOutOfRangeException(nameof(MaximumLineBytes)); + } + + if (MaximumBatchLines is < 1 or > 10_000) + { + throw new ArgumentOutOfRangeException(nameof(MaximumBatchLines)); + } + + foreach (HostLogSource source in Sources) + { + ArgumentException.ThrowIfNullOrWhiteSpace(source.Id); + ArgumentException.ThrowIfNullOrWhiteSpace(source.Path); + if (source.Path.Contains("..", StringComparison.Ordinal)) + { + throw new ArgumentException($"Source path for '{source.Id}' must not contain '..'."); + } + } + } +} + +public sealed class Ec2MetadataClient +{ + private readonly Func> fetch; + + public Ec2MetadataClient(Func>? fetch = null) + { + this.fetch = fetch ?? (_ => Task.FromResult(null)); + } + + public async Task GetAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + string? instanceId = await fetch("instance-id").ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(instanceId)) + { + return null; + } + + string accountId = await fetch("identity-account-id").ConfigureAwait(false) ?? "000000000000"; + string region = await fetch("region").ConfigureAwait(false) ?? "unknown"; + string az = await fetch("availability-zone").ConfigureAwait(false) ?? region; + string? vpc = await fetch("vpc-id").ConfigureAwait(false); + string? subnet = await fetch("subnet-id").ConfigureAwait(false); + string? ami = await fetch("ami-id").ConfigureAwait(false); + string? securityGroupsRaw = await fetch("security-groups").ConfigureAwait(false); + string? tagsRaw = await fetch("tags").ConfigureAwait(false); + string? asg = await fetch("auto-scaling-group").ConfigureAwait(false); + string? ecs = await fetch("ecs-cluster").ConfigureAwait(false); + string? eks = await fetch("eks-cluster").ConfigureAwait(false); + + Dictionary tags = new(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(tagsRaw)) + { + try + { + using JsonDocument document = JsonDocument.Parse(tagsRaw); + if (document.RootElement.ValueKind == JsonValueKind.Object) + { + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + { + tags[property.Name] = property.Value.ToString(); + } + } + } + catch (JsonException) + { + // ignore malformed tags payload + } + } + + string[] securityGroups = string.IsNullOrWhiteSpace(securityGroupsRaw) + ? [] + : securityGroupsRaw.Split( + [',', '\n', ' '], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + return new Ec2InstanceMetadata( + instanceId, + accountId, + region, + az, + vpc, + subnet, + ami, + securityGroups, + tags, + asg, + ecs, + eks); + } +} + +public sealed class Ec2HostCollector +{ + private readonly Ec2AgentOptions options; + private readonly Ec2InstanceMetadata? metadata; + + public Ec2HostCollector(Ec2AgentOptions options, Ec2InstanceMetadata? metadata = null) + { + ArgumentNullException.ThrowIfNull(options); + options.Validate(); + this.options = options; + this.metadata = metadata; + } + + public IReadOnlyList CollectLines( + string sourceId, + IEnumerable lines, + DateTimeOffset? observedAt = null) + { + HostLogSource source = options.Sources.FirstOrDefault(item => + string.Equals(item.Id, sourceId, StringComparison.Ordinal)) + ?? throw new ArgumentException($"Unknown host source '{sourceId}'."); + + if (!source.Enabled) + { + return []; + } + + DateTimeOffset observed = observedAt ?? DateTimeOffset.UtcNow; + List events = []; + int index = 0; + foreach (string line in lines) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + if (events.Count >= options.MaximumBatchLines) + { + throw new InvalidOperationException( + $"Host source '{sourceId}' exceeded maximum batch of {options.MaximumBatchLines} lines."); + } + + int size = Encoding.UTF8.GetByteCount(line); + if (size > options.MaximumLineBytes) + { + throw new InvalidOperationException( + $"Host source '{sourceId}' line is {size} bytes; maximum is {options.MaximumLineBytes}."); + } + + string originalId = Convert.ToHexString( + SHA256.HashData(Encoding.UTF8.GetBytes($"{sourceId}\u001f{index}\u001f{line}"))) + .ToLowerInvariant()[..24]; + index++; + + Dictionary labels = new(StringComparer.Ordinal) + { + ["host.sourceId"] = source.Id, + ["host.kind"] = source.Kind.ToString(), + ["host.path"] = source.Path, + ["host.linePreview"] = line.Length <= 256 ? line : line[..256] + }; + EnrichLabels(labels); + + events.Add( + new SecurityEventEnvelope + { + SchemaVersion = SecurityEventEnvelope.CurrentSchemaVersion, + EventId = Guid.CreateVersion7().ToString(), + EventOriginalId = originalId, + TimeGenerated = observed, + TimeObserved = observed, + EventCategory = MapCategory(source.Kind, line), + EventType = MapEventType(source.Kind), + EventAction = MapAction(source.Kind, line), + EventResult = MapResult(line), + EventSeverity = MapSeverity(line), + Application = new ApplicationContext + { + Name = options.ApplicationName, + Environment = options.Environment, + Instance = metadata?.InstanceId, + TenantId = metadata?.AccountId + }, + Source = new SourceContext + { + Hostname = metadata?.InstanceId + }, + Target = new TargetContext + { + Type = "host-log", + Name = source.Path + }, + Collector = new CollectorContext + { + Id = options.AgentId, + Version = "0.1.0", + SourceAdapter = "aws.ec2-agent", + ConfigurationHash = originalId[..16], + ReceivedAt = observed + }, + Labels = labels + }); + } + + return events; + } + + public static IReadOnlyList DefaultSourcesForPlatform(bool windows) + { + return windows + ? + [ + new HostLogSource("windows-security", HostLogKind.WindowsEvent, "Security"), + new HostLogSource("sysmon", HostLogKind.Sysmon, "Microsoft-Windows-Sysmon/Operational"), + new HostLogSource("powershell", HostLogKind.PowerShell, "Microsoft-Windows-PowerShell/Operational") + ] + : + [ + new HostLogSource("syslog", HostLogKind.Syslog, "/var/log/syslog"), + new HostLogSource("auth", HostLogKind.Auth, "/var/log/auth.log"), + new HostLogSource("auditd", HostLogKind.Auditd, "/var/log/audit/audit.log"), + new HostLogSource("docker", HostLogKind.Docker, "/var/lib/docker/containers") + ]; + } + + private void EnrichLabels(Dictionary labels) + { + if (metadata is null) + { + return; + } + + labels["aws.instanceId"] = metadata.InstanceId; + labels["aws.accountId"] = metadata.AccountId; + labels["aws.region"] = metadata.Region; + labels["aws.availabilityZone"] = metadata.AvailabilityZone; + if (metadata.VpcId is not null) labels["aws.vpcId"] = metadata.VpcId; + if (metadata.SubnetId is not null) labels["aws.subnetId"] = metadata.SubnetId; + if (metadata.AmiId is not null) labels["aws.amiId"] = metadata.AmiId; + if (metadata.SecurityGroups.Count > 0) + { + labels["aws.securityGroups"] = string.Join(',', metadata.SecurityGroups); + } + + if (metadata.AutoScalingGroup is not null) + { + labels["aws.autoScalingGroup"] = metadata.AutoScalingGroup; + } + + if (metadata.EcsCluster is not null) labels["aws.ecsCluster"] = metadata.EcsCluster; + if (metadata.EksCluster is not null) labels["aws.eksCluster"] = metadata.EksCluster; + foreach ((string key, string value) in metadata.Tags) + { + labels[$"aws.tag.{key}"] = value; + } + } + + private static string MapCategory(HostLogKind kind, string line) + { + if (line.Contains("Failed password", StringComparison.OrdinalIgnoreCase) + || line.Contains("4625", StringComparison.Ordinal)) + { + return SecurityEventCategories.Authentication; + } + + return kind switch + { + HostLogKind.Auth => SecurityEventCategories.Authentication, + HostLogKind.Auditd => SecurityEventCategories.AdministrativeActivity, + HostLogKind.Docker => SecurityEventCategories.ApplicationSecurity, + _ => SecurityEventCategories.ApplicationSecurity + }; + } + + private static string MapEventType(HostLogKind kind) + { + return kind switch + { + HostLogKind.WindowsEvent => "host_windows_event", + HostLogKind.Sysmon => "host_sysmon", + HostLogKind.PowerShell => "host_powershell", + HostLogKind.Syslog => "host_syslog", + HostLogKind.Journald => "host_journald", + HostLogKind.Auditd => "host_auditd", + HostLogKind.Auth => "host_auth", + HostLogKind.Docker => "host_docker", + _ => "host_custom" + }; + } + + private static string MapAction(HostLogKind kind, string line) + { + if (line.Contains("Failed password", StringComparison.OrdinalIgnoreCase)) + { + return "authentication.failure"; + } + + return kind.ToString().ToLowerInvariant() + ".event"; + } + + private static EventResult MapResult(string line) + { + if (line.Contains("Failed", StringComparison.OrdinalIgnoreCase) + || line.Contains("denied", StringComparison.OrdinalIgnoreCase) + || line.Contains("4625", StringComparison.Ordinal)) + { + return EventResult.Failure; + } + + if (line.Contains("Accepted", StringComparison.OrdinalIgnoreCase) + || line.Contains("4624", StringComparison.Ordinal)) + { + return EventResult.Success; + } + + return EventResult.Unknown; + } + + private static EventSeverity MapSeverity(string line) + { + if (line.Contains("critical", StringComparison.OrdinalIgnoreCase)) + { + return EventSeverity.Critical; + } + + if (line.Contains("Failed", StringComparison.OrdinalIgnoreCase) + || line.Contains("error", StringComparison.OrdinalIgnoreCase)) + { + return EventSeverity.Medium; + } + + return EventSeverity.Low; + } +} diff --git a/src/Bower.Agent.Aws/packages.lock.json b/src/Bower.Agent.Aws/packages.lock.json new file mode 100644 index 0000000..2088961 --- /dev/null +++ b/src/Bower.Agent.Aws/packages.lock.json @@ -0,0 +1,23 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "bower.abstractions": { + "type": "Project", + "dependencies": { + "Bower.Contracts": "[1.0.0, )" + } + }, + "bower.contracts": { + "type": "Project" + }, + "bower.source.aws": { + "type": "Project", + "dependencies": { + "Bower.Abstractions": "[1.0.0, )", + "Bower.Contracts": "[1.0.0, )" + } + } + } + } +} \ No newline at end of file diff --git a/tests/Bower.UnitTests/Bower.UnitTests.csproj b/tests/Bower.UnitTests/Bower.UnitTests.csproj index 6d9ec96..edc0d95 100644 --- a/tests/Bower.UnitTests/Bower.UnitTests.csproj +++ b/tests/Bower.UnitTests/Bower.UnitTests.csproj @@ -39,5 +39,6 @@ + diff --git a/tests/Bower.UnitTests/Ec2HostCollectorTests.cs b/tests/Bower.UnitTests/Ec2HostCollectorTests.cs new file mode 100644 index 0000000..88bed70 --- /dev/null +++ b/tests/Bower.UnitTests/Ec2HostCollectorTests.cs @@ -0,0 +1,93 @@ +using Bower.Agent.Aws; +using Bower.Contracts; + +namespace Bower.UnitTests; + +public sealed class Ec2HostCollectorTests +{ + [Fact] + public void Options_RejectPathTraversal() + { + Ec2AgentOptions options = new() + { + AgentId = "agent-1", + Sources = [new HostLogSource("x", HostLogKind.Custom, "../etc/passwd")] + }; + + Assert.Throws(options.Validate); + } + + [Fact] + public async Task MetadataClient_MapsInjectedImdsValues() + { + Ec2MetadataClient client = new(path => path switch + { + "instance-id" => Task.FromResult("i-abc"), + "identity-account-id" => Task.FromResult("123456789012"), + "region" => Task.FromResult("ap-southeast-2"), + "availability-zone" => Task.FromResult("ap-southeast-2a"), + "vpc-id" => Task.FromResult("vpc-1"), + "subnet-id" => Task.FromResult("subnet-1"), + "ami-id" => Task.FromResult("ami-1"), + "security-groups" => Task.FromResult("sg-1,sg-2"), + "tags" => Task.FromResult("""{"Name":"web","Env":"prod"}"""), + "auto-scaling-group" => Task.FromResult("asg-web"), + "ecs-cluster" => Task.FromResult("ecs-1"), + "eks-cluster" => Task.FromResult("eks-1"), + _ => Task.FromResult(null) + }); + + Ec2InstanceMetadata? metadata = await client.GetAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(metadata); + Assert.Equal("i-abc", metadata.InstanceId); + Assert.Equal("123456789012", metadata.AccountId); + Assert.Equal(["sg-1", "sg-2"], metadata.SecurityGroups); + Assert.Equal("web", metadata.Tags["Name"]); + Assert.Equal("asg-web", metadata.AutoScalingGroup); + } + + [Fact] + public void CollectLines_EnrichesWithMetadataAndMapsAuthFailure() + { + Ec2InstanceMetadata metadata = new( + "i-abc", + "123456789012", + "ap-southeast-2", + "ap-southeast-2a", + "vpc-1", + "subnet-1", + "ami-1", + ["sg-1"], + new Dictionary { ["Name"] = "web" }, + "asg-web", + null, + null); + + Ec2HostCollector collector = new( + new Ec2AgentOptions + { + AgentId = "agent-1", + Sources = [new HostLogSource("auth", HostLogKind.Auth, "/var/log/auth.log")] + }, + metadata); + + SecurityEventEnvelope envelope = Assert.Single( + collector.CollectLines("auth", ["Failed password for root from 203.0.113.10"])); + + Assert.Equal("host_auth", envelope.EventType); + Assert.Equal(EventResult.Failure, envelope.EventResult); + Assert.Equal("aws.ec2-agent", envelope.Collector?.SourceAdapter); + Assert.Equal("i-abc", envelope.Labels?["aws.instanceId"]); + Assert.Equal("123456789012", envelope.Labels?["aws.accountId"]); + Assert.Equal("web", envelope.Labels?["aws.tag.Name"]); + Assert.Equal(SecurityEventCategories.Authentication, envelope.EventCategory); + } + + [Fact] + public void DefaultSources_CoverWindowsAndLinux() + { + Assert.Contains(Ec2HostCollector.DefaultSourcesForPlatform(windows: true), item => item.Kind == HostLogKind.Sysmon); + Assert.Contains(Ec2HostCollector.DefaultSourcesForPlatform(windows: false), item => item.Kind == HostLogKind.Auditd); + } +} diff --git a/tests/Bower.UnitTests/packages.lock.json b/tests/Bower.UnitTests/packages.lock.json index ec52e2f..218dfb0 100644 --- a/tests/Bower.UnitTests/packages.lock.json +++ b/tests/Bower.UnitTests/packages.lock.json @@ -414,6 +414,13 @@ "Bower.Contracts": "[1.0.0, )" } }, + "bower.agent.aws": { + "type": "Project", + "dependencies": { + "Bower.Contracts": "[1.0.0, )", + "Bower.Source.Aws": "[1.0.0, )" + } + }, "bower.analytics": { "type": "Project", "dependencies": {