diff --git a/library/agent/Agent.test.ts b/library/agent/Agent.test.ts index 8eb6354bd..c7e4fcbe8 100644 --- a/library/agent/Agent.test.ts +++ b/library/agent/Agent.test.ts @@ -1558,3 +1558,102 @@ t.test("it sends heartbeat when shutdown is called", async () => { clock.uninstall(); }); + +t.test("it sends stats for wildcard routes", async () => { + const clock = FakeTimers.install(); + + const logger = new LoggerNoop(); + const api = new ReportingAPIForTesting(); + const agent = createTestAgent({ + api, + logger, + token: new Token("123"), + suppressConsoleLog: false, + }); + agent.start([]); + + agent.getConfig().updateDomains( + [ + { hostname: "*.aikido.dev", mode: "allow" }, + { hostname: "google.com", mode: "block" }, + { hostname: "example.com", mode: "block" }, + { hostname: "aikido.dev", mode: "allow" }, + ], + false + ); + + agent.onConnectHostname("aikido.dev", 443); + agent.onConnectHostname("aikido.dev", 80); + agent.onConnectHostname("google.com", 443); + agent.onConnectHostname("example.com", 443); + agent.onConnectHostname("test.aikido.dev", 443); + agent.onConnectHostname("test.aikido.dev", 80); + agent.onConnectHostname("sub.test.aikido.dev", 80); + agent.onConnectHostname("sub.test.aikido.dev", 443); + + api.clear(); + + await agent.flushStats(1000); + + t.match(api.getEvents(), [ + { + type: "heartbeat", + middlewareInstalled: false, + hostnames: [ + { + hostname: "aikido.dev", + port: 443, + hits: 1, + }, + { + hostname: "aikido.dev", + port: 80, + hits: 1, + }, + { + hostname: "google.com", + port: 443, + hits: 1, + }, + { + hostname: "example.com", + port: 443, + hits: 1, + }, + { + hostname: "test.aikido.dev", + port: 443, + hits: 1, + }, + { + hostname: "test.aikido.dev", + port: 80, + hits: 1, + }, + { + hostname: "*.aikido.dev", + port: 443, + hits: 2, + }, + { + hostname: "*.aikido.dev", + port: 80, + hits: 2, + }, + { + hostname: "sub.test.aikido.dev", + port: 80, + hits: 1, + }, + { + hostname: "sub.test.aikido.dev", + port: 443, + hits: 1, + }, + ], + routes: [], + }, + ]); + + clock.uninstall(); +}); diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index 045588fe2..dbf428c60 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -333,10 +333,10 @@ export class Agent { response.domains && Array.isArray(response.domains) ) { - this.serviceConfig.setBlockNewOutgoingRequests( + this.serviceConfig.updateDomains( + response.domains, response.blockNewOutgoingRequests ); - this.serviceConfig.updateDomains(response.domains); } if ( @@ -608,6 +608,14 @@ export class Agent { onConnectHostname(hostname: string, port: number) { this.hostnames.add(hostname, port); + + // Also report stats for wildcard domains + // e.g. if "sub.example.com" is accessed, we also want to report stats for "*.example.com" + const matchingWildcardDomain = + this.serviceConfig.getMatchingWildcardDomain(hostname); + if (matchingWildcardDomain) { + this.hostnames.add(matchingWildcardDomain.domain, port); + } } onRouteExecute(context: Context) { diff --git a/library/agent/OutgoingDomains.test.ts b/library/agent/OutgoingDomains.test.ts new file mode 100644 index 000000000..970f5f353 --- /dev/null +++ b/library/agent/OutgoingDomains.test.ts @@ -0,0 +1,287 @@ +import * as t from "tap"; +import { OutgoingDomains } from "./OutgoingDomains"; + +t.test("does not block by default", async (t) => { + const outgoingDomains = new OutgoingDomains(); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); +}); + +t.test("blocks domains with block mode", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "blocked.com", mode: "block" }, + ]); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("blocked.com"), true); +}); + +t.test("allows domains with allow mode", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "allowed.com", mode: "allow" }, + ]); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("allowed.com"), false); +}); + +t.test( + "blocks unknown domains when blockNewOutgoingRequests is true", + async (t) => { + const outgoingDomains = new OutgoingDomains([], true); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("unknown.com"), true); + } +); + +t.test( + "allows known domains even when blockNewOutgoingRequests is true", + async (t) => { + const outgoingDomains = new OutgoingDomains( + [{ hostname: "allowed.com", mode: "allow" }], + true + ); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("allowed.com"), false); + } +); + +t.test( + "blocks unknown domains but allows known allowed domains when blockNewOutgoingRequests is true", + async (t) => { + const outgoingDomains = new OutgoingDomains( + [ + { hostname: "allowed.com", mode: "allow" }, + { hostname: "blocked.com", mode: "block" }, + ], + true + ); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("unknown.com"), true); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("allowed.com"), false); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("blocked.com"), true); + } +); + +t.test( + "blocks wildcard domains if new outgoing requests are not blocked", + async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + { hostname: "allowed.com", mode: "allow" }, + ]); + + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.example.com"), + true + ); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("allowed.com"), false); + } +); + +t.test( + "blocks wildcard domains if new outgoing requests are blocked", + async (t) => { + const outgoingDomains = new OutgoingDomains( + [ + { hostname: "*.example.com", mode: "block" }, + { hostname: "allowed.com", mode: "allow" }, + ], + true + ); + + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.example.com"), + true + ); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), true); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("allowed.com"), false); + } +); + +t.test("allows wildcard domains if mode is allow", async (t) => { + const outgoingDomains = new OutgoingDomains( + [ + { hostname: "*.example.com", mode: "allow" }, + { hostname: "blocked.com", mode: "block" }, + ], + true + ); + + t.equal(outgoingDomains.shouldBlockOutgoingRequest("sub.example.com"), false); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), true); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("blocked.com"), true); +}); + +t.test( + "does not block wildcard domains if new outgoing requests are blocked but mode is allow", + async (t) => { + const outgoingDomains = new OutgoingDomains( + [{ hostname: "*.example.com", mode: "allow" }], + true + ); + + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.example.com"), + false + ); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), true); + } +); + +t.test( + "allows multiple levels of subdomains with wildcard domains", + async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + ]); + + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.example.com"), + true + ); + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.sub.example.com"), + true + ); + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.sub.sub.example.com"), + true + ); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); + } +); + +t.test("ignores tld wildcard matches", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.com", mode: "block" }, + ]); + + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("sub.example.com"), false); +}); + +t.test( + "works with empty domain list and blockNewOutgoingRequests false", + async (t) => { + const outgoingDomains = new OutgoingDomains([], false); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); + } +); + +t.test( + "works with empty domain list and blockNewOutgoingRequests true", + async (t) => { + const outgoingDomains = new OutgoingDomains([], true); + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), true); + } +); + +t.test("it does not match root domains with wildcard entries", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + ]); + + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); +}); + +t.test("allows multiple levels of subdomains within blocklist", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.sub.example.com", mode: "block" }, + ]); + + t.equal(outgoingDomains.shouldBlockOutgoingRequest("example.com"), false); + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("test.example.com"), + false + ); + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.sub.example.com"), + true + ); + t.equal( + outgoingDomains.shouldBlockOutgoingRequest("sub.sub.sub.example.com"), + true + ); +}); + +t.test( + "getWildcardMatch returns undefined for non-matching hostname", + async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + ]); + + t.equal(outgoingDomains.getWildcardMatch("example.com"), undefined); + t.equal(outgoingDomains.getWildcardMatch("other.com"), undefined); + t.equal(outgoingDomains.getWildcardMatch("com"), undefined); + } +); + +t.test( + "getWildcardMatch returns domain and mode for matching hostname", + async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + ]); + + t.same(outgoingDomains.getWildcardMatch("sub.example.com"), { + domain: "*.example.com", + mode: "block", + }); + } +); + +t.test("getWildcardMatch returns most specific wildcard match", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + { hostname: "*.sub.example.com", mode: "allow" }, + { hostname: "*.test.example.com", mode: "allow" }, + { hostname: "*.sub.sub.example.com", mode: "allow" }, + ]); + + t.same(outgoingDomains.getWildcardMatch("test.sub.sub.example.com"), { + domain: "*.sub.sub.example.com", + mode: "allow", + }); + t.same(outgoingDomains.getWildcardMatch("api.sub.example.com"), { + domain: "*.sub.example.com", + mode: "allow", + }); + t.same(outgoingDomains.getWildcardMatch("api.test.example.com"), { + domain: "*.test.example.com", + mode: "allow", + }); + t.same(outgoingDomains.getWildcardMatch("foo.bar.example.com"), { + domain: "*.example.com", + mode: "block", + }); + t.same(outgoingDomains.getWildcardMatch("api.example.com"), { + domain: "*.example.com", + mode: "block", + }); +}); + +t.test("getWildcardMatch returns domain and mode for allow mode", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "allow" }, + ]); + + t.same(outgoingDomains.getWildcardMatch("sub.example.com"), { + domain: "*.example.com", + mode: "allow", + }); +}); + +t.test("getWildcardMatch matches deeply nested subdomains", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "*.example.com", mode: "block" }, + ]); + + t.same(outgoingDomains.getWildcardMatch("a.b.c.example.com"), { + domain: "*.example.com", + mode: "block", + }); +}); + +t.test("wildcard rule overrides exact domain rule", async (t) => { + const outgoingDomains = new OutgoingDomains([ + { hostname: "api.example.com", mode: "allow" }, + { hostname: "*.example.com", mode: "block" }, + ]); + + t.equal(outgoingDomains.shouldBlockOutgoingRequest("api.example.com"), true); +}); diff --git a/library/agent/OutgoingDomains.ts b/library/agent/OutgoingDomains.ts new file mode 100644 index 000000000..6afa9d1ed --- /dev/null +++ b/library/agent/OutgoingDomains.ts @@ -0,0 +1,59 @@ +import type { Domain } from "./Config"; + +export class OutgoingDomains { + #domains: Map = new Map(); + #wildcardDomains: Map = new Map(); + #blockNewOutgoingRequests = false; + + constructor( + domains: Domain[] = [], + blockNewOutgoingRequests: boolean = false + ) { + this.#blockNewOutgoingRequests = blockNewOutgoingRequests; + + for (const domain of domains) { + if (domain.hostname.startsWith("*.")) { + this.#wildcardDomains.set(domain.hostname.slice(2), domain.mode); + } else { + this.#domains.set(domain.hostname, domain.mode); + } + } + } + + getWildcardMatch( + hostname: string + ): { domain: string; mode: Domain["mode"] } | undefined { + const parts = hostname.split("."); + if (parts.length <= 2) { + // Only check for wildcard matches if there are at least 3 parts (e.g., sub.example.com) + return undefined; + } + + return parts + .slice(1, -1) + .map((_, index) => { + const suffix = parts.slice(index + 1).join("."); + const mode = this.#wildcardDomains.get(suffix); + return mode !== undefined ? { domain: "*." + suffix, mode } : undefined; + }) + .find((match) => match !== undefined); + } + + shouldBlockOutgoingRequest(hostname: string): boolean { + const wildcardMatch = this.getWildcardMatch(hostname); + if (wildcardMatch !== undefined) { + return wildcardMatch.mode === "block"; + } + + const mode = this.#domains.get(hostname); + + if (this.#blockNewOutgoingRequests) { + // Only allow outgoing requests if the mode is "allow" + // mode is undefined for unknown hostnames, so they get blocked + return mode !== "allow"; + } + + // Only block outgoing requests if the mode is "block" + return mode === "block"; + } +} diff --git a/library/agent/ServiceConfig.test.ts b/library/agent/ServiceConfig.test.ts index ae28efc42..53d6e7e2c 100644 --- a/library/agent/ServiceConfig.test.ts +++ b/library/agent/ServiceConfig.test.ts @@ -400,26 +400,38 @@ t.test("outbound request blocking", async (t) => { t.same(config.shouldBlockOutgoingRequest("example.com"), false); - config.setBlockNewOutgoingRequests(true); + config.updateDomains([], true); t.same(config.shouldBlockOutgoingRequest("example.com"), true); - config.updateDomains([ - { hostname: "example.com", mode: "allow" }, - { hostname: "aikido.dev", mode: "block" }, - ]); + config.updateDomains( + [ + { hostname: "example.com", mode: "allow" }, + { hostname: "aikido.dev", mode: "block" }, + ], + true + ); t.same(config.shouldBlockOutgoingRequest("example.com"), false); t.same(config.shouldBlockOutgoingRequest("aikido.dev"), true); t.same(config.shouldBlockOutgoingRequest("unknown.com"), true); - config.updateDomains([ - { hostname: "example.com", mode: "block" }, - { hostname: "aikido.dev", mode: "allow" }, - ]); + config.updateDomains( + [ + { hostname: "example.com", mode: "block" }, + { hostname: "aikido.dev", mode: "allow" }, + ], + true + ); t.same(config.shouldBlockOutgoingRequest("example.com"), true); t.same(config.shouldBlockOutgoingRequest("aikido.dev"), false); t.same(config.shouldBlockOutgoingRequest("unknown.com"), true); - config.setBlockNewOutgoingRequests(false); + config.updateDomains( + [ + { hostname: "example.com", mode: "block" }, + { hostname: "aikido.dev", mode: "allow" }, + ], + false + ); t.same(config.shouldBlockOutgoingRequest("example.com"), true); t.same(config.shouldBlockOutgoingRequest("aikido.dev"), false); @@ -429,15 +441,25 @@ t.test("outbound request blocking", async (t) => { t.test("outbound request blocking normalizes trailing dots", async (t) => { const config = new ServiceConfig([], 0, [], [], [], []); - config.updateDomains([ - { hostname: "example.com", mode: "block" }, - { hostname: "aikido.dev", mode: "allow" }, - ]); + config.updateDomains( + [ + { hostname: "example.com", mode: "block" }, + { hostname: "aikido.dev", mode: "allow" }, + ], + false + ); t.same(config.shouldBlockOutgoingRequest("example.com."), true); t.same(config.shouldBlockOutgoingRequest("aikido.dev."), false); - config.setBlockNewOutgoingRequests(true); + config.updateDomains( + [ + { hostname: "example.com", mode: "block" }, + { hostname: "aikido.dev", mode: "allow" }, + ], + true + ); + t.same(config.shouldBlockOutgoingRequest("aikido.dev."), false); t.same(config.shouldBlockOutgoingRequest("unknown.com."), true); }); diff --git a/library/agent/ServiceConfig.ts b/library/agent/ServiceConfig.ts index 44964492b..a2fa86373 100644 --- a/library/agent/ServiceConfig.ts +++ b/library/agent/ServiceConfig.ts @@ -4,6 +4,7 @@ import { LimitedContext, matchEndpoints } from "../helpers/matchEndpoints"; import { normalizeHostname } from "../helpers/normalizeHostname"; import { isPrivateIP } from "../vulnerabilities/ssrf/isPrivateIP"; import type { Endpoint, EndpointConfig, Domain } from "./Config"; +import { OutgoingDomains } from "./OutgoingDomains"; import type { IPList, UserAgentDetails } from "./api/FetchListsAPI"; import { safeCreateRegExp } from "./safeCreateRegExp"; @@ -29,8 +30,7 @@ export class ServiceConfig { private monitoredUserAgentRegex: RegExp | undefined; private userAgentDetails: { pattern: RegExp; key: string }[] = []; - private blockNewOutgoingRequests = false; - private domains = new Map(); + private domains = new OutgoingDomains(); private excludedUserIdsFromRateLimiting = new Set(); @@ -288,25 +288,18 @@ export class ServiceConfig { return this.lastUpdatedAt; } - setBlockNewOutgoingRequests(block: boolean) { - this.blockNewOutgoingRequests = block; - } - - updateDomains(domains: Domain[]) { - this.domains = new Map(domains.map((i) => [i.hostname, i.mode])); + updateDomains(domains: Domain[], blockNewOutgoingRequests: boolean) { + this.domains = new OutgoingDomains(domains, blockNewOutgoingRequests); } shouldBlockOutgoingRequest(hostname: string): boolean { - const mode = this.domains.get(normalizeHostname(hostname)); - - if (this.blockNewOutgoingRequests) { - // Only allow outgoing requests if the mode is "allow" - // mode is undefined for unknown hostnames, so they get blocked - return mode !== "allow"; - } + return this.domains.shouldBlockOutgoingRequest(normalizeHostname(hostname)); + } - // Only block outgoing requests if the mode is "block" - return mode === "block"; + getMatchingWildcardDomain( + hostname: string + ): ReturnType { + return this.domains.getWildcardMatch(hostname); } updateUsersExcludedFromRateLimiting(userIds: string[]) { diff --git a/library/sinks/Fetch.test.ts b/library/sinks/Fetch.test.ts index 42d295da2..a22db9bfb 100644 --- a/library/sinks/Fetch.test.ts +++ b/library/sinks/Fetch.test.ts @@ -568,10 +568,13 @@ t.test( ); agent.getHostnames().clear(); - agent.getConfig().updateDomains([ - { hostname: "aikido.dev", mode: "block" }, - { hostname: "app.aikido.dev", mode: "allow" }, - ]); + agent.getConfig().updateDomains( + [ + { hostname: "aikido.dev", mode: "block" }, + { hostname: "app.aikido.dev", mode: "allow" }, + ], + false + ); const blockedError1 = await t.rejects(() => fetch("https://aikido.dev/block") @@ -591,7 +594,13 @@ t.test( { hostname: "app.aikido.dev", port: 443, hits: 1 }, ]); - agent.getConfig().setBlockNewOutgoingRequests(true); + agent.getConfig().updateDomains( + [ + { hostname: "aikido.dev", mode: "block" }, + { hostname: "app.aikido.dev", mode: "allow" }, + ], + true + ); const blockedError2 = await t.rejects(() => fetch("https://example.com")); t.ok(blockedError2 instanceof Error); @@ -609,5 +618,30 @@ t.test( { hostname: "app.aikido.dev", port: 443, hits: 2 }, { hostname: "example.com", port: 443, hits: 1 }, ]); + + agent + .getConfig() + .updateDomains([{ hostname: "*.example.com", mode: "block" }], false); + + const blockedError3 = await t.rejects(() => + fetch("https://sub.example.com") + ); + t.ok(blockedError3 instanceof Error); + if (blockedError3 instanceof Error) { + t.same( + blockedError3.message, + "Zen has blocked an outbound connection: fetch(...) to sub.example.com" + ); + } + + await fetch("https://app.aikido.dev"); + + t.same(agent.getHostnames().asArray(), [ + { hostname: "aikido.dev", port: 443, hits: 1 }, + { hostname: "app.aikido.dev", port: 443, hits: 3 }, + { hostname: "example.com", port: 443, hits: 1 }, + { hostname: "sub.example.com", port: 443, hits: 1 }, + { hostname: "*.example.com", port: 443, hits: 1 }, + ]); } ); diff --git a/library/sinks/HTTPRequest.test.ts b/library/sinks/HTTPRequest.test.ts index b7869304f..fca2f0c98 100644 --- a/library/sinks/HTTPRequest.test.ts +++ b/library/sinks/HTTPRequest.test.ts @@ -376,10 +376,13 @@ t.test("it works", (t) => { ); agent.getHostnames().clear(); - agent.getConfig().updateDomains([ - { hostname: "aikido.dev", mode: "block" }, - { hostname: "app.aikido.dev", mode: "allow" }, - ]); + agent.getConfig().updateDomains( + [ + { hostname: "aikido.dev", mode: "block" }, + { hostname: "app.aikido.dev", mode: "allow" }, + ], + false + ); const blockedError1 = t.throws(() => https.request("https://aikido.dev/block") @@ -399,7 +402,13 @@ t.test("it works", (t) => { { hostname: "app.aikido.dev", port: 443, hits: 1 }, ]); - agent.getConfig().setBlockNewOutgoingRequests(true); + agent.getConfig().updateDomains( + [ + { hostname: "aikido.dev", mode: "block" }, + { hostname: "app.aikido.dev", mode: "allow" }, + ], + true + ); const blockedError2 = t.throws(() => https.request("https://example.com")); if (blockedError2 instanceof Error) {