Skip to content

feat: add ISP, organization, usage type and connection type tracking #490

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Blaumaus wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: add ISP, organization, usage type and connection type tracking #490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 60 additions & 7 deletions backend/apps/cloud/src/analytics/analytics.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1183,7 +1183,16 @@ export class AnalyticsController {
profileId,
)

const { city, region, regionCode, country } = getGeoDetails(ip, errorDTO.tz)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getGeoDetails(ip, errorDTO.tz)

this.analyticsService.checkCountryBlacklist(project, country)

Expand Down Expand Up @@ -1215,6 +1224,10 @@ export class AnalyticsController {
region,
regionCode,
city,
isp,
organization,
userType,
connectionType,
name,
message,
lineno,
Expand Down Expand Up @@ -1298,10 +1311,16 @@ export class AnalyticsController {
}
}

const { city, region, regionCode, country } = getGeoDetails(
ip,
eventsDTO.tz,
)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getGeoDetails(ip, eventsDTO.tz)

this.analyticsService.checkCountryBlacklist(project, country)

Expand Down Expand Up @@ -1355,6 +1374,10 @@ export class AnalyticsController {
region,
regionCode,
city,
isp,
organization,
userType,
connectionType,
eventsDTO.meta,
)

Expand Down Expand Up @@ -1463,7 +1486,16 @@ export class AnalyticsController {
)
}

const { city, region, regionCode, country } = getGeoDetails(ip, logDTO.tz)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getGeoDetails(ip, logDTO.tz)

this.analyticsService.checkCountryBlacklist(project, country)

Expand Down Expand Up @@ -1494,6 +1526,10 @@ export class AnalyticsController {
region,
regionCode,
city,
isp,
organization,
userType,
connectionType,
logDTO.meta,
)

Expand Down Expand Up @@ -1522,6 +1558,10 @@ export class AnalyticsController {
region,
regionCode,
city,
isp,
organization,
userType,
connectionType,
dns,
tls,
conn,
Expand Down Expand Up @@ -1604,7 +1644,16 @@ export class AnalyticsController {
profileId,
)

const { city, region, regionCode, country } = getGeoDetails(ip)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getGeoDetails(ip)

this.analyticsService.checkCountryBlacklist(project, country)

Expand Down Expand Up @@ -1635,6 +1684,10 @@ export class AnalyticsController {
region,
regionCode,
city,
isp,
organization,
userType,
connectionType,
null,
)

Expand Down
14 changes: 13 additions & 1 deletion backend/apps/cloud/src/analytics/analytics.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,19 @@ export const getLowestPossibleTimeBucket = (
return _head(tbMap.tb)
}

const EXCLUDE_NULL_FOR = ['so', 'me', 'ca', 'te', 'co', 'rg', 'ct']
const EXCLUDE_NULL_FOR = [
'so',
'me',
'ca',
'te',
'co',
'rg',
'ct',
'isp',
'og',
'ut',
'ctp',
]

const generateParamsQuery = (
col: string,
Expand Down
32 changes: 32 additions & 0 deletions backend/apps/cloud/src/analytics/utils/transformers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ export const trafficTransformer = (
rg: string | null,
rgc: string | null,
ct: string | null,
isp: string | null,
og: string | null,
ut: string | null,
ctp: string | null,
meta: Record<string, string> | null,
) => {
return {
Expand All @@ -69,6 +73,10 @@ export const trafficTransformer = (
rg: rg || null,
rgc: rgc || null,
ct: ct || null,
isp: isp || null,
og: og || null,
ut: ut || null,
ctp: ctp || null,
...processMetaKV(meta),
created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
}
Expand Down Expand Up @@ -97,6 +105,10 @@ export const customEventTransformer = (
rg: string | null,
rgc: string | null,
ct: string | null,
isp: string | null,
og: string | null,
ut: string | null,
ctp: string | null,
meta: Record<string, string> | null,
) => {
return {
Expand All @@ -122,6 +134,10 @@ export const customEventTransformer = (
rg: rg || null,
rgc: rgc || null,
ct: ct || null,
isp: isp || null,
og: og || null,
ut: ut || null,
ctp: ctp || null,
...processMetaKV(meta),
created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
}
Expand All @@ -144,6 +160,10 @@ export const errorEventTransformer = (
rg: string | null,
rgc: string | null,
ct: string | null,
isp: string | null,
og: string | null,
ut: string | null,
ctp: string | null,
name: string | null,
message: string | null,
lineno: number | null,
Expand All @@ -169,6 +189,10 @@ export const errorEventTransformer = (
rg: rg || null,
rgc: rgc || null,
ct: ct || null,
isp: isp || null,
og: og || null,
ut: ut || null,
ctp: ctp || null,
name: name || null,
message: message || null,
lineno: lineno || null,
Expand All @@ -191,6 +215,10 @@ export const performanceTransformer = (
rg: string | null,
rgc: string | null,
ct: string | null,
isp: string | null,
og: string | null,
ut: string | null,
ctp: string | null,
dns: number,
tls: number,
conn: number,
Expand All @@ -211,6 +239,10 @@ export const performanceTransformer = (
rg: rg || null,
rgc: rgc || null,
ct: ct || null,
isp: isp || null,
og: og || null,
ut: ut || null,
ctp: ctp || null,
dns: _round(dns),
tls: _round(tls),
conn: _round(conn),
Expand Down
23 changes: 22 additions & 1 deletion backend/apps/cloud/src/common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ const TRAFFIC_COLUMNS = [
'cc',
'rg',
'ct',
'isp',
'og',
'ut',
'ctp',
'host',
'pg',
'lc',
Expand All @@ -103,7 +107,20 @@ const TRAFFIC_METAKEY_COLUMNS = ['tag:key', 'tag:value']

const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev', 'entryPage', 'exitPage']
const CAPTCHA_COLUMNS = ['cc', 'br', 'os', 'dv']
const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'host', 'pg', 'dv', 'br', 'brv']
const PERFORMANCE_COLUMNS = [
'cc',
'rg',
'ct',
'isp',
'og',
'ut',
'ctp',
'host',
'pg',
'dv',
'br',
'brv',
]
const ERROR_COLUMNS = [
'host',
'pg',
Expand All @@ -116,6 +133,10 @@ const ERROR_COLUMNS = [
'cc',
'rg',
'ct',
'isp',
'og',
'ut',
'ctp',
]

const sentryIgnoreErrors: (string | RegExp)[] = [
Expand Down
21 changes: 17 additions & 4 deletions backend/apps/cloud/src/common/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,36 +260,49 @@ interface IPGeoDetails {
region: string | null
regionCode: string | null
city: string | null
isp: string | null
organization: string | null
userType: string | null
connectionType: string | null
}

export const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
// Stage 1: Using IP address based geo lookup
const data = lookup.get(ip)

const country = data?.country?.iso_code || null
// TODO: Add city overrides, for example, Colinton -> Edinburgh, etc.
const city = data?.city?.names?.en || null
// TODO: Store ISO code, not region name
const region = data?.subdivisions?.[0]?.names?.en || null
const regionCode = data?.subdivisions?.[0]?.iso_code || null

const isp = data?.traits?.isp || null
const organization = data?.traits?.organization || null
const userType = data?.traits?.user_type || null
const connectionType = data?.traits?.connection_type || null

if (country) {
return {
country,
city,
region,
regionCode,
isp,
organization,
userType,
connectionType,
}
}

// Stage 2: Using timezone based geo lookup as a fallback
const tzCountry = timezones.getCountryForTimezone(tz)?.id || null

return {
country: tzCountry,
city: null,
region: null,
regionCode: null,
isp: null,
organization: null,
userType: null,
connectionType: null,
}
}
Comment on lines 269 to 307
Copy link
Copy Markdown

@coderabbitai coderabbitai bot Mar 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Check if there's documentation or configuration indicating which GeoIP database is used
rg -n -i "ip-geolocation|mmdb|maxmind|geoip" --type=md --type=txt -g '*.env*' -g 'README*' -g 'docker*' 2>/dev/null || echo "No explicit database documentation found"

# Look for any database download/update scripts
fd -e sh -e js -e ts --exec grep -l -i "mmdb\|geoip\|maxmind" {} 2>/dev/null | head -5

Repository: Swetrix/swetrix

Length of output: 700

🏁 Script executed:

# Examine the utils.ts file to understand the lookup mechanism and imports
cat -n backend/apps/cloud/src/common/utils.ts | head -50

Repository: Swetrix/swetrix

Length of output: 1922

🏁 Script executed:

# Check the database sync scripts to confirm which database is being used
head -30 backend/meta/dbip-commercial-sync.js
head -30 backend/meta/dbip-free-sync.js

Repository: Swetrix/swetrix

Length of output: 1930

🏁 Script executed:

# Search for the lookup library initialization and imports
rg -n "lookup\|import.*geoip\|require.*geoip" backend/apps/cloud/src/common/utils.ts -B 3 -A 3

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check which sync script is executed for the cloud app
rg -n "dbip-commercial-sync|dbip-free-sync" --type=js --type=ts backend/

Repository: Swetrix/swetrix

Length of output: 189

🏁 Script executed:

# Look for environment variables or conditions that determine which database is used
rg -n "GEOIP|DBIP|DATABASE" --type=ts -g "*.env*" backend/ 2>/dev/null || rg -n "GEOIP|DBIP|DATABASE" backend/ -g "*.env*"

Repository: Swetrix/swetrix

Length of output: 161

🏁 Script executed:

# Check the lookup initialization to understand the database setup
rg -n "lookup.*=" backend/apps/cloud/src/common/utils.ts -B 5 -A 5 | head -40

Repository: Swetrix/swetrix

Length of output: 659

🏁 Script executed:

# Verify the maxmind package documentation or type definitions available
rg -n "CityResponse\|traits" backend/apps/cloud/src/common/utils.ts -B 2 -A 2

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check if there's a production configuration that specifies which sync script is used
rg -n "dbip-commercial\|dbip-free" --type=json --type=ts --type=js backend/ -g "package.json" -g "docker*" -g "*.yml" 2>/dev/null | head -20

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check the README to understand which database cloud app uses
cat README.md | grep -i "geo\|geoip\|db-ip" -A 3 -B 3

Repository: Swetrix/swetrix

Length of output: 3699

🏁 Script executed:

# Check docker or deployment configs for cloud app to see which database is used
fd docker -o docker-compose | xargs cat 2>/dev/null | grep -i "geoip\|dbip\|database" -B 2 -A 2 | head -30

Repository: Swetrix/swetrix

Length of output: 206

🏁 Script executed:

# Look at what the DB-IP databases actually include
# Check if there's any documentation in the codebase about database structure
rg -n "isp\|organization\|user_type\|connection_type" backend/apps/cloud/src/common/ -B 2 -A 2 | head -50

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check if there's deployment configuration or package.json that shows which database is used
cat backend/package.json | grep -i "maxmind\|geoip"

Repository: Swetrix/swetrix

Length of output: 83

🏁 Script executed:

# Look at the dbip-commercial-sync.js to understand what database it downloads
cat backend/meta/dbip-commercial-sync.js | grep -i "download\|url\|api" | head -20

Repository: Swetrix/swetrix

Length of output: 1143

🏁 Script executed:

# Check if there's any validation or warning about null traits
rg -n "isp\|organization\|user_type\|connection_type" backend/apps/cloud/src/common/utils.ts -B 5 -A 5

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Look for any comments or documentation about database requirements
rg -n "TODO\|FIXME\|NOTE\|WARN" backend/apps/cloud/src/common/utils.ts | head -20

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check what the actual return type includes and if traits are documented
ast-grep --pattern 'interface IPGeoDetails {
  $$$
}'

Repository: Swetrix/swetrix

Length of output: 1137

🏁 Script executed:

# Check the full dbip-commercial-sync.js to see what exact database it downloads
cat backend/meta/dbip-commercial-sync.js | tail -40

Repository: Swetrix/swetrix

Length of output: 1152

🏁 Script executed:

# Verify what API URL is being used for the commercial database
rg -n "db-ip.com" backend/meta/dbip-commercial-sync.js -B 2 -A 2

Repository: Swetrix/swetrix

Length of output: 302

🏁 Script executed:

# Check if there's any environment configuration or deployment scripts that specify which sync to run
find backend -name "*.yml" -o -name "docker*" -o -name "Dockerfile*" 2>/dev/null | xargs grep -l "dbip\|geoip" 2>/dev/null

Repository: Swetrix/swetrix

Length of output: 77

🏁 Script executed:

# Check the community app's database setup for comparison
cat backend/apps/community/src/common/utils.ts | grep -n "isp\|organization" -B 10 -A 10 | head -50

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check the Dockerfile to see which database sync script is used for cloud
cat backend/Dockerfile | grep -i "dbip\|geoip\|sync" -B 3 -A 3

Repository: Swetrix/swetrix

Length of output: 201

🏁 Script executed:

# Check if there's deployment documentation that clarifies which database is used
rg -n "db-ip\|dbip\|geoip\|premium" --type=md backend/docs/ 2>/dev/null || echo "No docs directory"

Repository: Swetrix/swetrix

Length of output: 76

🏁 Script executed:

# Confirm the actual behavior - check if traits are accessed elsewhere in cloud app
rg -n "\.isp\|\.organization\|\.userType\|\.connectionType" backend/apps/cloud/src/ -l

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Verify that community app doesn't use these traits since it uses free DB-IP
rg -n "\.isp\|\.organization\|\.userType\|\.connectionType" backend/apps/community/src/

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Check if there's a separate deployment config for cloud app
rg -n "deploy:cloud\|cloud.*deploy\|GEOIP" backend/package.json -A 5 -B 5

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Look for any Docker or deployment configs specific to cloud app
fd dockerfile -o docker-compose -o yml | xargs cat 2>/dev/null | grep -i "cloud\|commercial" -B 5 -A 5

Repository: Swetrix/swetrix

Length of output: 206

🏁 Script executed:

# Check package.json scripts to see all deployment targets
cat backend/package.json | grep -i "deploy\|script" -A 50 | head -60

Repository: Swetrix/swetrix

Length of output: 3046

🏁 Script executed:

# Check what database file path is actually used and confirm the setup
rg -n "PRODUCTION_GEOIP_DB_PATH\|DEVELOPMENT_GEOIP_DB_PATH\|ip-geolocation-db" backend/apps/cloud/src/common/utils.ts -B 3 -A 3

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Verify the difference between what community and cloud extract from the database
diff <(rg -n "export const getGeoDetails" backend/apps/community/src/common/utils.ts -A 30) <(rg -n "export const getGeoDetails" backend/apps/cloud/src/common/utils.ts -A 30)

Repository: Swetrix/swetrix

Length of output: 146

🏁 Script executed:

# Check if there's any production deployment configuration
find . -name "*.env*" -o -name "docker-compose*" 2>/dev/null | xargs grep -l "dbip\|geoip\|cloud" 2>/dev/null | head -5

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Search for where PRODUCTION_GEOIP_DB_PATH is defined
rg -n "PRODUCTION_GEOIP_DB_PATH\|DEVELOPMENT_GEOIP_DB_PATH" backend/apps/cloud/src/common/utils.ts -B 5

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Compare the getGeoDetails functions between community and cloud
cat backend/apps/community/src/common/utils.ts | sed -n '1100,1200p'

Repository: Swetrix/swetrix

Length of output: 2762

🏁 Script executed:

# Check the full getGeoDetails in cloud to see the complete picture
cat backend/apps/cloud/src/common/utils.ts | sed -n '269,310p'

Repository: Swetrix/swetrix

Length of output: 1072

🏁 Script executed:

# Search for any initialization or documentation about database types
rg -n "CityResponse\|Reader" backend/apps/cloud/src/common/utils.ts -B 3 -A 3

Repository: Swetrix/swetrix

Length of output: 41

🏁 Script executed:

# Get the exact line numbers and context for the cloud app's getGeoDetails function
cat -n backend/apps/cloud/src/common/utils.ts | sed -n '258,310p'

Repository: Swetrix/swetrix

Length of output: 1679

🏁 Script executed:

# Verify the maxmind Reader type to confirm what traits it supports
rg -n "traits" backend/apps/cloud/src/common/utils.ts -B 2 -A 2

Repository: Swetrix/swetrix

Length of output: 384

🏁 Script executed:

# Double-check what database the Dockerfile uses for cloud specifically
cat backend/Dockerfile | grep -n "deploy\|dbip\|node" | head -20

Repository: Swetrix/swetrix

Length of output: 246

Database mismatch: traits will always be null with current DB-IP free database configuration.

The code attempts to extract isp, organization, user_type, and connection_type traits from the GeoIP database. However, the Dockerfile currently runs only dbip-free-sync.js, which downloads the free DB-IP City Lite database that does not include ISP/organization traits. These fields will always be null in production.

Either:

  1. Use dbip-commercial-sync.js instead (requires DB-IP API key) to get a database with trait support, or
  2. Remove the trait extraction logic (lines 277–280, 288–291, 303–305) if ISP data is not needed.

Note: The code uses the DB-IP library and database, not MaxMind.

🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@backend/apps/cloud/src/common/utils.ts` around lines 269 - 307, getGeoDetails
currently extracts ISP/organization/userType/connectionType traits (isp,
organization, userType, connectionType) but the deployed DB-IP City Lite
(dbip-free-sync.js) does not contain those traits so they'll always be null; fix
by either switching the sync script to dbip-commercial-sync.js and wiring the
DB-IP API key so the commercial DB with traits is used, or remove the trait
extraction and related return fields from getGeoDetails (and any upstream
consumers) so the function only returns country, city, region, and regionCode
when using the free DB; update callers/types accordingly (refer to getGeoDetails
and the trait variables isp/organization/userType/connectionType).


Expand Down
2 changes: 1 addition & 1 deletion backend/meta/dbip-commercial-sync.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ function followableGet(url, options = {}, maxRedirects = 5) {
}

async function fetchDbipAccountInfo(apiKey) {
const metadataUrl = `https://db-ip.com/account/${apiKey}/db/ip-to-location/`
const metadataUrl = `https://db-ip.com/account/${apiKey}/db/ip-to-location-isp/`
const response = await followableGet(metadataUrl, {
headers: { Accept: 'application/json' },
})
Expand Down
12 changes: 12 additions & 0 deletions backend/migrations/clickhouse/2026_03_19_isp_network.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
const { queriesRunner, dbName } = require('./setup')

const tables = ['analytics', 'customEV', 'performance', 'errors']

const queries = tables.flatMap((table) => [
`ALTER TABLE ${dbName}.${table} ADD COLUMN IF NOT EXISTS isp LowCardinality(Nullable(String)) AFTER ct`,
`ALTER TABLE ${dbName}.${table} ADD COLUMN IF NOT EXISTS og Nullable(String) CODEC(ZSTD(3)) AFTER isp`,
`ALTER TABLE ${dbName}.${table} ADD COLUMN IF NOT EXISTS ut LowCardinality(Nullable(String)) AFTER og`,
`ALTER TABLE ${dbName}.${table} ADD COLUMN IF NOT EXISTS ctp LowCardinality(Nullable(String)) AFTER ut`,
])

queriesRunner(queries)
Loading
Loading