From 3173db0aa7b26aa2fede00ab5ffa59088bf415a4 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 19 Dec 2025 10:45:58 -0600 Subject: [PATCH 01/21] feat(threatwinds): add ThreatWinds credentials section and parameters to configuration --- .../shared_types/enums/SectionType.java | 3 +- ...insert_threatwinds_credentials_section.xml | 45 +++++++++++++++++++ .../resources/config/liquibase/master.xml | 2 + 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml diff --git a/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java b/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java index f3aec586c..a2a2ecb4f 100644 --- a/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java +++ b/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java @@ -5,6 +5,7 @@ public enum SectionType { EMAIL, TFA, ALERTS, - DATE_SETTINGS + DATE_SETTINGS, + THREATWINDS_CREDENTIALS } diff --git a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml new file mode 100644 index 000000000..47c95121b --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index f6ce831a6..8f778706c 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -113,5 +113,7 @@ + + From b0ec2bd88f344da27b06fce145010a6291abd572 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 23 Dec 2025 10:59:28 -0500 Subject: [PATCH 02/21] ci: add ThreadWinds ingestion build job to deployment pipeline --- .github/workflows/v10-deployment-pipeline.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/v10-deployment-pipeline.yml b/.github/workflows/v10-deployment-pipeline.yml index 3e6c9c17c..943c5d0fe 100644 --- a/.github/workflows/v10-deployment-pipeline.yml +++ b/.github/workflows/v10-deployment-pipeline.yml @@ -249,6 +249,15 @@ jobs: image_name: sophos tag: ${{ needs.setup_deployment.outputs.tag }} + build_threadwinds_ingestion: + name: Build Threadwinds-Ingestion Microservice + needs: [validations,setup_deployment] + if: ${{ needs.setup_deployment.outputs.tag != '' }} + uses: ./.github/workflows/reusable-golang.yml + with: + image_name: threadwinds-ingestion + tag: ${{ needs.setup_deployment.outputs.tag }} + build_user_auditor: name: Build User-Auditor Microservice needs: [validations,setup_deployment] @@ -281,6 +290,7 @@ jobs: build_aws, build_backend, build_correlation, build_frontend, build_bitdefender, build_mutate, build_office365, build_log_auth_proxy, build_soc_ai, build_sophos, + build_threadwinds_ingestion, build_user_auditor, build_web_pdf ] if: ${{ needs.setup_deployment.outputs.tag != '' }} From 40f1c14ae876c7616afe0b3d624c957eaf67da89 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 23 Dec 2025 11:00:23 -0500 Subject: [PATCH 03/21] feat: integrate ThreadWinds ingestion service into UTMStack installer --- installer/types/compose.go | 31 +++++++++++++++++++++++++++++++ installer/types/stack.go | 1 + 2 files changed, 32 insertions(+) diff --git a/installer/types/compose.go b/installer/types/compose.go index 49da51178..f768ed1f8 100644 --- a/installer/types/compose.go +++ b/installer/types/compose.go @@ -596,6 +596,37 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { }, } + threadwindsIngestionMem := stack.ServiceResources["threadwinds-ingestion"].AssignedMemory + c.Services["threadwinds-ingestion"] = Service{ + Image: utils.Str("ghcr.io/utmstack/utmstack/threadwinds-ingestion:" + conf.Branch), + DependsOn: []string{ + "postgres", + "node1", + "backend", + }, + Environment: []string{ + "INTERNAL_KEY=" + conf.InternalKey, + "BACKEND_URL=http://backend:8080", + "ENV=" + conf.Branch, + "OPENSEARCH_HOST=node1", + "OPENSEARCH_PORT=9200", + "DB_HOST=postgres", + "DB_PORT=5432", + "DB_USER=postgres", + "DB_PASS=" + conf.Password, + "DB_NAME=utmstack", + }, + Logging: &dLogging, + Deploy: &Deploy{ + Placement: &pManager, + Resources: &Resources{ + Limits: &Res{ + Memory: utils.Str(fmt.Sprintf("%vM", threadwindsIngestionMem)), + }, + }, + }, + } + webPDFMem := stack.ServiceResources["web-pdf"].AssignedMemory c.Services["web-pdf"] = Service{ Image: utils.Str("ghcr.io/utmstack/utmstack/web-pdf:" + conf.Branch), diff --git a/installer/types/stack.go b/installer/types/stack.go index 490f3cfc2..a4a32a837 100644 --- a/installer/types/stack.go +++ b/installer/types/stack.go @@ -39,6 +39,7 @@ var Services = []utils.ServiceConfig{ {Name: "socai", Priority: 3, MinMemory: 30, MaxMemory: 512}, {Name: "bitdefender", Priority: 3, MinMemory: 30, MaxMemory: 100}, {Name: "office365", Priority: 3, MinMemory: 30, MaxMemory: 100}, + {Name: "threadwinds-ingestion", Priority: 3, MinMemory: 50, MaxMemory: 256}, } func (s *StackConfig) Populate(c *Config) error { From dee17eb8dc5096785e65c1ac499165ba6b5d63bf Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 23 Dec 2025 11:05:25 -0500 Subject: [PATCH 04/21] feat: integrate ThreadWinds threat intelligence platform --- threadwinds-ingestion/Dockerfile | 10 + threadwinds-ingestion/config/config.go | 33 ++ threadwinds-ingestion/config/const.go | 71 ++++ threadwinds-ingestion/go.mod | 50 +++ threadwinds-ingestion/go.sum | 172 +++++++++ .../association/association_builder.go | 156 ++++++++ .../association/association_context.go | 40 ++ .../internal/association/association_rules.go | 164 ++++++++ .../internal/client/backend_client.go | 202 ++++++++++ .../internal/client/cm_client.go | 77 ++++ .../internal/client/opensearch_client.go | 96 +++++ .../internal/client/postgres_client.go | 158 ++++++++ .../internal/client/threadwinds_client.go | 154 ++++++++ .../internal/extractor/field_extractor.go | 111 ++++++ .../internal/mapper/entity_mapper.go | 127 +++++++ .../internal/models/alert.go | 30 ++ .../internal/models/event.go | 7 + .../internal/models/incident.go | 21 ++ .../internal/scheduler/ingestion_scheduler.go | 349 ++++++++++++++++++ threadwinds-ingestion/main.go | 109 ++++++ threadwinds-ingestion/utils/env.go | 20 + threadwinds-ingestion/utils/req.go | 46 +++ 22 files changed, 2203 insertions(+) create mode 100644 threadwinds-ingestion/Dockerfile create mode 100644 threadwinds-ingestion/config/config.go create mode 100644 threadwinds-ingestion/config/const.go create mode 100644 threadwinds-ingestion/go.mod create mode 100644 threadwinds-ingestion/go.sum create mode 100644 threadwinds-ingestion/internal/association/association_builder.go create mode 100644 threadwinds-ingestion/internal/association/association_context.go create mode 100644 threadwinds-ingestion/internal/association/association_rules.go create mode 100644 threadwinds-ingestion/internal/client/backend_client.go create mode 100644 threadwinds-ingestion/internal/client/cm_client.go create mode 100644 threadwinds-ingestion/internal/client/opensearch_client.go create mode 100644 threadwinds-ingestion/internal/client/postgres_client.go create mode 100644 threadwinds-ingestion/internal/client/threadwinds_client.go create mode 100644 threadwinds-ingestion/internal/extractor/field_extractor.go create mode 100644 threadwinds-ingestion/internal/mapper/entity_mapper.go create mode 100644 threadwinds-ingestion/internal/models/alert.go create mode 100644 threadwinds-ingestion/internal/models/event.go create mode 100644 threadwinds-ingestion/internal/models/incident.go create mode 100644 threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go create mode 100644 threadwinds-ingestion/main.go create mode 100644 threadwinds-ingestion/utils/env.go create mode 100644 threadwinds-ingestion/utils/req.go diff --git a/threadwinds-ingestion/Dockerfile b/threadwinds-ingestion/Dockerfile new file mode 100644 index 000000000..39d4bbb42 --- /dev/null +++ b/threadwinds-ingestion/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:24.04 + +RUN apt-get update +RUN apt-get install -y ca-certificates +RUN update-ca-certificates + +COPY threadwinds-ingestion . + +RUN chmod +x threadwinds-ingestion +ENTRYPOINT ./threadwinds-ingestion \ No newline at end of file diff --git a/threadwinds-ingestion/config/config.go b/threadwinds-ingestion/config/config.go new file mode 100644 index 000000000..e8a4128ad --- /dev/null +++ b/threadwinds-ingestion/config/config.go @@ -0,0 +1,33 @@ +package config + +type TWConfig struct { + InternalKey string + BackendURL string + CustomersManagerURL string + ThreadWindsURL string + OpenSearchHost string + OpenSearchPort string + DBHost string + DBPort string + DBUser string + DBPassword string + DBName string +} + +func GetTWConfig() (*TWConfig, error) { + cfg := &TWConfig{ + InternalKey: GetInternalKey(), + BackendURL: GetBackendUrl(), + CustomersManagerURL: GetCustomersManagerUrl(), + ThreadWindsURL: GetThreadWindsURL(), + OpenSearchHost: GetOpenSearchHost(), + OpenSearchPort: GetOpenSearchPort(), + DBHost: GetDBHost(), + DBPort: GetDBPort(), + DBUser: GetDBUser(), + DBPassword: GetDBPassword(), + DBName: GetDBName(), + } + + return cfg, nil +} diff --git a/threadwinds-ingestion/config/const.go b/threadwinds-ingestion/config/const.go new file mode 100644 index 000000000..616c80955 --- /dev/null +++ b/threadwinds-ingestion/config/const.go @@ -0,0 +1,71 @@ +package config + +import ( + "os" + "strings" + + "github.com/utmstack/UTMStack/threadwinds-ingestion/utils" +) + +func GetInternalKey() string { + return utils.Getenv("INTERNAL_KEY") +} + +func GetBackendUrl() string { + return utils.Getenv("BACKEND_URL") +} + +func GetCustomersManagerUrl() string { + if isDevEnvironment() { + return "https://cm.dev.utmstack.com" + } + return "https://cm.utmstack.com" +} + +func GetThreadWindsURL() string { + if isDevEnvironment() { + return "https://apis.dev.threatwinds.com" + } + return "https://apis.threatwinds.com" +} + +func GetOpenSearchHost() string { + return utils.Getenv("OPENSEARCH_HOST") +} + +func GetOpenSearchPort() string { + return utils.Getenv("OPENSEARCH_PORT") +} + +func GetDBHost() string { + return utils.Getenv("DB_HOST") +} + +func GetDBPort() string { + return utils.Getenv("DB_PORT") +} + +func GetDBUser() string { + return utils.Getenv("DB_USER") +} + +func GetDBPassword() string { + return utils.Getenv("DB_PASS") +} + +func GetDBName() string { + return utils.Getenv("DB_NAME") +} + +func isDevEnvironment() bool { + env := os.Getenv("ENV") + if env != "" { + if strings.Contains(env, "-dev") || + strings.Contains(env, "-qa") || + strings.Contains(env, "-rc") { + return true + } + } + + return false +} diff --git a/threadwinds-ingestion/go.mod b/threadwinds-ingestion/go.mod new file mode 100644 index 000000000..12ec71bcf --- /dev/null +++ b/threadwinds-ingestion/go.mod @@ -0,0 +1,50 @@ +module github.com/utmstack/UTMStack/threadwinds-ingestion + +go 1.25.4 + +require ( + github.com/lib/pq v1.10.9 + github.com/opensearch-project/opensearch-go/v2 v2.3.0 + github.com/threatwinds/go-sdk v1.0.47 +) + +require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.14.2 // indirect + github.com/bytedance/sonic/loader v0.4.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.11 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/gin-gonic/gin v1.11.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.29.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.57.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.uber.org/mock v0.6.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/threadwinds-ingestion/go.sum b/threadwinds-ingestion/go.sum new file mode 100644 index 000000000..08884ce01 --- /dev/null +++ b/threadwinds-ingestion/go.sum @@ -0,0 +1,172 @@ +github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= +github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= +github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= +github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= +github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik= +github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= +github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.29.0 h1:lQlF5VNJWNlRbRZNeOIkWElR+1LL/OuHcc0Kp14w1xk= +github.com/go-playground/validator/v10 v10.29.0/go.mod h1:D6QxqeMlgIPuT02L66f2ccrZ7AGgHkzKmmTMZhk/Kc4= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= +github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= +github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= +github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/threatwinds/go-sdk v1.0.47 h1:z54WA6pt95IiCwjfARx7S0I1GRD3X5WH2hspwhKrgPU= +github.com/threatwinds/go-sdk v1.0.47/go.mod h1:NXdavG6meLH3WzIy0rWvqHJusun8HyfneW7zW2VyJeI= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= +go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= +golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= +golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/threadwinds-ingestion/internal/association/association_builder.go b/threadwinds-ingestion/internal/association/association_builder.go new file mode 100644 index 000000000..175d0a829 --- /dev/null +++ b/threadwinds-ingestion/internal/association/association_builder.go @@ -0,0 +1,156 @@ +package association + +import ( + "sync" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/threatwinds/go-sdk/entities" +) + +type AssociationBuilder struct { + rules []*AssociationRule + entityRegistry *sync.Map + mu sync.RWMutex +} + +func NewAssociationBuilder() *AssociationBuilder { + builder := &AssociationBuilder{ + rules: GetEnabledRules(), + entityRegistry: &sync.Map{}, + } + catcher.Info("association builder initialized", map[string]any{ + "total_rules": len(builder.rules), + }) + return builder +} + +func (b *AssociationBuilder) RegisterEntity(entity *entities.Entity, entityID, sourcePath string, ctx AssociationContext) { + ref := &EntityReference{ + Entity: entity, + EntityID: entityID, + EntityType: entity.Type, + SourcePath: sourcePath, + Context: ctx, + } + b.entityRegistry.Store(entityID, ref) + catcher.Info("entity registered", map[string]any{ + "entity_id": entityID, + "entity_type": entity.Type, + "source_path": sourcePath, + }) +} + +func (b *AssociationBuilder) BuildAssociations() []*entities.Entity { + contextGroups := b.groupByContext() + for _, refs := range contextGroups { + b.detectAssociationsInContext(refs) + } + result := make([]*entities.Entity, 0) + b.entityRegistry.Range(func(key, value any) bool { + if ref, ok := value.(*EntityReference); ok { + if entity, ok := ref.Entity.(*entities.Entity); ok { + result = append(result, entity) + } + } + return true + }) + catcher.Info("associations built", map[string]any{ + "total_entities": len(result), + "context_groups": len(contextGroups), + "total_associations": b.CountAssociations(result), + }) + return result +} + +func (b *AssociationBuilder) groupByContext() map[string][]*EntityReference { + groups := make(map[string][]*EntityReference) + b.entityRegistry.Range(func(key, value any) bool { + if ref, ok := value.(*EntityReference); ok { + contextKey := ref.Context.AlertID + if contextKey != "" { + groups[contextKey] = append(groups[contextKey], ref) + } + } + return true + }) + return groups +} + +func (b *AssociationBuilder) detectAssociationsInContext(refs []*EntityReference) { + for _, rule := range b.rules { + if !rule.Enabled { + continue + } + for i, sourceRef := range refs { + if sourceRef.EntityType != rule.SourceType { + continue + } + for j, targetRef := range refs { + if i == j { + continue + } + if targetRef.EntityType != rule.TargetType { + continue + } + if b.shouldCreateAssociation(sourceRef, targetRef) { + b.createAssociation(sourceRef, targetRef, rule) + } + } + } + } +} + +func (b *AssociationBuilder) shouldCreateAssociation(source, target *EntityReference) bool { + if source.Context.SameContext(target.Context) { + return true + } + if source.Context.IsOriginToTarget(target.Context) { + return true + } + if source.Context.IsTargetToOrigin(target.Context) { + return true + } + return false +} + +func (b *AssociationBuilder) createAssociation(source, target *EntityReference, rule *AssociationRule) { + sourceEntity, ok := source.Entity.(*entities.Entity) + if !ok { + return + } + targetEntity, ok := target.Entity.(*entities.Entity) + if !ok { + return + } + associatedEntity := entities.EntityAssociation{ + Mode: string(rule.Mode), + Entity: entities.Entity{ + Type: targetEntity.Type, + Attributes: targetEntity.Attributes, + }, + } + if sourceEntity.Associations == nil { + sourceEntity.Associations = make([]entities.EntityAssociation, 0) + } + sourceEntity.Associations = append(sourceEntity.Associations, associatedEntity) + catcher.Info("association created", map[string]any{ + "rule": rule.Name, + "source_type": sourceEntity.Type, + "target_type": targetEntity.Type, + "mode": rule.Mode, + "context": source.Context.AlertID, + }) +} + +func (b *AssociationBuilder) CountAssociations(entities []*entities.Entity) int { + count := 0 + for _, entity := range entities { + count += len(entity.Associations) + } + return count +} + +func (b *AssociationBuilder) ClearRegistry() { + b.entityRegistry = &sync.Map{} + catcher.Info("entity registry cleared", nil) +} diff --git a/threadwinds-ingestion/internal/association/association_context.go b/threadwinds-ingestion/internal/association/association_context.go new file mode 100644 index 000000000..b2e88bd86 --- /dev/null +++ b/threadwinds-ingestion/internal/association/association_context.go @@ -0,0 +1,40 @@ +package association + +type ContextType string + +type AssociationContext struct { + AlertID string + IncidentID string + SourceField string +} + +type EntityReference struct { + Entity any + EntityID string + EntityType string + SourcePath string + Context AssociationContext +} + +func (ctx *AssociationContext) IsOrigin() bool { + return ctx.SourceField == "source" +} + +func (ctx *AssociationContext) IsTarget() bool { + return ctx.SourceField == "destination" +} + +func (ctx *AssociationContext) SameContext(other AssociationContext) bool { + if ctx.AlertID != "" && ctx.AlertID == other.AlertID { + return true + } + return false +} + +func (ctx *AssociationContext) IsOriginToTarget(other AssociationContext) bool { + return ctx.IsOrigin() && other.IsTarget() && ctx.SameContext(other) +} + +func (ctx *AssociationContext) IsTargetToOrigin(other AssociationContext) bool { + return ctx.IsTarget() && other.IsOrigin() && ctx.SameContext(other) +} diff --git a/threadwinds-ingestion/internal/association/association_rules.go b/threadwinds-ingestion/internal/association/association_rules.go new file mode 100644 index 000000000..797f839fb --- /dev/null +++ b/threadwinds-ingestion/internal/association/association_rules.go @@ -0,0 +1,164 @@ +package association + +type AssociationMode string + +const ( + Association AssociationMode = "association" + Aggregation AssociationMode = "aggregation" +) + +type RuleCategory string + +const ( + CategoryNetwork RuleCategory = "network" + CategoryIdentity RuleCategory = "identity" +) + +type AssociationRule struct { + Name string + SourceType string + TargetType string + Mode AssociationMode + Category RuleCategory + Description string + Enabled bool + Priority int +} + +var DefaultRules = []*AssociationRule{ + // Network Associations + { + Name: "ip-to-port", + SourceType: "ip", + TargetType: "port", + Mode: Association, + Category: CategoryNetwork, + Description: "IP exposes port", + Enabled: true, + Priority: 10, + }, + { + Name: "port-to-ip", + SourceType: "port", + TargetType: "ip", + Mode: Association, + Category: CategoryNetwork, + Description: "Port exposed on IP", + Enabled: true, + Priority: 10, + }, + { + Name: "hostname-to-ip", + SourceType: "hostname", + TargetType: "ip", + Mode: Association, + Category: CategoryNetwork, + Description: "Hostname resolves to IP", + Enabled: true, + Priority: 10, + }, + { + Name: "ip-to-hostname", + SourceType: "ip", + TargetType: "hostname", + Mode: Association, + Category: CategoryNetwork, + Description: "IP resolves to hostname", + Enabled: true, + Priority: 10, + }, + + // Identity Associations + { + Name: "username-to-ip", + SourceType: "username", + TargetType: "ip", + Mode: Association, + Category: CategoryIdentity, + Description: "User accessed from IP", + Enabled: true, + Priority: 10, + }, + { + Name: "ip-to-username", + SourceType: "ip", + TargetType: "username", + Mode: Association, + Category: CategoryIdentity, + Description: "IP accessed by user", + Enabled: true, + Priority: 10, + }, + { + Name: "username-to-hostname", + SourceType: "username", + TargetType: "hostname", + Mode: Association, + Category: CategoryIdentity, + Description: "User accessed from hostname", + Enabled: true, + Priority: 9, + }, + { + Name: "hostname-to-username", + SourceType: "hostname", + TargetType: "username", + Mode: Association, + Category: CategoryIdentity, + Description: "Hostname accessed by user", + Enabled: true, + Priority: 9, + }, + + // ASN Associations + { + Name: "ip-to-asn", + SourceType: "ip", + TargetType: "asn", + Mode: Association, + Category: CategoryNetwork, + Description: "IP belongs to ASN", + Enabled: true, + Priority: 10, + }, + { + Name: "asn-to-ip", + SourceType: "asn", + TargetType: "ip", + Mode: Association, + Category: CategoryNetwork, + Description: "ASN contains IP", + Enabled: true, + Priority: 10, + }, + { + Name: "hostname-to-asn", + SourceType: "hostname", + TargetType: "asn", + Mode: Association, + Category: CategoryNetwork, + Description: "Hostname resolves to IP in ASN", + Enabled: true, + Priority: 9, + }, + { + Name: "asn-to-hostname", + SourceType: "asn", + TargetType: "hostname", + Mode: Association, + Category: CategoryNetwork, + Description: "ASN contains hostname", + Enabled: true, + Priority: 9, + }, +} + +func GetEnabledRules() []*AssociationRule { + rules := make([]*AssociationRule, 0) + for _, rule := range DefaultRules { + if rule.Enabled { + rules = append(rules, rule) + } + } + return rules +} diff --git a/threadwinds-ingestion/internal/client/backend_client.go b/threadwinds-ingestion/internal/client/backend_client.go new file mode 100644 index 000000000..243b7c152 --- /dev/null +++ b/threadwinds-ingestion/internal/client/backend_client.go @@ -0,0 +1,202 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" +) + +type BackendClient struct { + baseURL string + internalKey string + httpClient *http.Client +} + +func NewBackendClient(cfg *config.TWConfig) *BackendClient { + return &BackendClient{ + baseURL: cfg.BackendURL, + internalKey: cfg.InternalKey, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +func (c *BackendClient) GetRecentIncidents(ctx context.Context) ([]*models.Incident, error) { + url := fmt.Sprintf("%s/api/utm-incidents?incidentStatus.in=OPEN,IN_REVIEW&sort=incidentCreatedDate,desc&size=100", c.baseURL) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Utm-Internal-Key", c.internalKey) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + var incidents []*models.Incident + if err := json.NewDecoder(resp.Body).Decode(&incidents); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + catcher.Info("fetched recent incidents", map[string]any{ + "count": len(incidents), + }) + + return incidents, nil +} + +func (c *BackendClient) GetIncidentAlerts(ctx context.Context, incidentID int64) ([]*models.IncidentAlert, error) { + url := fmt.Sprintf("%s/api/utm-incident-alerts?incidentId.equals=%d", c.baseURL, incidentID) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Utm-Internal-Key", c.internalKey) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + var alerts []*models.IncidentAlert + if err := json.NewDecoder(resp.Body).Decode(&alerts); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + catcher.Info("fetched incident alerts", map[string]any{ + "incident_id": incidentID, + "alert_count": len(alerts), + }) + + return alerts, nil +} + +type ThreadWindsConfig struct { + APIKey string + APISecret string + KeyID int64 + SecretID int64 +} + +type ConfigParameter struct { + ID int64 `json:"id"` + SectionID int64 `json:"sectionId"` + ConfParamShort string `json:"confParamShort"` + ConfParamValue string `json:"confParamValue"` +} + +func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsConfig, error) { + const threadwindsSectionID = 6 + url := fmt.Sprintf("%s/api/utm-configuration-parameters?sectionId.equals=%d&size=100", c.baseURL, threadwindsSectionID) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Utm-Internal-Key", c.internalKey) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + var params []ConfigParameter + if err := json.NewDecoder(resp.Body).Decode(¶ms); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + config := &ThreadWindsConfig{} + + for _, param := range params { + switch param.ConfParamShort { + case "THREADWINDS_API_KEY": + config.APIKey = param.ConfParamValue + config.KeyID = param.ID + case "THREADWINDS_API_SECRET": + config.APISecret = param.ConfParamValue + config.SecretID = param.ID + } + } + + return config, nil +} + +func (c *BackendClient) SaveThreadWindsCredentials(ctx context.Context, apiKey, apiSecret string, keyID, secretID int64) error { + const threadwindsSectionID = 6 + url := fmt.Sprintf("%s/api/utm-configuration-parameters", c.baseURL) + + params := []ConfigParameter{ + { + ID: keyID, + SectionID: threadwindsSectionID, + ConfParamShort: "THREADWINDS_API_KEY", + ConfParamValue: apiKey, + }, + { + ID: secretID, + SectionID: threadwindsSectionID, + ConfParamShort: "THREADWINDS_API_SECRET", + ConfParamValue: apiSecret, + }, + } + + payload, err := json.Marshal(params) + if err != nil { + return fmt.Errorf("failed to marshal parameters: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "PUT", url, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Utm-Internal-Key", c.internalKey) + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + catcher.Info("ThreadWinds credentials saved successfully", nil) + return nil +} diff --git a/threadwinds-ingestion/internal/client/cm_client.go b/threadwinds-ingestion/internal/client/cm_client.go new file mode 100644 index 000000000..bd10e20c1 --- /dev/null +++ b/threadwinds-ingestion/internal/client/cm_client.go @@ -0,0 +1,77 @@ +package client + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/threatwinds/go-sdk/utils" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" +) + +type CustomersManagerClient struct { + cmURL string +} + +func NewCustomersManagerClient(cfg *config.TWConfig) *CustomersManagerClient { + return &CustomersManagerClient{ + cmURL: cfg.CustomersManagerURL, + } +} + +type RegistrationRequest struct { + Email string `json:"email" validate:"required,email"` +} + +type RegistrationResponse struct { + APIKey string `json:"api_key"` + APISecret string `json:"api_secret"` +} + +func (c *CustomersManagerClient) RegisterUserReporter(email string) (*RegistrationResponse, error) { + endpoint := fmt.Sprintf("%s/api/v1/intelligence/register", c.cmURL) + + reqBody := RegistrationRequest{ + Email: email, + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal registration request: %w", err) + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + credentials, statusCode, err := utils.DoReq[RegistrationResponse]( + endpoint, + jsonData, + http.MethodPost, + headers, + ) + + if err != nil { + switch statusCode { + case http.StatusBadRequest: + return nil, catcher.Error("invalid registration data", err, map[string]any{ + "status": statusCode, + }) + case http.StatusInternalServerError: + return nil, catcher.Error("registration service error", err, map[string]any{ + "status": statusCode, + }) + default: + return nil, catcher.Error("registration failed", err, map[string]any{ + "status": statusCode, + }) + } + } + + catcher.Info("Successfully registered ThreadWinds intelligence reporter", map[string]any{ + "email": email, + }) + + return &credentials, nil +} diff --git a/threadwinds-ingestion/internal/client/opensearch_client.go b/threadwinds-ingestion/internal/client/opensearch_client.go new file mode 100644 index 000000000..1f90095d8 --- /dev/null +++ b/threadwinds-ingestion/internal/client/opensearch_client.go @@ -0,0 +1,96 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + + opensearch "github.com/opensearch-project/opensearch-go/v2" + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" +) + +type OpenSearchClient struct { + client *opensearch.Client +} + +func NewOpenSearchClient(cfg *config.TWConfig) (*OpenSearchClient, error) { + osConfig := opensearch.Config{ + Addresses: []string{ + fmt.Sprintf("http://%s:%s", cfg.OpenSearchHost, cfg.OpenSearchPort), + }, + } + + client, err := opensearch.NewClient(osConfig) + if err != nil { + return nil, fmt.Errorf("failed to create opensearch client: %w", err) + } + + info, err := client.Info() + if err != nil { + return nil, fmt.Errorf("opensearch connection failed: %w", err) + } + defer info.Body.Close() + + catcher.Info("opensearch client connected successfully", map[string]any{ + "host": cfg.OpenSearchHost, + "port": cfg.OpenSearchPort, + }) + + return &OpenSearchClient{client: client}, nil +} + +func (c *OpenSearchClient) GetAlertByID(ctx context.Context, alertID string) (*models.Alert, error) { + query := map[string]any{ + "query": map[string]any{ + "term": map[string]any{ + "id.keyword": alertID, + }, + }, + } + + return c.searchSingleAlert(ctx, "alert-*", query) +} + +func (c *OpenSearchClient) searchSingleAlert(ctx context.Context, index string, query map[string]any) (*models.Alert, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query); err != nil { + return nil, fmt.Errorf("failed to encode query: %w", err) + } + + res, err := c.client.Search( + c.client.Search.WithContext(ctx), + c.client.Search.WithIndex(index), + c.client.Search.WithBody(&buf), + ) + if err != nil { + return nil, fmt.Errorf("search failed: %w", err) + } + defer res.Body.Close() + + if res.IsError() { + body, _ := io.ReadAll(res.Body) + return nil, fmt.Errorf("search error %d: %s", res.StatusCode, string(body)) + } + + var result struct { + Hits struct { + Hits []struct { + Source models.Alert `json:"_source"` + } `json:"hits"` + } `json:"hits"` + } + + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + if len(result.Hits.Hits) == 0 { + return nil, fmt.Errorf("alert not found") + } + + return &result.Hits.Hits[0].Source, nil +} diff --git a/threadwinds-ingestion/internal/client/postgres_client.go b/threadwinds-ingestion/internal/client/postgres_client.go new file mode 100644 index 000000000..90ffb43a8 --- /dev/null +++ b/threadwinds-ingestion/internal/client/postgres_client.go @@ -0,0 +1,158 @@ +package client + +import ( + "context" + "database/sql" + "fmt" + "time" + + _ "github.com/lib/pq" + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" +) + +type PostgresClient struct { + db *sql.DB +} + +type AdminEmailResult struct { + Email string + IsConfigured bool + LastModified time.Time + LastModifiedBy string +} + +func NewPostgresClient(cfg *config.TWConfig) (*PostgresClient, error) { + dsn := fmt.Sprintf( + "host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", + cfg.DBHost, + cfg.DBPort, + cfg.DBUser, + cfg.DBPassword, + cfg.DBName, + ) + + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, fmt.Errorf("failed to open database: %w", err) + } + + db.SetMaxOpenConns(2) + db.SetMaxIdleConns(1) + db.SetConnMaxLifetime(5 * time.Minute) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, fmt.Errorf("failed to ping database: %w", err) + } + + catcher.Info("PostgreSQL connection established via native SQL", map[string]any{ + "host": cfg.DBHost, + "port": cfg.DBPort, + "db": cfg.DBName, + }) + + return &PostgresClient{db: db}, nil +} + +func (c *PostgresClient) Close() error { + if c.db != nil { + return c.db.Close() + } + return nil +} + +func (c *PostgresClient) GetAdminEmail(ctx context.Context) (*AdminEmailResult, error) { + query := ` + SELECT email, last_modified_by, last_modified_date + FROM jhi_user + WHERE login = $1 AND created_by = $2 + LIMIT 1 + ` + + var email, lastModifiedBy string + var lastModifiedDate sql.NullTime + + err := c.db.QueryRowContext(ctx, query, "admin", "system"). + Scan(&email, &lastModifiedBy, &lastModifiedDate) + + if err == sql.ErrNoRows { + return nil, fmt.Errorf("admin user not found in database") + } + if err != nil { + return nil, fmt.Errorf("failed to query admin user: %w", err) + } + + result := &AdminEmailResult{ + Email: email, + IsConfigured: email != "admin@localhost", + LastModifiedBy: lastModifiedBy, + } + + if lastModifiedDate.Valid { + result.LastModified = lastModifiedDate.Time + } + + return result, nil +} + +func (c *PostgresClient) WaitForValidAdminEmail(ctx context.Context, timeout time.Duration) (string, error) { + deadline := time.Now().Add(timeout) + retryInterval := 10 * time.Second + attempt := 1 + + catcher.Info("waiting for admin email configuration before starting service", map[string]any{ + "timeout_minutes": timeout.Minutes(), + "retry_interval": retryInterval.String(), + }) + + query := ` + SELECT email + FROM jhi_user + WHERE login = $1 AND created_by = $2 AND email != $3 + LIMIT 1 + ` + + for time.Now().Before(deadline) { + queryCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + + var email string + err := c.db.QueryRowContext(queryCtx, query, "admin", "system", "admin@localhost"). + Scan(&email) + cancel() + + if err == nil { + catcher.Info("Admin email configured successfully, starting ThreadWinds ingestion", map[string]any{ + "attempts": attempt, + "waited": time.Since(time.Now().Add(-timeout)).Round(time.Second).String(), + }) + return email, nil + } + + if err != sql.ErrNoRows { + catcher.Error("database error while checking admin email", err, map[string]any{ + "attempt": attempt, + }) + } else { + remainingTime := time.Until(deadline).Round(time.Second) + catcher.Info("Admin email not configured yet, waiting...", map[string]any{ + "attempt": attempt, + "next_retry_in": retryInterval.String(), + "remaining_time": remainingTime.String(), + "message": "Service will start automatically once admin configures their email", + }) + } + + select { + case <-ctx.Done(): + return "", fmt.Errorf("context cancelled while waiting for admin email: %w", ctx.Err()) + case <-time.After(retryInterval): + attempt++ + } + } + + return "", fmt.Errorf("timeout after %v waiting for admin email configuration (%d attempts). Admin must configure email in first login", timeout, attempt-1) +} diff --git a/threadwinds-ingestion/internal/client/threadwinds_client.go b/threadwinds-ingestion/internal/client/threadwinds_client.go new file mode 100644 index 000000000..cbe9964c3 --- /dev/null +++ b/threadwinds-ingestion/internal/client/threadwinds_client.go @@ -0,0 +1,154 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/threatwinds/go-sdk/entities" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" +) + +type ThreadWindsClient struct { + baseURL string + apiKey string + apiSecret string + httpClient *http.Client + mu sync.RWMutex +} + +func NewThreadWindsClient(cfg *config.TWConfig) *ThreadWindsClient { + return &ThreadWindsClient{ + baseURL: cfg.ThreadWindsURL, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +func (c *ThreadWindsClient) UpdateCredentials(apiKey, apiSecret string) { + c.mu.Lock() + defer c.mu.Unlock() + + c.apiKey = apiKey + c.apiSecret = apiSecret + + catcher.Info("ThreadWinds credentials updated", map[string]any{ + "api_key_length": len(apiKey), + }) +} + +func (c *ThreadWindsClient) ingestEntity(ctx context.Context, entity *entities.Entity) error { + url := fmt.Sprintf("%s/api/ingest/v1/entity", c.baseURL) + + payload, err := json.Marshal(entity) + if err != nil { + return fmt.Errorf("failed to marshal entity: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("api-key", c.apiKey) + req.Header.Set("api-secret", c.apiSecret) + + return c.executeWithRetry(req, entity.Type) +} + +func (c *ThreadWindsClient) executeWithRetry(req *http.Request, entityType string) error { + maxRetries := 3 + backoff := time.Second + + for attempt := 1; attempt <= maxRetries; attempt++ { + resp, err := c.httpClient.Do(req) + if err != nil { + catcher.Error("http request failed", err, map[string]any{ + "attempt": attempt, + "entity_type": entityType, + }) + if attempt < maxRetries { + time.Sleep(backoff) + backoff *= 2 + continue + } + return fmt.Errorf("failed after %d attempts: %w", maxRetries, err) + } + + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode == http.StatusAccepted { + catcher.Info("entity ingested successfully", map[string]any{ + "entity_type": entityType, + "status_code": resp.StatusCode, + "response": string(body), + }) + return nil + } + + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + return fmt.Errorf("client error %d: %s", resp.StatusCode, string(body)) + } + + if resp.StatusCode >= 500 && attempt < maxRetries { + catcher.Error("server error, retrying", fmt.Errorf("server error %d", resp.StatusCode), map[string]any{ + "attempt": attempt, + "entity_type": entityType, + "response": string(body), + }) + time.Sleep(backoff) + backoff *= 2 + continue + } + + return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) + } + + return fmt.Errorf("max retries exceeded") +} + +func (c *ThreadWindsClient) IngestBatch(ctx context.Context, entityBatch []*entities.Entity) error { + successCount := 0 + errorCount := 0 + + for i, entity := range entityBatch { + select { + case <-ctx.Done(): + return fmt.Errorf("batch ingestion cancelled: %w (processed %d/%d)", ctx.Err(), successCount, len(entityBatch)) + default: + } + + err := c.ingestEntity(ctx, entity) + if err != nil { + errorCount++ + catcher.Error("failed to ingest entity", err, map[string]any{ + "entity_type": entity.Type, + "batch_index": i, + "success_count": successCount, + "error_count": errorCount, + }) + continue + } + successCount++ + + if i < len(entityBatch)-1 { + time.Sleep(100 * time.Millisecond) + } + } + + if errorCount > 0 { + return fmt.Errorf("batch completed with %d errors out of %d entities", errorCount, len(entityBatch)) + } + + return nil +} diff --git a/threadwinds-ingestion/internal/extractor/field_extractor.go b/threadwinds-ingestion/internal/extractor/field_extractor.go new file mode 100644 index 000000000..b32f9ca36 --- /dev/null +++ b/threadwinds-ingestion/internal/extractor/field_extractor.go @@ -0,0 +1,111 @@ +package extractor + +import ( + "fmt" + "strings" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" +) + +type FieldExtractor struct{} + +func NewFieldExtractor() *FieldExtractor { + return &FieldExtractor{} +} + +func (e *FieldExtractor) ExtractFromAlert(alert *models.Alert) []*models.FlattenedField { + fields := make([]*models.FlattenedField, 0, 4) + + if alert.Source != nil { + fields = append(fields, e.extractFromHost(alert.Source, "alert.source")...) + } + + if alert.Destination != nil { + fields = append(fields, e.extractFromHost(alert.Destination, "alert.destination")...) + } + + catcher.Info("extracted fields from alert", map[string]any{ + "alert_id": alert.ID, + "field_count": len(fields), + }) + + return fields +} + +func (e *FieldExtractor) extractFromHost(host *models.Host, prefix string) []*models.FlattenedField { + fields := make([]*models.FlattenedField, 0, 5) + + if host.IP != "" && !isInvalidValue(host.IP) { + fields = append(fields, &models.FlattenedField{ + Path: prefix + ".ip", + Key: "ip", + Value: host.IP, + }) + } + + if host.Host != "" && !isInvalidValue(host.Host) { + fields = append(fields, &models.FlattenedField{ + Path: prefix + ".host", + Key: "hostname", + Value: host.Host, + }) + } + + if host.User != "" && !isInvalidValue(host.User) { + fields = append(fields, &models.FlattenedField{ + Path: prefix + ".user", + Key: "username", + Value: host.User, + }) + } + + if host.Port != 0 { + fields = append(fields, &models.FlattenedField{ + Path: prefix + ".port", + Key: "port", + Value: host.Port, + }) + } + + if host.ASN != 0 && !isInvalidValue(host.ASN) { + fields = append(fields, &models.FlattenedField{ + Path: prefix + ".asn", + Key: "asn", + Value: host.ASN, + }) + } + + return fields +} + +func isInvalidValue(value any) bool { + strValue := fmt.Sprintf("%v", value) + if strings.TrimSpace(strValue) == "" { + return true + } + + invalidValues := []string{ + "-", + "N/A", + "n/a", + "unknown", + "null", + "(null)", + "none", + "0", + "-1", + "0.0.0.0", + "255.255.255.255", + "127.0.0.1", + "localhost", + } + + for _, invalid := range invalidValues { + if strings.EqualFold(strValue, invalid) { + return true + } + } + + return false +} diff --git a/threadwinds-ingestion/internal/mapper/entity_mapper.go b/threadwinds-ingestion/internal/mapper/entity_mapper.go new file mode 100644 index 000000000..967761d91 --- /dev/null +++ b/threadwinds-ingestion/internal/mapper/entity_mapper.go @@ -0,0 +1,127 @@ +package mapper + +import ( + "fmt" + "strings" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/threatwinds/go-sdk/entities" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" +) + +type EntityMapper struct { + entityTypes map[string]bool +} + +func NewEntityMapper() *EntityMapper { + mapper := &EntityMapper{ + entityTypes: make(map[string]bool), + } + + for _, def := range entities.Definitions { + mapper.entityTypes[def.Type] = true + } + + catcher.Info("entity mapper initialized", map[string]any{ + "total_entity_types": len(mapper.entityTypes), + }) + + return mapper +} + +func (m *EntityMapper) MapFieldToEntityType(field *models.FlattenedField) (string, bool) { + leafKey := normalizeKey(field.Key) + + if m.entityTypes[leafKey] { + return leafKey, true + } + + return "", false +} + +func normalizeKey(key string) string { + key = strings.ToLower(key) + key = strings.ReplaceAll(key, "_", "-") + return key +} + +func (m *EntityMapper) BuildEntity(entityType string, value any, context EntityEnrichmentContext) (*entities.Entity, string, error) { + validatedValue, hash, err := entities.ValidateValue(value, entityType) + if err != nil { + return nil, "", fmt.Errorf("validation failed for type %s: %w", entityType, err) + } + + attrs := entities.Attributes{} + if !attrs.SetAttribute(entityType, validatedValue) { + return nil, "", fmt.Errorf("failed to set attribute for type %s", entityType) + } + + if context.Country != "" { + attrs.Country = &context.Country + } + if context.City != "" { + attrs.City = &context.City + } + if context.ASO != "" { + attrs.Aso = &context.ASO + } + if context.Latitude != nil { + attrs.Latitude = context.Latitude + } + if context.Longitude != nil { + attrs.Longitude = context.Longitude + } + if context.AccuracyRadius != nil { + attrs.AccuracyRadius = context.AccuracyRadius + } + + reputation := calculateReputation(context.Severity) + + tags := []string{ + "utmstack", + "incident-" + context.IncidentID, + } + if context.DataType != "" { + tags = append(tags, "datasource-"+context.DataType) + } + + entity := &entities.Entity{ + Type: entityType, + Attributes: attrs, + Reputation: reputation, + Tags: tags, + VisibleBy: []string{"utmstack"}, + Associations: nil, + } + + entityID := fmt.Sprintf("%s-%s", entityType, hash) + + catcher.Info("entity built successfully", map[string]any{ + "entity_type": entityType, + "entity_id": entityID, + "reputation": reputation, + }) + + return entity, entityID, nil +} + +type EntityEnrichmentContext struct { + IncidentID string + Severity int + DataType string + Country string + City string + Latitude *float64 + Longitude *float64 + ASO string + AccuracyRadius *float64 +} + +func calculateReputation(severity int) int { + if severity >= 7 { + return -3 + } else if severity >= 4 { + return -1 + } + return 0 +} diff --git a/threadwinds-ingestion/internal/models/alert.go b/threadwinds-ingestion/internal/models/alert.go new file mode 100644 index 000000000..1ff41a2a9 --- /dev/null +++ b/threadwinds-ingestion/internal/models/alert.go @@ -0,0 +1,30 @@ +package models + +type Alert struct { + ID string `json:"id"` + Name string `json:"name"` + Timestamp string `json:"@timestamp"` + DataType string `json:"dataType"` + DataSource string `json:"dataSource"` + Severity int `json:"severity"` + Status int `json:"status"` + Source *Host `json:"source,omitempty"` + Destination *Host `json:"destination,omitempty"` + ASO string `json:"aso,omitempty"` + ASN int `json:"asn,omitempty"` +} + +type Host struct { + IP string `json:"ip,omitempty"` + Host string `json:"host,omitempty"` + User string `json:"user,omitempty"` + Port int `json:"port,omitempty"` + City string `json:"city,omitempty"` + Country string `json:"country,omitempty"` + Coordinates []float64 `json:"coordinates,omitempty"` + ASO string `json:"aso,omitempty"` + ASN int `json:"asn,omitempty"` + AccuracyRadius int `json:"accuracyRadius,omitempty"` + IsAnonymousProxy bool `json:"isAnonymousProxy,omitempty"` + IsSatelliteProvider bool `json:"isSatelliteProvider,omitempty"` +} diff --git a/threadwinds-ingestion/internal/models/event.go b/threadwinds-ingestion/internal/models/event.go new file mode 100644 index 000000000..7091bc879 --- /dev/null +++ b/threadwinds-ingestion/internal/models/event.go @@ -0,0 +1,7 @@ +package models + +type FlattenedField struct { + Path string + Key string + Value any +} diff --git a/threadwinds-ingestion/internal/models/incident.go b/threadwinds-ingestion/internal/models/incident.go new file mode 100644 index 000000000..3e513fd45 --- /dev/null +++ b/threadwinds-ingestion/internal/models/incident.go @@ -0,0 +1,21 @@ +package models + +import "time" + +type Incident struct { + ID int64 `json:"id"` + Name string `json:"incidentName"` + Description string `json:"incidentDescription"` + Status string `json:"incidentStatus"` + Severity int `json:"incidentSeverity"` + CreatedDate time.Time `json:"incidentCreatedDate"` +} + +type IncidentAlert struct { + ID int64 `json:"id"` + IncidentID int64 `json:"incidentId"` + AlertID string `json:"alertId"` + AlertName string `json:"alertName"` + AlertStatus int `json:"alertStatus"` + AlertSeverity int `json:"alertSeverity"` +} diff --git a/threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go b/threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go new file mode 100644 index 000000000..b4d051d3f --- /dev/null +++ b/threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go @@ -0,0 +1,349 @@ +package scheduler + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/association" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/client" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/extractor" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/mapper" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" +) + +const ( + pollInterval = 5 * time.Minute + incidentRetentionPeriod = 48 * time.Hour + alertRetentionPeriod = 72 * time.Hour + cleanupInterval = 6 * time.Hour +) + +type IncidentState struct { + LastProcessedAt time.Time + ProcessedAlerts map[string]time.Time + TotalEntities int +} + +type IngestionScheduler struct { + cfg *config.TWConfig + backendClient *client.BackendClient + opensearchClient *client.OpenSearchClient + threadwindsClient *client.ThreadWindsClient + fieldExtractor *extractor.FieldExtractor + entityMapper *mapper.EntityMapper + associationBuilder *association.AssociationBuilder + processedIncidents map[int64]*IncidentState + mu sync.RWMutex +} + +func NewIngestionScheduler( + cfg *config.TWConfig, + backendClient *client.BackendClient, + opensearchClient *client.OpenSearchClient, + threadwindsClient *client.ThreadWindsClient, +) *IngestionScheduler { + return &IngestionScheduler{ + cfg: cfg, + backendClient: backendClient, + opensearchClient: opensearchClient, + threadwindsClient: threadwindsClient, + fieldExtractor: extractor.NewFieldExtractor(), + entityMapper: mapper.NewEntityMapper(), + associationBuilder: association.NewAssociationBuilder(), + processedIncidents: make(map[int64]*IncidentState), + } +} + +func (s *IngestionScheduler) Start(ctx context.Context) { + ticker := time.NewTicker(pollInterval) + cleanupTicker := time.NewTicker(cleanupInterval) + defer ticker.Stop() + defer cleanupTicker.Stop() + + catcher.Info("ingestion scheduler started", map[string]any{ + "poll_interval": pollInterval, + "cleanup_interval": cleanupInterval, + }) + + s.runIngestionCycle(ctx) + + for { + select { + case <-ctx.Done(): + catcher.Info("scheduler stopped", nil) + return + case <-ticker.C: + s.runIngestionCycle(ctx) + case <-cleanupTicker.C: + s.cleanOldState() + } + } +} + +func (s *IngestionScheduler) runIngestionCycle(ctx context.Context) { + catcher.Info("starting ingestion cycle", nil) + startTime := time.Now() + + if err := s.updateThreadWindsCredentials(ctx); err != nil { + catcher.Error("failed to update ThreadWinds credentials from database", err, nil) + } + + cycleTimeout := time.Duration(float64(pollInterval) * 0.9) + cycleCtx, cancel := context.WithTimeout(ctx, cycleTimeout) + defer cancel() + + incidents, err := s.backendClient.GetRecentIncidents(cycleCtx) + if err != nil { + catcher.Error("failed to fetch incidents", err, nil) + return + } + + if len(incidents) == 0 { + catcher.Info("no recent incidents to process", nil) + return + } + + totalEntities := 0 + for i, incident := range incidents { + select { + case <-cycleCtx.Done(): + catcher.Info("cycle timeout or cancellation, stopping", map[string]any{ + "processed_incidents": i, + "total_incidents": len(incidents), + "reason": cycleCtx.Err().Error(), + }) + return + default: + } + + entitiesCount, err := s.processIncident(cycleCtx, incident) + if err != nil { + catcher.Error("failed to process incident", err, map[string]any{ + "incident_id": incident.ID, + "incident_name": incident.Name, + }) + continue + } + totalEntities += entitiesCount + } + + duration := time.Since(startTime) + catcher.Info("ingestion cycle completed", map[string]any{ + "duration_seconds": duration.Seconds(), + "incidents_processed": len(incidents), + "total_entities": totalEntities, + }) +} + +func (s *IngestionScheduler) processIncident(ctx context.Context, incident *models.Incident) (int, error) { + s.mu.Lock() + state, exists := s.processedIncidents[incident.ID] + if !exists { + state = &IncidentState{ + ProcessedAlerts: make(map[string]time.Time), + } + s.processedIncidents[incident.ID] = state + } + s.mu.Unlock() + + incidentAlerts, err := s.backendClient.GetIncidentAlerts(ctx, incident.ID) + if err != nil { + return 0, fmt.Errorf("failed to get incident alerts: %w", err) + } + + if len(incidentAlerts) == 0 { + return 0, nil + } + + newAlerts := s.filterNewAlerts(incidentAlerts, state) + if len(newAlerts) == 0 { + return 0, nil + } + + catcher.Info("processing incident with new alerts", map[string]any{ + "incident_id": incident.ID, + "new_alerts": len(newAlerts), + "total_alerts": len(incidentAlerts), + }) + + s.associationBuilder.ClearRegistry() + + for _, incidentAlert := range newAlerts { + err := s.processAlertWithAssociations(ctx, incidentAlert, incident) + if err != nil { + catcher.Error("failed to process alert", err, map[string]any{ + "alert_id": incidentAlert.AlertID, + "incident_id": incident.ID, + }) + continue + } + + s.mu.Lock() + state.ProcessedAlerts[incidentAlert.AlertID] = time.Now() + s.mu.Unlock() + } + + allEntities := s.associationBuilder.BuildAssociations() + + if len(allEntities) > 0 { + if err := s.threadwindsClient.IngestBatch(ctx, allEntities); err != nil { + return 0, fmt.Errorf("failed to ingest batch: %w", err) + } + } + + s.mu.Lock() + state.LastProcessedAt = time.Now() + state.TotalEntities += len(allEntities) + s.mu.Unlock() + + return len(allEntities), nil +} + +func (s *IngestionScheduler) processAlertWithAssociations(ctx context.Context, incidentAlert *models.IncidentAlert, incident *models.Incident) error { + alert, err := s.opensearchClient.GetAlertByID(ctx, incidentAlert.AlertID) + if err != nil { + return fmt.Errorf("failed to get alert: %w", err) + } + + alertFields := s.fieldExtractor.ExtractFromAlert(alert) + s.mapAndRegisterFieldsToEntities(alertFields, incident, alert) + + return nil +} + +func (s *IngestionScheduler) mapAndRegisterFieldsToEntities( + fields []*models.FlattenedField, + incident *models.Incident, + alert *models.Alert, +) { + for _, field := range fields { + entityType, matched := s.entityMapper.MapFieldToEntityType(field) + if !matched { + continue + } + + sourceField := "" + var hostContext *models.Host + if strings.Contains(field.Path, "source") { + sourceField = "source" + hostContext = alert.Source + } else if strings.Contains(field.Path, "destination") { + sourceField = "destination" + hostContext = alert.Destination + } + + enrichmentCtx := s.buildEnrichmentContext(incident, alert, hostContext) + entity, entityID, err := s.entityMapper.BuildEntity(entityType, field.Value, enrichmentCtx) + if err != nil { + catcher.Error("failed to build entity", err, map[string]any{ + "entity_type": entityType, + "field_path": field.Path, + }) + continue + } + + assocContext := association.AssociationContext{ + AlertID: alert.ID, + IncidentID: fmt.Sprintf("%d", incident.ID), + SourceField: sourceField, + } + s.associationBuilder.RegisterEntity(entity, entityID, field.Path, assocContext) + } +} + +func (s *IngestionScheduler) filterNewAlerts(incidentAlerts []*models.IncidentAlert, state *IncidentState) []*models.IncidentAlert { + s.mu.RLock() + defer s.mu.RUnlock() + + newAlerts := make([]*models.IncidentAlert, 0, len(incidentAlerts)) + for _, alert := range incidentAlerts { + if _, processed := state.ProcessedAlerts[alert.AlertID]; !processed { + newAlerts = append(newAlerts, alert) + } + } + return newAlerts +} + +func (s *IngestionScheduler) cleanOldState() { + s.mu.Lock() + defer s.mu.Unlock() + + incidentCutoff := time.Now().Add(-incidentRetentionPeriod) + alertCutoff := time.Now().Add(-alertRetentionPeriod) + + cleanedIncidents := 0 + cleanedAlerts := 0 + + for incidentID, state := range s.processedIncidents { + if state.LastProcessedAt.Before(incidentCutoff) { + delete(s.processedIncidents, incidentID) + cleanedIncidents++ + continue + } + + for alertID, processedAt := range state.ProcessedAlerts { + if processedAt.Before(alertCutoff) { + delete(state.ProcessedAlerts, alertID) + cleanedAlerts++ + } + } + } + + if cleanedIncidents > 0 || cleanedAlerts > 0 { + catcher.Info("state cleanup completed", map[string]any{ + "active_incidents": len(s.processedIncidents), + "cleaned_incidents": cleanedIncidents, + "cleaned_alerts": cleanedAlerts, + }) + } +} + +func (s *IngestionScheduler) buildEnrichmentContext(incident *models.Incident, alert *models.Alert, host *models.Host) mapper.EntityEnrichmentContext { + ctx := mapper.EntityEnrichmentContext{ + IncidentID: fmt.Sprintf("%d", incident.ID), + Severity: incident.Severity, + DataType: alert.DataType, + } + + if host != nil { + ctx.Country = host.Country + ctx.City = host.City + ctx.ASO = host.ASO + + if len(host.Coordinates) == 2 { + lat := host.Coordinates[0] + lon := host.Coordinates[1] + if lat != 0.0 || lon != 0.0 { + ctx.Latitude = &lat + ctx.Longitude = &lon + } + } + + if host.AccuracyRadius > 0 { + radius := float64(host.AccuracyRadius) + ctx.AccuracyRadius = &radius + } + } + + return ctx +} + +func (s *IngestionScheduler) updateThreadWindsCredentials(ctx context.Context) error { + config, err := s.backendClient.GetThreadWindsConfig(ctx) + if err != nil { + return fmt.Errorf("failed to get ThreadWinds credentials from backend: %w", err) + } + + if config.APIKey == "" || config.APISecret == "" { + return fmt.Errorf("ThreadWinds credentials are empty in backend configuration") + } + + s.threadwindsClient.UpdateCredentials(config.APIKey, config.APISecret) + + return nil +} diff --git a/threadwinds-ingestion/main.go b/threadwinds-ingestion/main.go new file mode 100644 index 000000000..45be62c44 --- /dev/null +++ b/threadwinds-ingestion/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + + "github.com/threatwinds/go-sdk/catcher" + "github.com/utmstack/UTMStack/threadwinds-ingestion/config" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/client" + "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/scheduler" +) + +func main() { + catcher.Info("Starting ThreadWinds Ingestion Service", nil) + + cfg, err := config.GetTWConfig() + if err != nil { + catcher.Error("failed to load configuration", err, nil) + os.Exit(1) + } + + postgresClient, err := client.NewPostgresClient(cfg) + if err != nil { + catcher.Error("failed to initialize postgres client", err, nil) + os.Exit(1) + } + + ctx := context.Background() + + adminEmail, err := postgresClient.WaitForValidAdminEmail(ctx, 60*time.Minute) + if err != nil { + catcher.Error("cannot start ThreadWinds Ingestion without valid admin email", err, nil) + os.Exit(1) + } + + catcher.Info("Valid admin email obtained", map[string]any{ + "admin_email": adminEmail, + }) + + cmClient := client.NewCustomersManagerClient(cfg) + backendClient := client.NewBackendClient(cfg) + opensearchClient, err := client.NewOpenSearchClient(cfg) + if err != nil { + catcher.Error("failed to initialize opensearch client", err, nil) + os.Exit(1) + } + + threadwindsClient := client.NewThreadWindsClient(cfg) + + twConfig, err := backendClient.GetThreadWindsConfig(ctx) + if err != nil { + catcher.Error("failed to check ThreadWinds configuration", err, nil) + os.Exit(1) + } + + if twConfig.APIKey == "" || twConfig.APISecret == "" { + catcher.Info("ThreadWinds not configured, registering in platform...", nil) + + regResp, err := cmClient.RegisterUserReporter(adminEmail) + if err != nil { + catcher.Error("failed to register in ThreadWinds Platform", err, nil) + os.Exit(1) + } + + err = backendClient.SaveThreadWindsCredentials(ctx, + regResp.APIKey, + regResp.APISecret, + twConfig.KeyID, + twConfig.SecretID) + if err != nil { + catcher.Error("failed to save ThreadWinds credentials", err, nil) + os.Exit(1) + } + + threadwindsClient.UpdateCredentials(regResp.APIKey, regResp.APISecret) + } else { + catcher.Info("ThreadWinds already configured", nil) + threadwindsClient.UpdateCredentials(twConfig.APIKey, twConfig.APISecret) + } + + ingestionScheduler := scheduler.NewIngestionScheduler( + cfg, + backendClient, + opensearchClient, + threadwindsClient, + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + go ingestionScheduler.Start(ctx) + + sig := <-sigChan + catcher.Info("received shutdown signal, initiating graceful shutdown", map[string]any{ + "signal": sig.String(), + }) + + cancel() + + time.Sleep(5 * time.Second) + + catcher.Info("ThreadWinds Ingestion Service stopped", nil) +} diff --git a/threadwinds-ingestion/utils/env.go b/threadwinds-ingestion/utils/env.go new file mode 100644 index 000000000..779b3dc6a --- /dev/null +++ b/threadwinds-ingestion/utils/env.go @@ -0,0 +1,20 @@ +package utils + +import ( + "os" + + "github.com/threatwinds/go-sdk/catcher" +) + +func Getenv(key string) string { + value, defined := os.LookupEnv(key) + if !defined { + catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) + os.Exit(1) + } + if (value == "") || (value == " ") { + catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) + os.Exit(1) + } + return value +} diff --git a/threadwinds-ingestion/utils/req.go b/threadwinds-ingestion/utils/req.go new file mode 100644 index 000000000..3bc51ca70 --- /dev/null +++ b/threadwinds-ingestion/utils/req.go @@ -0,0 +1,46 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" +) + +func DoReq[response any](url string, data []byte, method string, headers map[string]string) (response, int, error) { + var result response + + req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) + if err != nil { + return result, http.StatusInternalServerError, err + } + + for k, v := range headers { + req.Header.Add(k, v) + } + + client := &http.Client{} + + resp, err := client.Do(req) + if err != nil { + return result, http.StatusInternalServerError, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return result, http.StatusInternalServerError, err + } + + if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { + return result, resp.StatusCode, fmt.Errorf("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) + } + + err = json.Unmarshal(body, &result) + if err != nil { + return result, http.StatusInternalServerError, err + } + + return result, resp.StatusCode, nil +} From c7774b4c44179dfd8a3e3b5e8cf3ae8c8e034b14 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 23 Dec 2025 13:00:13 -0500 Subject: [PATCH 05/21] refactor: simplify PostgreSQL connection initialization in postgres_client --- threadwinds-ingestion/internal/client/postgres_client.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/threadwinds-ingestion/internal/client/postgres_client.go b/threadwinds-ingestion/internal/client/postgres_client.go index 90ffb43a8..1706173b9 100644 --- a/threadwinds-ingestion/internal/client/postgres_client.go +++ b/threadwinds-ingestion/internal/client/postgres_client.go @@ -41,10 +41,7 @@ func NewPostgresClient(cfg *config.TWConfig) (*PostgresClient, error) { db.SetMaxIdleConns(1) db.SetConnMaxLifetime(5 * time.Minute) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := db.PingContext(ctx); err != nil { + if err := db.Ping(); err != nil { db.Close() return nil, fmt.Errorf("failed to ping database: %w", err) } @@ -52,7 +49,6 @@ func NewPostgresClient(cfg *config.TWConfig) (*PostgresClient, error) { catcher.Info("PostgreSQL connection established via native SQL", map[string]any{ "host": cfg.DBHost, "port": cfg.DBPort, - "db": cfg.DBName, }) return &PostgresClient{db: db}, nil From 72fcfb04df08f1c41c3f52c54513ebd7663854ab Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 23 Dec 2025 13:53:42 -0500 Subject: [PATCH 06/21] feat: add infinite retry mechanism for ThreadWinds registration --- threadwinds-ingestion/main.go | 20 +++++++++++--- threadwinds-ingestion/utils/check.go | 39 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 threadwinds-ingestion/utils/check.go diff --git a/threadwinds-ingestion/main.go b/threadwinds-ingestion/main.go index 45be62c44..249af1a31 100644 --- a/threadwinds-ingestion/main.go +++ b/threadwinds-ingestion/main.go @@ -11,6 +11,7 @@ import ( "github.com/utmstack/UTMStack/threadwinds-ingestion/config" "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/client" "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/scheduler" + "github.com/utmstack/UTMStack/threadwinds-ingestion/utils" ) func main() { @@ -57,14 +58,27 @@ func main() { } if twConfig.APIKey == "" || twConfig.APISecret == "" { - catcher.Info("ThreadWinds not configured, registering in platform...", nil) + catcher.Info("ThreadWinds not configured, will attempt registration with retry...", nil) - regResp, err := cmClient.RegisterUserReporter(adminEmail) + var regResp *client.RegistrationResponse + + registerFunc := func() error { + resp, err := cmClient.RegisterUserReporter(adminEmail) + if err != nil { + return err + } + regResp = resp + return nil + } + + err = utils.InfiniteRetryIfXError(registerFunc, "404", "Not Found", "connection refused") if err != nil { - catcher.Error("failed to register in ThreadWinds Platform", err, nil) + catcher.Error("failed to register in ThreadWinds Platform after retries", err, nil) os.Exit(1) } + catcher.Info("ThreadWinds registration successful", nil) + err = backendClient.SaveThreadWindsCredentials(ctx, regResp.APIKey, regResp.APISecret, diff --git a/threadwinds-ingestion/utils/check.go b/threadwinds-ingestion/utils/check.go new file mode 100644 index 000000000..493a35dae --- /dev/null +++ b/threadwinds-ingestion/utils/check.go @@ -0,0 +1,39 @@ +package utils + +import ( + "strings" + "time" + + "github.com/threatwinds/go-sdk/catcher" +) + +const ( + wait = 5 * time.Second +) + +func InfiniteRetryIfXError(f func() error, exceptions ...string) error { + var xErrorWasLogged bool + + for { + err := f() + if err != nil && is(err, exceptions...) { + if !xErrorWasLogged { + _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) + xErrorWasLogged = true + } + time.Sleep(wait) + continue + } + + return err + } +} + +func is(e error, args ...string) bool { + for _, arg := range args { + if strings.Contains(e.Error(), arg) { + return true + } + } + return false +} From 264e7207ebad7ef3e6aae6fab6e2bcc1eedbcd42 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Mon, 29 Dec 2025 20:04:52 -0500 Subject: [PATCH 07/21] feat(threadwinds-ingestion): implement infinite retry with exponential backoff --- threadwinds-ingestion/main.go | 6 +--- threadwinds-ingestion/utils/check.go | 49 ++++++++++++++++------------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/threadwinds-ingestion/main.go b/threadwinds-ingestion/main.go index 249af1a31..440d1ba80 100644 --- a/threadwinds-ingestion/main.go +++ b/threadwinds-ingestion/main.go @@ -71,11 +71,7 @@ func main() { return nil } - err = utils.InfiniteRetryIfXError(registerFunc, "404", "Not Found", "connection refused") - if err != nil { - catcher.Error("failed to register in ThreadWinds Platform after retries", err, nil) - os.Exit(1) - } + utils.InfiniteRetry(registerFunc, "ThreadWinds registration") catcher.Info("ThreadWinds registration successful", nil) diff --git a/threadwinds-ingestion/utils/check.go b/threadwinds-ingestion/utils/check.go index 493a35dae..1f74df131 100644 --- a/threadwinds-ingestion/utils/check.go +++ b/threadwinds-ingestion/utils/check.go @@ -1,39 +1,48 @@ package utils import ( - "strings" + "fmt" "time" "github.com/threatwinds/go-sdk/catcher" ) const ( - wait = 5 * time.Second + retryInitialBackoff = 5 * time.Second + retryMaxBackoff = 2 * time.Minute + retryBackoffMultiplier = 2.0 + retryLogInterval = 10 ) -func InfiniteRetryIfXError(f func() error, exceptions ...string) error { - var xErrorWasLogged bool +func InfiniteRetry(f func() error, operationName string) { + attempt := 0 + currentBackoff := retryInitialBackoff + + catcher.Info(fmt.Sprintf("Starting %s with infinite retry and exponential backoff", operationName), map[string]any{ + "initial_backoff": retryInitialBackoff.String(), + "max_backoff": retryMaxBackoff.String(), + }) for { + attempt++ err := f() - if err != nil && is(err, exceptions...) { - if !xErrorWasLogged { - _ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil) - xErrorWasLogged = true - } - time.Sleep(wait) - continue - } - return err - } -} + if err == nil { + catcher.Info(fmt.Sprintf("%s completed successfully", operationName), map[string]any{ + "attempts": attempt, + }) + return + } -func is(e error, args ...string) bool { - for _, arg := range args { - if strings.Contains(e.Error(), arg) { - return true + if attempt == 1 || attempt%retryLogInterval == 0 { + _ = catcher.Error(fmt.Sprintf("%s failed, will retry indefinitely...", operationName), err, map[string]any{ + "attempt": attempt, + "next_retry_in": currentBackoff.String(), + }) } + + time.Sleep(currentBackoff) + + currentBackoff = min(time.Duration(float64(currentBackoff)*retryBackoffMultiplier), retryMaxBackoff) } - return false } From 666ee1d09cb89e980593b93205c30a2388d513ec Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Fri, 2 Jan 2026 13:17:46 -0500 Subject: [PATCH 08/21] fix: refresh admin email on registration retry and improve logging --- threadwinds-ingestion/internal/client/cm_client.go | 3 +++ .../internal/client/postgres_client.go | 5 +++++ threadwinds-ingestion/main.go | 11 ++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/threadwinds-ingestion/internal/client/cm_client.go b/threadwinds-ingestion/internal/client/cm_client.go index bd10e20c1..2afdfe4b3 100644 --- a/threadwinds-ingestion/internal/client/cm_client.go +++ b/threadwinds-ingestion/internal/client/cm_client.go @@ -57,14 +57,17 @@ func (c *CustomersManagerClient) RegisterUserReporter(email string) (*Registrati case http.StatusBadRequest: return nil, catcher.Error("invalid registration data", err, map[string]any{ "status": statusCode, + "email": email, }) case http.StatusInternalServerError: return nil, catcher.Error("registration service error", err, map[string]any{ "status": statusCode, + "email": email, }) default: return nil, catcher.Error("registration failed", err, map[string]any{ "status": statusCode, + "email": email, }) } } diff --git a/threadwinds-ingestion/internal/client/postgres_client.go b/threadwinds-ingestion/internal/client/postgres_client.go index 1706173b9..ed54e56be 100644 --- a/threadwinds-ingestion/internal/client/postgres_client.go +++ b/threadwinds-ingestion/internal/client/postgres_client.go @@ -92,6 +92,11 @@ func (c *PostgresClient) GetAdminEmail(ctx context.Context) (*AdminEmailResult, result.LastModified = lastModifiedDate.Time } + catcher.Info("retrieved current admin email", map[string]any{ + "email": email, + "last_modified_by": lastModifiedBy, + }) + return result, nil } diff --git a/threadwinds-ingestion/main.go b/threadwinds-ingestion/main.go index 440d1ba80..cade60c4b 100644 --- a/threadwinds-ingestion/main.go +++ b/threadwinds-ingestion/main.go @@ -63,7 +63,16 @@ func main() { var regResp *client.RegistrationResponse registerFunc := func() error { - resp, err := cmClient.RegisterUserReporter(adminEmail) + currentEmail, emailErr := postgresClient.GetAdminEmail(ctx) + if emailErr != nil { + return catcher.Error("failed to get current admin email", emailErr, nil) + } + + catcher.Info("attempting ThreadWinds registration", map[string]any{ + "email": currentEmail.Email, + }) + + resp, err := cmClient.RegisterUserReporter(currentEmail.Email) if err != nil { return err } From aabd05b6bbee08cc4283cbdfcbbed2fed5d9ddbe Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Fri, 2 Jan 2026 15:25:10 -0500 Subject: [PATCH 09/21] refactor: update ThreatWinds config parameter keys and add metadata fields --- .../internal/client/backend_client.go | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/threadwinds-ingestion/internal/client/backend_client.go b/threadwinds-ingestion/internal/client/backend_client.go index 243b7c152..5fdc7a378 100644 --- a/threadwinds-ingestion/internal/client/backend_client.go +++ b/threadwinds-ingestion/internal/client/backend_client.go @@ -105,10 +105,14 @@ type ThreadWindsConfig struct { } type ConfigParameter struct { - ID int64 `json:"id"` - SectionID int64 `json:"sectionId"` - ConfParamShort string `json:"confParamShort"` - ConfParamValue string `json:"confParamValue"` + ID int64 `json:"id"` + SectionID int64 `json:"sectionId"` + ConfParamShort string `json:"confParamShort"` + ConfParamLarge string `json:"confParamLarge,omitempty"` + ConfParamDescription string `json:"confParamDescription,omitempty"` + ConfParamValue string `json:"confParamValue"` + ConfParamRequired bool `json:"confParamRequired,omitempty"` + ConfParamDatatype string `json:"confParamDatatype,omitempty"` } func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsConfig, error) { @@ -142,10 +146,10 @@ func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsC for _, param := range params { switch param.ConfParamShort { - case "THREADWINDS_API_KEY": + case "utmstack.tw.apiKey": config.APIKey = param.ConfParamValue config.KeyID = param.ID - case "THREADWINDS_API_SECRET": + case "utmstack.tw.apiSecret": config.APISecret = param.ConfParamValue config.SecretID = param.ID } @@ -160,16 +164,24 @@ func (c *BackendClient) SaveThreadWindsCredentials(ctx context.Context, apiKey, params := []ConfigParameter{ { - ID: keyID, - SectionID: threadwindsSectionID, - ConfParamShort: "THREADWINDS_API_KEY", - ConfParamValue: apiKey, + ID: keyID, + SectionID: threadwindsSectionID, + ConfParamShort: "utmstack.tw.apiKey", + ConfParamLarge: "ThreatWinds API Key", + ConfParamDescription: "API Key for ThreatWinds integration.", + ConfParamValue: apiKey, + ConfParamRequired: true, + ConfParamDatatype: "text", }, { - ID: secretID, - SectionID: threadwindsSectionID, - ConfParamShort: "THREADWINDS_API_SECRET", - ConfParamValue: apiSecret, + ID: secretID, + SectionID: threadwindsSectionID, + ConfParamShort: "utmstack.tw.apiSecret", + ConfParamLarge: "ThreatWinds API Secret", + ConfParamDescription: "API Secret for ThreatWinds integration.", + ConfParamValue: apiSecret, + ConfParamRequired: true, + ConfParamDatatype: "password", }, } From 258b3962b9ee99caeac580babd8c6c20e838a4b8 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Fri, 2 Jan 2026 18:17:22 -0500 Subject: [PATCH 10/21] feat(threadwinds-ingestion): add AES decryption support for API secret --- threadwinds-ingestion/go.mod | 1 + threadwinds-ingestion/go.sum | 2 ++ .../internal/client/backend_client.go | 15 ++++++++++++++- threadwinds-ingestion/utils/aes.go | 10 ++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 threadwinds-ingestion/utils/aes.go diff --git a/threadwinds-ingestion/go.mod b/threadwinds-ingestion/go.mod index 12ec71bcf..1bb52269b 100644 --- a/threadwinds-ingestion/go.mod +++ b/threadwinds-ingestion/go.mod @@ -3,6 +3,7 @@ module github.com/utmstack/UTMStack/threadwinds-ingestion go 1.25.4 require ( + github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 github.com/lib/pq v1.10.9 github.com/opensearch-project/opensearch-go/v2 v2.3.0 github.com/threatwinds/go-sdk v1.0.47 diff --git a/threadwinds-ingestion/go.sum b/threadwinds-ingestion/go.sum index 08884ce01..fce2de27c 100644 --- a/threadwinds-ingestion/go.sum +++ b/threadwinds-ingestion/go.sum @@ -1,3 +1,5 @@ +github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 h1:TBiBl9KCa4i4epY0/q9WSC4ugavL6+6JUkOXWDnMM6I= +github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0/go.mod h1:cRhQ3TS/VEfu/z+qaciyuDZdtxgaXgaX8+G6Wa5NzBk= github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= diff --git a/threadwinds-ingestion/internal/client/backend_client.go b/threadwinds-ingestion/internal/client/backend_client.go index 5fdc7a378..53f07fe56 100644 --- a/threadwinds-ingestion/internal/client/backend_client.go +++ b/threadwinds-ingestion/internal/client/backend_client.go @@ -12,6 +12,7 @@ import ( "github.com/threatwinds/go-sdk/catcher" "github.com/utmstack/UTMStack/threadwinds-ingestion/config" "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" + "github.com/utmstack/UTMStack/threadwinds-ingestion/utils" ) type BackendClient struct { @@ -150,7 +151,19 @@ func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsC config.APIKey = param.ConfParamValue config.KeyID = param.ID case "utmstack.tw.apiSecret": - config.APISecret = param.ConfParamValue + if param.ConfParamDatatype == "password" && param.ConfParamValue != "" { + decrypted, err := utils.DecryptValue(param.ConfParamValue) + if err != nil { + return nil, fmt.Errorf("failed to decrypt API Secret: %w", err) + } + config.APISecret = decrypted + catcher.Info("API Secret decrypted successfully", map[string]any{ + "encrypted_length": len(param.ConfParamValue), + "decrypted_length": len(decrypted), + }) + } else { + config.APISecret = param.ConfParamValue + } config.SecretID = param.ID } } diff --git a/threadwinds-ingestion/utils/aes.go b/threadwinds-ingestion/utils/aes.go new file mode 100644 index 000000000..3b355aa7b --- /dev/null +++ b/threadwinds-ingestion/utils/aes.go @@ -0,0 +1,10 @@ +package utils + +import ( + "github.com/AtlasInsideCorp/AtlasInsideAES" +) + +func DecryptValue(encryptedValue string) (string, error) { + passphrase := Getenv("ENCRYPTION_KEY") + return AtlasInsideAES.AESDecrypt(encryptedValue, []byte(passphrase)) +} From 65078b7a5cd4e551c981b6ede5d1b95a7a601a96 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Fri, 2 Jan 2026 18:23:00 -0500 Subject: [PATCH 11/21] feat(installer): add ENCRYPTION_KEY env var to threadwinds-ingestion service --- installer/types/compose.go | 1 + 1 file changed, 1 insertion(+) diff --git a/installer/types/compose.go b/installer/types/compose.go index f768ed1f8..c18efb1cb 100644 --- a/installer/types/compose.go +++ b/installer/types/compose.go @@ -606,6 +606,7 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { }, Environment: []string{ "INTERNAL_KEY=" + conf.InternalKey, + "ENCRYPTION_KEY=" + conf.InternalKey, "BACKEND_URL=http://backend:8080", "ENV=" + conf.Branch, "OPENSEARCH_HOST=node1", From c398e3e1626573ebd2e4eb507e19a937b1aba70c Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Mon, 5 Jan 2026 12:41:16 -0500 Subject: [PATCH 12/21] refactor(threadwinds-ingestion): remove VisibleBy field from entity creation in EntityMapper --- threadwinds-ingestion/internal/mapper/entity_mapper.go | 1 - 1 file changed, 1 deletion(-) diff --git a/threadwinds-ingestion/internal/mapper/entity_mapper.go b/threadwinds-ingestion/internal/mapper/entity_mapper.go index 967761d91..11601e7cd 100644 --- a/threadwinds-ingestion/internal/mapper/entity_mapper.go +++ b/threadwinds-ingestion/internal/mapper/entity_mapper.go @@ -90,7 +90,6 @@ func (m *EntityMapper) BuildEntity(entityType string, value any, context EntityE Attributes: attrs, Reputation: reputation, Tags: tags, - VisibleBy: []string{"utmstack"}, Associations: nil, } From fd242a8963a341d82e10cd3267ab026ff8e204f8 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Thu, 8 Jan 2026 11:13:56 -0500 Subject: [PATCH 13/21] refactor: sanitize sensitive data from log messages --- threadwinds-ingestion/internal/client/backend_client.go | 5 +---- threadwinds-ingestion/internal/client/threadwinds_client.go | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/threadwinds-ingestion/internal/client/backend_client.go b/threadwinds-ingestion/internal/client/backend_client.go index 53f07fe56..7dc595920 100644 --- a/threadwinds-ingestion/internal/client/backend_client.go +++ b/threadwinds-ingestion/internal/client/backend_client.go @@ -157,10 +157,7 @@ func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsC return nil, fmt.Errorf("failed to decrypt API Secret: %w", err) } config.APISecret = decrypted - catcher.Info("API Secret decrypted successfully", map[string]any{ - "encrypted_length": len(param.ConfParamValue), - "decrypted_length": len(decrypted), - }) + catcher.Info("API Secret decrypted successfully", nil) } else { config.APISecret = param.ConfParamValue } diff --git a/threadwinds-ingestion/internal/client/threadwinds_client.go b/threadwinds-ingestion/internal/client/threadwinds_client.go index cbe9964c3..1c38f9ebc 100644 --- a/threadwinds-ingestion/internal/client/threadwinds_client.go +++ b/threadwinds-ingestion/internal/client/threadwinds_client.go @@ -39,9 +39,7 @@ func (c *ThreadWindsClient) UpdateCredentials(apiKey, apiSecret string) { c.apiKey = apiKey c.apiSecret = apiSecret - catcher.Info("ThreadWinds credentials updated", map[string]any{ - "api_key_length": len(apiKey), - }) + catcher.Info("ThreadWinds credentials updated", nil) } func (c *ThreadWindsClient) ingestEntity(ctx context.Context, entity *entities.Entity) error { From ad0624b506e366e758b33eab9b3d802cc84a11d8 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Thu, 8 Jan 2026 23:56:44 -0500 Subject: [PATCH 14/21] refactor: simplify association rules and expand threat intelligence coverage --- .../internal/association/association_rules.go | 326 ++++++++++-------- 1 file changed, 191 insertions(+), 135 deletions(-) diff --git a/threadwinds-ingestion/internal/association/association_rules.go b/threadwinds-ingestion/internal/association/association_rules.go index 797f839fb..8ed00d3d8 100644 --- a/threadwinds-ingestion/internal/association/association_rules.go +++ b/threadwinds-ingestion/internal/association/association_rules.go @@ -7,149 +7,205 @@ const ( Aggregation AssociationMode = "aggregation" ) -type RuleCategory string - -const ( - CategoryNetwork RuleCategory = "network" - CategoryIdentity RuleCategory = "identity" -) - type AssociationRule struct { - Name string - SourceType string - TargetType string - Mode AssociationMode - Category RuleCategory - Description string - Enabled bool - Priority int + Name string + SourceType string + TargetType string + Mode AssociationMode + Enabled bool } var DefaultRules = []*AssociationRule{ - // Network Associations - { - Name: "ip-to-port", - SourceType: "ip", - TargetType: "port", - Mode: Association, - Category: CategoryNetwork, - Description: "IP exposes port", - Enabled: true, - Priority: 10, - }, - { - Name: "port-to-ip", - SourceType: "port", - TargetType: "ip", - Mode: Association, - Category: CategoryNetwork, - Description: "Port exposed on IP", - Enabled: true, - Priority: 10, - }, - { - Name: "hostname-to-ip", - SourceType: "hostname", - TargetType: "ip", - Mode: Association, - Category: CategoryNetwork, - Description: "Hostname resolves to IP", - Enabled: true, - Priority: 10, - }, - { - Name: "ip-to-hostname", - SourceType: "ip", - TargetType: "hostname", - Mode: Association, - Category: CategoryNetwork, - Description: "IP resolves to hostname", - Enabled: true, - Priority: 10, + // Network Infrastructure + { + Name: "ip-to-port", + SourceType: "ip", + TargetType: "port", + Mode: Association, + Enabled: true, + }, + { + Name: "port-to-service", + SourceType: "port", + TargetType: "service", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "domain-to-ip", + SourceType: "domain", + TargetType: "ip", + Mode: Association, + Enabled: true, + }, + { + Name: "ip-to-domain", + SourceType: "ip", + TargetType: "domain", + Mode: Association, + Enabled: true, + }, + { + Name: "subdomain-to-domain", + SourceType: "domain", + TargetType: "domain", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "url-to-domain", + SourceType: "url", + TargetType: "domain", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "url-to-ip", + SourceType: "url", + TargetType: "ip", + Mode: Association, + Enabled: true, }, - // Identity Associations - { - Name: "username-to-ip", - SourceType: "username", - TargetType: "ip", - Mode: Association, - Category: CategoryIdentity, - Description: "User accessed from IP", - Enabled: true, - Priority: 10, - }, - { - Name: "ip-to-username", - SourceType: "ip", - TargetType: "username", - Mode: Association, - Category: CategoryIdentity, - Description: "IP accessed by user", - Enabled: true, - Priority: 10, - }, - { - Name: "username-to-hostname", - SourceType: "username", - TargetType: "hostname", - Mode: Association, - Category: CategoryIdentity, - Description: "User accessed from hostname", - Enabled: true, - Priority: 9, - }, - { - Name: "hostname-to-username", - SourceType: "hostname", - TargetType: "username", - Mode: Association, - Category: CategoryIdentity, - Description: "Hostname accessed by user", - Enabled: true, - Priority: 9, + // Geographic and ASN + { + Name: "ip-to-asn", + SourceType: "ip", + TargetType: "asn", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "asn-to-organization", + SourceType: "asn", + TargetType: "organization", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "domain-to-asn", + SourceType: "domain", + TargetType: "asn", + Mode: Association, + Enabled: true, }, - // ASN Associations - { - Name: "ip-to-asn", - SourceType: "ip", - TargetType: "asn", - Mode: Association, - Category: CategoryNetwork, - Description: "IP belongs to ASN", - Enabled: true, - Priority: 10, - }, - { - Name: "asn-to-ip", - SourceType: "asn", - TargetType: "ip", - Mode: Association, - Category: CategoryNetwork, - Description: "ASN contains IP", - Enabled: true, - Priority: 10, - }, - { - Name: "hostname-to-asn", - SourceType: "hostname", - TargetType: "asn", - Mode: Association, - Category: CategoryNetwork, - Description: "Hostname resolves to IP in ASN", - Enabled: true, - Priority: 9, - }, - { - Name: "asn-to-hostname", - SourceType: "asn", - TargetType: "hostname", - Mode: Association, - Category: CategoryNetwork, - Description: "ASN contains hostname", - Enabled: true, - Priority: 9, + // Identity and Access + { + Name: "user-to-ip", + SourceType: "user", + TargetType: "ip", + Mode: Association, + Enabled: true, + }, + { + Name: "user-to-hostname", + SourceType: "user", + TargetType: "hostname", + Mode: Association, + Enabled: true, + }, + { + Name: "user-to-account", + SourceType: "user", + TargetType: "account", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "email-to-user", + SourceType: "email", + TargetType: "user", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "email-to-domain", + SourceType: "email", + TargetType: "domain", + Mode: Aggregation, + Enabled: true, + }, + + // Threat Intelligence + { + Name: "malware-to-ip", + SourceType: "malware", + TargetType: "ip", + Mode: Association, + Enabled: true, + }, + { + Name: "malware-to-domain", + SourceType: "malware", + TargetType: "domain", + Mode: Association, + Enabled: true, + }, + { + Name: "malware-to-url", + SourceType: "malware", + TargetType: "url", + Mode: Association, + Enabled: true, + }, + { + Name: "hash-to-malware", + SourceType: "hash", + TargetType: "malware", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "ip-to-threat-actor", + SourceType: "ip", + TargetType: "threat-actor", + Mode: Association, + Enabled: true, + }, + { + Name: "domain-to-threat-actor", + SourceType: "domain", + TargetType: "threat-actor", + Mode: Association, + Enabled: true, + }, + { + Name: "cve-to-exploit", + SourceType: "cve", + TargetType: "exploit", + Mode: Aggregation, + Enabled: true, + }, + { + Name: "vulnerability-to-ip", + SourceType: "vulnerability", + TargetType: "ip", + Mode: Association, + Enabled: true, + }, + + // Legacy (backward compatibility) + { + Name: "hostname-to-ip", + SourceType: "hostname", + TargetType: "ip", + Mode: Association, + Enabled: true, + }, + { + Name: "ip-to-hostname", + SourceType: "ip", + TargetType: "hostname", + Mode: Association, + Enabled: true, + }, + { + Name: "hostname-to-user", + SourceType: "hostname", + TargetType: "user", + Mode: Association, + Enabled: true, }, } From ab16ad05843b691da1c81a2b23ba9cf9e3ee751d Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 12 Jan 2026 11:18:45 -0600 Subject: [PATCH 15/21] feat(threatwinds): add configuration parameters for ThreatWinds integration --- ...1_insert_threatwinds_credentials_section.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml index 47c95121b..c0efaf6c9 100644 --- a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml +++ b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml @@ -15,6 +15,22 @@ + + + + + + + + + + + + + + + + @@ -29,6 +45,7 @@ + From 263993af2139162467f86246f477aff754d6efec Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 13 Jan 2026 10:34:36 -0500 Subject: [PATCH 16/21] chore: remove threadwinds-ingestion microservice --- threadwinds-ingestion/Dockerfile | 10 - threadwinds-ingestion/config/config.go | 33 -- threadwinds-ingestion/config/const.go | 71 ---- threadwinds-ingestion/go.mod | 51 --- threadwinds-ingestion/go.sum | 174 --------- .../association/association_builder.go | 156 -------- .../association/association_context.go | 40 -- .../internal/association/association_rules.go | 220 ----------- .../internal/client/backend_client.go | 224 ----------- .../internal/client/cm_client.go | 80 ---- .../internal/client/opensearch_client.go | 96 ----- .../internal/client/postgres_client.go | 159 -------- .../internal/client/threadwinds_client.go | 152 -------- .../internal/extractor/field_extractor.go | 111 ------ .../internal/mapper/entity_mapper.go | 126 ------- .../internal/models/alert.go | 30 -- .../internal/models/event.go | 7 - .../internal/models/incident.go | 21 -- .../internal/scheduler/ingestion_scheduler.go | 349 ------------------ threadwinds-ingestion/main.go | 128 ------- threadwinds-ingestion/utils/aes.go | 10 - threadwinds-ingestion/utils/check.go | 48 --- threadwinds-ingestion/utils/env.go | 20 - threadwinds-ingestion/utils/req.go | 46 --- 24 files changed, 2362 deletions(-) delete mode 100644 threadwinds-ingestion/Dockerfile delete mode 100644 threadwinds-ingestion/config/config.go delete mode 100644 threadwinds-ingestion/config/const.go delete mode 100644 threadwinds-ingestion/go.mod delete mode 100644 threadwinds-ingestion/go.sum delete mode 100644 threadwinds-ingestion/internal/association/association_builder.go delete mode 100644 threadwinds-ingestion/internal/association/association_context.go delete mode 100644 threadwinds-ingestion/internal/association/association_rules.go delete mode 100644 threadwinds-ingestion/internal/client/backend_client.go delete mode 100644 threadwinds-ingestion/internal/client/cm_client.go delete mode 100644 threadwinds-ingestion/internal/client/opensearch_client.go delete mode 100644 threadwinds-ingestion/internal/client/postgres_client.go delete mode 100644 threadwinds-ingestion/internal/client/threadwinds_client.go delete mode 100644 threadwinds-ingestion/internal/extractor/field_extractor.go delete mode 100644 threadwinds-ingestion/internal/mapper/entity_mapper.go delete mode 100644 threadwinds-ingestion/internal/models/alert.go delete mode 100644 threadwinds-ingestion/internal/models/event.go delete mode 100644 threadwinds-ingestion/internal/models/incident.go delete mode 100644 threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go delete mode 100644 threadwinds-ingestion/main.go delete mode 100644 threadwinds-ingestion/utils/aes.go delete mode 100644 threadwinds-ingestion/utils/check.go delete mode 100644 threadwinds-ingestion/utils/env.go delete mode 100644 threadwinds-ingestion/utils/req.go diff --git a/threadwinds-ingestion/Dockerfile b/threadwinds-ingestion/Dockerfile deleted file mode 100644 index 39d4bbb42..000000000 --- a/threadwinds-ingestion/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:24.04 - -RUN apt-get update -RUN apt-get install -y ca-certificates -RUN update-ca-certificates - -COPY threadwinds-ingestion . - -RUN chmod +x threadwinds-ingestion -ENTRYPOINT ./threadwinds-ingestion \ No newline at end of file diff --git a/threadwinds-ingestion/config/config.go b/threadwinds-ingestion/config/config.go deleted file mode 100644 index e8a4128ad..000000000 --- a/threadwinds-ingestion/config/config.go +++ /dev/null @@ -1,33 +0,0 @@ -package config - -type TWConfig struct { - InternalKey string - BackendURL string - CustomersManagerURL string - ThreadWindsURL string - OpenSearchHost string - OpenSearchPort string - DBHost string - DBPort string - DBUser string - DBPassword string - DBName string -} - -func GetTWConfig() (*TWConfig, error) { - cfg := &TWConfig{ - InternalKey: GetInternalKey(), - BackendURL: GetBackendUrl(), - CustomersManagerURL: GetCustomersManagerUrl(), - ThreadWindsURL: GetThreadWindsURL(), - OpenSearchHost: GetOpenSearchHost(), - OpenSearchPort: GetOpenSearchPort(), - DBHost: GetDBHost(), - DBPort: GetDBPort(), - DBUser: GetDBUser(), - DBPassword: GetDBPassword(), - DBName: GetDBName(), - } - - return cfg, nil -} diff --git a/threadwinds-ingestion/config/const.go b/threadwinds-ingestion/config/const.go deleted file mode 100644 index 616c80955..000000000 --- a/threadwinds-ingestion/config/const.go +++ /dev/null @@ -1,71 +0,0 @@ -package config - -import ( - "os" - "strings" - - "github.com/utmstack/UTMStack/threadwinds-ingestion/utils" -) - -func GetInternalKey() string { - return utils.Getenv("INTERNAL_KEY") -} - -func GetBackendUrl() string { - return utils.Getenv("BACKEND_URL") -} - -func GetCustomersManagerUrl() string { - if isDevEnvironment() { - return "https://cm.dev.utmstack.com" - } - return "https://cm.utmstack.com" -} - -func GetThreadWindsURL() string { - if isDevEnvironment() { - return "https://apis.dev.threatwinds.com" - } - return "https://apis.threatwinds.com" -} - -func GetOpenSearchHost() string { - return utils.Getenv("OPENSEARCH_HOST") -} - -func GetOpenSearchPort() string { - return utils.Getenv("OPENSEARCH_PORT") -} - -func GetDBHost() string { - return utils.Getenv("DB_HOST") -} - -func GetDBPort() string { - return utils.Getenv("DB_PORT") -} - -func GetDBUser() string { - return utils.Getenv("DB_USER") -} - -func GetDBPassword() string { - return utils.Getenv("DB_PASS") -} - -func GetDBName() string { - return utils.Getenv("DB_NAME") -} - -func isDevEnvironment() bool { - env := os.Getenv("ENV") - if env != "" { - if strings.Contains(env, "-dev") || - strings.Contains(env, "-qa") || - strings.Contains(env, "-rc") { - return true - } - } - - return false -} diff --git a/threadwinds-ingestion/go.mod b/threadwinds-ingestion/go.mod deleted file mode 100644 index 1bb52269b..000000000 --- a/threadwinds-ingestion/go.mod +++ /dev/null @@ -1,51 +0,0 @@ -module github.com/utmstack/UTMStack/threadwinds-ingestion - -go 1.25.4 - -require ( - github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 - github.com/lib/pq v1.10.9 - github.com/opensearch-project/opensearch-go/v2 v2.3.0 - github.com/threatwinds/go-sdk v1.0.47 -) - -require ( - github.com/bytedance/gopkg v0.1.3 // indirect - github.com/bytedance/sonic v1.14.2 // indirect - github.com/bytedance/sonic/loader v0.4.0 // indirect - github.com/cloudwego/base64x v0.1.6 // indirect - github.com/gabriel-vasile/mimetype v1.4.11 // indirect - github.com/gin-contrib/sse v1.1.0 // indirect - github.com/gin-gonic/gin v1.11.0 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.29.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/goccy/go-yaml v1.19.0 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/leodido/go-urn v1.4.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.1 // indirect - github.com/tidwall/gjson v1.18.0 // indirect - github.com/tidwall/match v1.2.0 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.3.1 // indirect - go.uber.org/mock v0.6.0 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/arch v0.23.0 // indirect - golang.org/x/crypto v0.46.0 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/text v0.32.0 // indirect - google.golang.org/protobuf v1.36.10 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect -) diff --git a/threadwinds-ingestion/go.sum b/threadwinds-ingestion/go.sum deleted file mode 100644 index fce2de27c..000000000 --- a/threadwinds-ingestion/go.sum +++ /dev/null @@ -1,174 +0,0 @@ -github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 h1:TBiBl9KCa4i4epY0/q9WSC4ugavL6+6JUkOXWDnMM6I= -github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0/go.mod h1:cRhQ3TS/VEfu/z+qaciyuDZdtxgaXgaX8+G6Wa5NzBk= -github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= -github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= -github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= -github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= -github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= -github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= -github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= -github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik= -github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= -github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= -github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= -github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.29.0 h1:lQlF5VNJWNlRbRZNeOIkWElR+1LL/OuHcc0Kp14w1xk= -github.com/go-playground/validator/v10 v10.29.0/go.mod h1:D6QxqeMlgIPuT02L66f2ccrZ7AGgHkzKmmTMZhk/Kc4= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= -github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= -github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= -github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= -github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/threatwinds/go-sdk v1.0.47 h1:z54WA6pt95IiCwjfARx7S0I1GRD3X5WH2hspwhKrgPU= -github.com/threatwinds/go-sdk v1.0.47/go.mod h1:NXdavG6meLH3WzIy0rWvqHJusun8HyfneW7zW2VyJeI= -github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= -github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= -github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= -github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= -go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= -go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= -go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= -golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= -golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/threadwinds-ingestion/internal/association/association_builder.go b/threadwinds-ingestion/internal/association/association_builder.go deleted file mode 100644 index 175d0a829..000000000 --- a/threadwinds-ingestion/internal/association/association_builder.go +++ /dev/null @@ -1,156 +0,0 @@ -package association - -import ( - "sync" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/entities" -) - -type AssociationBuilder struct { - rules []*AssociationRule - entityRegistry *sync.Map - mu sync.RWMutex -} - -func NewAssociationBuilder() *AssociationBuilder { - builder := &AssociationBuilder{ - rules: GetEnabledRules(), - entityRegistry: &sync.Map{}, - } - catcher.Info("association builder initialized", map[string]any{ - "total_rules": len(builder.rules), - }) - return builder -} - -func (b *AssociationBuilder) RegisterEntity(entity *entities.Entity, entityID, sourcePath string, ctx AssociationContext) { - ref := &EntityReference{ - Entity: entity, - EntityID: entityID, - EntityType: entity.Type, - SourcePath: sourcePath, - Context: ctx, - } - b.entityRegistry.Store(entityID, ref) - catcher.Info("entity registered", map[string]any{ - "entity_id": entityID, - "entity_type": entity.Type, - "source_path": sourcePath, - }) -} - -func (b *AssociationBuilder) BuildAssociations() []*entities.Entity { - contextGroups := b.groupByContext() - for _, refs := range contextGroups { - b.detectAssociationsInContext(refs) - } - result := make([]*entities.Entity, 0) - b.entityRegistry.Range(func(key, value any) bool { - if ref, ok := value.(*EntityReference); ok { - if entity, ok := ref.Entity.(*entities.Entity); ok { - result = append(result, entity) - } - } - return true - }) - catcher.Info("associations built", map[string]any{ - "total_entities": len(result), - "context_groups": len(contextGroups), - "total_associations": b.CountAssociations(result), - }) - return result -} - -func (b *AssociationBuilder) groupByContext() map[string][]*EntityReference { - groups := make(map[string][]*EntityReference) - b.entityRegistry.Range(func(key, value any) bool { - if ref, ok := value.(*EntityReference); ok { - contextKey := ref.Context.AlertID - if contextKey != "" { - groups[contextKey] = append(groups[contextKey], ref) - } - } - return true - }) - return groups -} - -func (b *AssociationBuilder) detectAssociationsInContext(refs []*EntityReference) { - for _, rule := range b.rules { - if !rule.Enabled { - continue - } - for i, sourceRef := range refs { - if sourceRef.EntityType != rule.SourceType { - continue - } - for j, targetRef := range refs { - if i == j { - continue - } - if targetRef.EntityType != rule.TargetType { - continue - } - if b.shouldCreateAssociation(sourceRef, targetRef) { - b.createAssociation(sourceRef, targetRef, rule) - } - } - } - } -} - -func (b *AssociationBuilder) shouldCreateAssociation(source, target *EntityReference) bool { - if source.Context.SameContext(target.Context) { - return true - } - if source.Context.IsOriginToTarget(target.Context) { - return true - } - if source.Context.IsTargetToOrigin(target.Context) { - return true - } - return false -} - -func (b *AssociationBuilder) createAssociation(source, target *EntityReference, rule *AssociationRule) { - sourceEntity, ok := source.Entity.(*entities.Entity) - if !ok { - return - } - targetEntity, ok := target.Entity.(*entities.Entity) - if !ok { - return - } - associatedEntity := entities.EntityAssociation{ - Mode: string(rule.Mode), - Entity: entities.Entity{ - Type: targetEntity.Type, - Attributes: targetEntity.Attributes, - }, - } - if sourceEntity.Associations == nil { - sourceEntity.Associations = make([]entities.EntityAssociation, 0) - } - sourceEntity.Associations = append(sourceEntity.Associations, associatedEntity) - catcher.Info("association created", map[string]any{ - "rule": rule.Name, - "source_type": sourceEntity.Type, - "target_type": targetEntity.Type, - "mode": rule.Mode, - "context": source.Context.AlertID, - }) -} - -func (b *AssociationBuilder) CountAssociations(entities []*entities.Entity) int { - count := 0 - for _, entity := range entities { - count += len(entity.Associations) - } - return count -} - -func (b *AssociationBuilder) ClearRegistry() { - b.entityRegistry = &sync.Map{} - catcher.Info("entity registry cleared", nil) -} diff --git a/threadwinds-ingestion/internal/association/association_context.go b/threadwinds-ingestion/internal/association/association_context.go deleted file mode 100644 index b2e88bd86..000000000 --- a/threadwinds-ingestion/internal/association/association_context.go +++ /dev/null @@ -1,40 +0,0 @@ -package association - -type ContextType string - -type AssociationContext struct { - AlertID string - IncidentID string - SourceField string -} - -type EntityReference struct { - Entity any - EntityID string - EntityType string - SourcePath string - Context AssociationContext -} - -func (ctx *AssociationContext) IsOrigin() bool { - return ctx.SourceField == "source" -} - -func (ctx *AssociationContext) IsTarget() bool { - return ctx.SourceField == "destination" -} - -func (ctx *AssociationContext) SameContext(other AssociationContext) bool { - if ctx.AlertID != "" && ctx.AlertID == other.AlertID { - return true - } - return false -} - -func (ctx *AssociationContext) IsOriginToTarget(other AssociationContext) bool { - return ctx.IsOrigin() && other.IsTarget() && ctx.SameContext(other) -} - -func (ctx *AssociationContext) IsTargetToOrigin(other AssociationContext) bool { - return ctx.IsTarget() && other.IsOrigin() && ctx.SameContext(other) -} diff --git a/threadwinds-ingestion/internal/association/association_rules.go b/threadwinds-ingestion/internal/association/association_rules.go deleted file mode 100644 index 8ed00d3d8..000000000 --- a/threadwinds-ingestion/internal/association/association_rules.go +++ /dev/null @@ -1,220 +0,0 @@ -package association - -type AssociationMode string - -const ( - Association AssociationMode = "association" - Aggregation AssociationMode = "aggregation" -) - -type AssociationRule struct { - Name string - SourceType string - TargetType string - Mode AssociationMode - Enabled bool -} - -var DefaultRules = []*AssociationRule{ - // Network Infrastructure - { - Name: "ip-to-port", - SourceType: "ip", - TargetType: "port", - Mode: Association, - Enabled: true, - }, - { - Name: "port-to-service", - SourceType: "port", - TargetType: "service", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "domain-to-ip", - SourceType: "domain", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-domain", - SourceType: "ip", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "subdomain-to-domain", - SourceType: "domain", - TargetType: "domain", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "url-to-domain", - SourceType: "url", - TargetType: "domain", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "url-to-ip", - SourceType: "url", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - - // Geographic and ASN - { - Name: "ip-to-asn", - SourceType: "ip", - TargetType: "asn", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "asn-to-organization", - SourceType: "asn", - TargetType: "organization", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "domain-to-asn", - SourceType: "domain", - TargetType: "asn", - Mode: Association, - Enabled: true, - }, - - // Identity and Access - { - Name: "user-to-ip", - SourceType: "user", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "user-to-hostname", - SourceType: "user", - TargetType: "hostname", - Mode: Association, - Enabled: true, - }, - { - Name: "user-to-account", - SourceType: "user", - TargetType: "account", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "email-to-user", - SourceType: "email", - TargetType: "user", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "email-to-domain", - SourceType: "email", - TargetType: "domain", - Mode: Aggregation, - Enabled: true, - }, - - // Threat Intelligence - { - Name: "malware-to-ip", - SourceType: "malware", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "malware-to-domain", - SourceType: "malware", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "malware-to-url", - SourceType: "malware", - TargetType: "url", - Mode: Association, - Enabled: true, - }, - { - Name: "hash-to-malware", - SourceType: "hash", - TargetType: "malware", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "ip-to-threat-actor", - SourceType: "ip", - TargetType: "threat-actor", - Mode: Association, - Enabled: true, - }, - { - Name: "domain-to-threat-actor", - SourceType: "domain", - TargetType: "threat-actor", - Mode: Association, - Enabled: true, - }, - { - Name: "cve-to-exploit", - SourceType: "cve", - TargetType: "exploit", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "vulnerability-to-ip", - SourceType: "vulnerability", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - - // Legacy (backward compatibility) - { - Name: "hostname-to-ip", - SourceType: "hostname", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-hostname", - SourceType: "ip", - TargetType: "hostname", - Mode: Association, - Enabled: true, - }, - { - Name: "hostname-to-user", - SourceType: "hostname", - TargetType: "user", - Mode: Association, - Enabled: true, - }, -} - -func GetEnabledRules() []*AssociationRule { - rules := make([]*AssociationRule, 0) - for _, rule := range DefaultRules { - if rule.Enabled { - rules = append(rules, rule) - } - } - return rules -} diff --git a/threadwinds-ingestion/internal/client/backend_client.go b/threadwinds-ingestion/internal/client/backend_client.go deleted file mode 100644 index 7dc595920..000000000 --- a/threadwinds-ingestion/internal/client/backend_client.go +++ /dev/null @@ -1,224 +0,0 @@ -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" - "github.com/utmstack/UTMStack/threadwinds-ingestion/utils" -) - -type BackendClient struct { - baseURL string - internalKey string - httpClient *http.Client -} - -func NewBackendClient(cfg *config.TWConfig) *BackendClient { - return &BackendClient{ - baseURL: cfg.BackendURL, - internalKey: cfg.InternalKey, - httpClient: &http.Client{ - Timeout: 30 * time.Second, - }, - } -} - -func (c *BackendClient) GetRecentIncidents(ctx context.Context) ([]*models.Incident, error) { - url := fmt.Sprintf("%s/api/utm-incidents?incidentStatus.in=OPEN,IN_REVIEW&sort=incidentCreatedDate,desc&size=100", c.baseURL) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) - } - - var incidents []*models.Incident - if err := json.NewDecoder(resp.Body).Decode(&incidents); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - catcher.Info("fetched recent incidents", map[string]any{ - "count": len(incidents), - }) - - return incidents, nil -} - -func (c *BackendClient) GetIncidentAlerts(ctx context.Context, incidentID int64) ([]*models.IncidentAlert, error) { - url := fmt.Sprintf("%s/api/utm-incident-alerts?incidentId.equals=%d", c.baseURL, incidentID) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) - } - - var alerts []*models.IncidentAlert - if err := json.NewDecoder(resp.Body).Decode(&alerts); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - catcher.Info("fetched incident alerts", map[string]any{ - "incident_id": incidentID, - "alert_count": len(alerts), - }) - - return alerts, nil -} - -type ThreadWindsConfig struct { - APIKey string - APISecret string - KeyID int64 - SecretID int64 -} - -type ConfigParameter struct { - ID int64 `json:"id"` - SectionID int64 `json:"sectionId"` - ConfParamShort string `json:"confParamShort"` - ConfParamLarge string `json:"confParamLarge,omitempty"` - ConfParamDescription string `json:"confParamDescription,omitempty"` - ConfParamValue string `json:"confParamValue"` - ConfParamRequired bool `json:"confParamRequired,omitempty"` - ConfParamDatatype string `json:"confParamDatatype,omitempty"` -} - -func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsConfig, error) { - const threadwindsSectionID = 6 - url := fmt.Sprintf("%s/api/utm-configuration-parameters?sectionId.equals=%d&size=100", c.baseURL, threadwindsSectionID) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) - } - - var params []ConfigParameter - if err := json.NewDecoder(resp.Body).Decode(¶ms); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - config := &ThreadWindsConfig{} - - for _, param := range params { - switch param.ConfParamShort { - case "utmstack.tw.apiKey": - config.APIKey = param.ConfParamValue - config.KeyID = param.ID - case "utmstack.tw.apiSecret": - if param.ConfParamDatatype == "password" && param.ConfParamValue != "" { - decrypted, err := utils.DecryptValue(param.ConfParamValue) - if err != nil { - return nil, fmt.Errorf("failed to decrypt API Secret: %w", err) - } - config.APISecret = decrypted - catcher.Info("API Secret decrypted successfully", nil) - } else { - config.APISecret = param.ConfParamValue - } - config.SecretID = param.ID - } - } - - return config, nil -} - -func (c *BackendClient) SaveThreadWindsCredentials(ctx context.Context, apiKey, apiSecret string, keyID, secretID int64) error { - const threadwindsSectionID = 6 - url := fmt.Sprintf("%s/api/utm-configuration-parameters", c.baseURL) - - params := []ConfigParameter{ - { - ID: keyID, - SectionID: threadwindsSectionID, - ConfParamShort: "utmstack.tw.apiKey", - ConfParamLarge: "ThreatWinds API Key", - ConfParamDescription: "API Key for ThreatWinds integration.", - ConfParamValue: apiKey, - ConfParamRequired: true, - ConfParamDatatype: "text", - }, - { - ID: secretID, - SectionID: threadwindsSectionID, - ConfParamShort: "utmstack.tw.apiSecret", - ConfParamLarge: "ThreatWinds API Secret", - ConfParamDescription: "API Secret for ThreatWinds integration.", - ConfParamValue: apiSecret, - ConfParamRequired: true, - ConfParamDatatype: "password", - }, - } - - payload, err := json.Marshal(params) - if err != nil { - return fmt.Errorf("failed to marshal parameters: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, "PUT", url, bytes.NewReader(payload)) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) - } - - catcher.Info("ThreadWinds credentials saved successfully", nil) - return nil -} diff --git a/threadwinds-ingestion/internal/client/cm_client.go b/threadwinds-ingestion/internal/client/cm_client.go deleted file mode 100644 index 2afdfe4b3..000000000 --- a/threadwinds-ingestion/internal/client/cm_client.go +++ /dev/null @@ -1,80 +0,0 @@ -package client - -import ( - "encoding/json" - "fmt" - "net/http" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/utils" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" -) - -type CustomersManagerClient struct { - cmURL string -} - -func NewCustomersManagerClient(cfg *config.TWConfig) *CustomersManagerClient { - return &CustomersManagerClient{ - cmURL: cfg.CustomersManagerURL, - } -} - -type RegistrationRequest struct { - Email string `json:"email" validate:"required,email"` -} - -type RegistrationResponse struct { - APIKey string `json:"api_key"` - APISecret string `json:"api_secret"` -} - -func (c *CustomersManagerClient) RegisterUserReporter(email string) (*RegistrationResponse, error) { - endpoint := fmt.Sprintf("%s/api/v1/intelligence/register", c.cmURL) - - reqBody := RegistrationRequest{ - Email: email, - } - - jsonData, err := json.Marshal(reqBody) - if err != nil { - return nil, fmt.Errorf("failed to marshal registration request: %w", err) - } - - headers := map[string]string{ - "Content-Type": "application/json", - } - - credentials, statusCode, err := utils.DoReq[RegistrationResponse]( - endpoint, - jsonData, - http.MethodPost, - headers, - ) - - if err != nil { - switch statusCode { - case http.StatusBadRequest: - return nil, catcher.Error("invalid registration data", err, map[string]any{ - "status": statusCode, - "email": email, - }) - case http.StatusInternalServerError: - return nil, catcher.Error("registration service error", err, map[string]any{ - "status": statusCode, - "email": email, - }) - default: - return nil, catcher.Error("registration failed", err, map[string]any{ - "status": statusCode, - "email": email, - }) - } - } - - catcher.Info("Successfully registered ThreadWinds intelligence reporter", map[string]any{ - "email": email, - }) - - return &credentials, nil -} diff --git a/threadwinds-ingestion/internal/client/opensearch_client.go b/threadwinds-ingestion/internal/client/opensearch_client.go deleted file mode 100644 index 1f90095d8..000000000 --- a/threadwinds-ingestion/internal/client/opensearch_client.go +++ /dev/null @@ -1,96 +0,0 @@ -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - - opensearch "github.com/opensearch-project/opensearch-go/v2" - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" -) - -type OpenSearchClient struct { - client *opensearch.Client -} - -func NewOpenSearchClient(cfg *config.TWConfig) (*OpenSearchClient, error) { - osConfig := opensearch.Config{ - Addresses: []string{ - fmt.Sprintf("http://%s:%s", cfg.OpenSearchHost, cfg.OpenSearchPort), - }, - } - - client, err := opensearch.NewClient(osConfig) - if err != nil { - return nil, fmt.Errorf("failed to create opensearch client: %w", err) - } - - info, err := client.Info() - if err != nil { - return nil, fmt.Errorf("opensearch connection failed: %w", err) - } - defer info.Body.Close() - - catcher.Info("opensearch client connected successfully", map[string]any{ - "host": cfg.OpenSearchHost, - "port": cfg.OpenSearchPort, - }) - - return &OpenSearchClient{client: client}, nil -} - -func (c *OpenSearchClient) GetAlertByID(ctx context.Context, alertID string) (*models.Alert, error) { - query := map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "id.keyword": alertID, - }, - }, - } - - return c.searchSingleAlert(ctx, "alert-*", query) -} - -func (c *OpenSearchClient) searchSingleAlert(ctx context.Context, index string, query map[string]any) (*models.Alert, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(query); err != nil { - return nil, fmt.Errorf("failed to encode query: %w", err) - } - - res, err := c.client.Search( - c.client.Search.WithContext(ctx), - c.client.Search.WithIndex(index), - c.client.Search.WithBody(&buf), - ) - if err != nil { - return nil, fmt.Errorf("search failed: %w", err) - } - defer res.Body.Close() - - if res.IsError() { - body, _ := io.ReadAll(res.Body) - return nil, fmt.Errorf("search error %d: %s", res.StatusCode, string(body)) - } - - var result struct { - Hits struct { - Hits []struct { - Source models.Alert `json:"_source"` - } `json:"hits"` - } `json:"hits"` - } - - if err := json.NewDecoder(res.Body).Decode(&result); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) - } - - if len(result.Hits.Hits) == 0 { - return nil, fmt.Errorf("alert not found") - } - - return &result.Hits.Hits[0].Source, nil -} diff --git a/threadwinds-ingestion/internal/client/postgres_client.go b/threadwinds-ingestion/internal/client/postgres_client.go deleted file mode 100644 index ed54e56be..000000000 --- a/threadwinds-ingestion/internal/client/postgres_client.go +++ /dev/null @@ -1,159 +0,0 @@ -package client - -import ( - "context" - "database/sql" - "fmt" - "time" - - _ "github.com/lib/pq" - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" -) - -type PostgresClient struct { - db *sql.DB -} - -type AdminEmailResult struct { - Email string - IsConfigured bool - LastModified time.Time - LastModifiedBy string -} - -func NewPostgresClient(cfg *config.TWConfig) (*PostgresClient, error) { - dsn := fmt.Sprintf( - "host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", - cfg.DBHost, - cfg.DBPort, - cfg.DBUser, - cfg.DBPassword, - cfg.DBName, - ) - - db, err := sql.Open("postgres", dsn) - if err != nil { - return nil, fmt.Errorf("failed to open database: %w", err) - } - - db.SetMaxOpenConns(2) - db.SetMaxIdleConns(1) - db.SetConnMaxLifetime(5 * time.Minute) - - if err := db.Ping(); err != nil { - db.Close() - return nil, fmt.Errorf("failed to ping database: %w", err) - } - - catcher.Info("PostgreSQL connection established via native SQL", map[string]any{ - "host": cfg.DBHost, - "port": cfg.DBPort, - }) - - return &PostgresClient{db: db}, nil -} - -func (c *PostgresClient) Close() error { - if c.db != nil { - return c.db.Close() - } - return nil -} - -func (c *PostgresClient) GetAdminEmail(ctx context.Context) (*AdminEmailResult, error) { - query := ` - SELECT email, last_modified_by, last_modified_date - FROM jhi_user - WHERE login = $1 AND created_by = $2 - LIMIT 1 - ` - - var email, lastModifiedBy string - var lastModifiedDate sql.NullTime - - err := c.db.QueryRowContext(ctx, query, "admin", "system"). - Scan(&email, &lastModifiedBy, &lastModifiedDate) - - if err == sql.ErrNoRows { - return nil, fmt.Errorf("admin user not found in database") - } - if err != nil { - return nil, fmt.Errorf("failed to query admin user: %w", err) - } - - result := &AdminEmailResult{ - Email: email, - IsConfigured: email != "admin@localhost", - LastModifiedBy: lastModifiedBy, - } - - if lastModifiedDate.Valid { - result.LastModified = lastModifiedDate.Time - } - - catcher.Info("retrieved current admin email", map[string]any{ - "email": email, - "last_modified_by": lastModifiedBy, - }) - - return result, nil -} - -func (c *PostgresClient) WaitForValidAdminEmail(ctx context.Context, timeout time.Duration) (string, error) { - deadline := time.Now().Add(timeout) - retryInterval := 10 * time.Second - attempt := 1 - - catcher.Info("waiting for admin email configuration before starting service", map[string]any{ - "timeout_minutes": timeout.Minutes(), - "retry_interval": retryInterval.String(), - }) - - query := ` - SELECT email - FROM jhi_user - WHERE login = $1 AND created_by = $2 AND email != $3 - LIMIT 1 - ` - - for time.Now().Before(deadline) { - queryCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - - var email string - err := c.db.QueryRowContext(queryCtx, query, "admin", "system", "admin@localhost"). - Scan(&email) - cancel() - - if err == nil { - catcher.Info("Admin email configured successfully, starting ThreadWinds ingestion", map[string]any{ - "attempts": attempt, - "waited": time.Since(time.Now().Add(-timeout)).Round(time.Second).String(), - }) - return email, nil - } - - if err != sql.ErrNoRows { - catcher.Error("database error while checking admin email", err, map[string]any{ - "attempt": attempt, - }) - } else { - remainingTime := time.Until(deadline).Round(time.Second) - catcher.Info("Admin email not configured yet, waiting...", map[string]any{ - "attempt": attempt, - "next_retry_in": retryInterval.String(), - "remaining_time": remainingTime.String(), - "message": "Service will start automatically once admin configures their email", - }) - } - - select { - case <-ctx.Done(): - return "", fmt.Errorf("context cancelled while waiting for admin email: %w", ctx.Err()) - case <-time.After(retryInterval): - attempt++ - } - } - - return "", fmt.Errorf("timeout after %v waiting for admin email configuration (%d attempts). Admin must configure email in first login", timeout, attempt-1) -} diff --git a/threadwinds-ingestion/internal/client/threadwinds_client.go b/threadwinds-ingestion/internal/client/threadwinds_client.go deleted file mode 100644 index 1c38f9ebc..000000000 --- a/threadwinds-ingestion/internal/client/threadwinds_client.go +++ /dev/null @@ -1,152 +0,0 @@ -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "sync" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/entities" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" -) - -type ThreadWindsClient struct { - baseURL string - apiKey string - apiSecret string - httpClient *http.Client - mu sync.RWMutex -} - -func NewThreadWindsClient(cfg *config.TWConfig) *ThreadWindsClient { - return &ThreadWindsClient{ - baseURL: cfg.ThreadWindsURL, - httpClient: &http.Client{ - Timeout: 30 * time.Second, - }, - } -} - -func (c *ThreadWindsClient) UpdateCredentials(apiKey, apiSecret string) { - c.mu.Lock() - defer c.mu.Unlock() - - c.apiKey = apiKey - c.apiSecret = apiSecret - - catcher.Info("ThreadWinds credentials updated", nil) -} - -func (c *ThreadWindsClient) ingestEntity(ctx context.Context, entity *entities.Entity) error { - url := fmt.Sprintf("%s/api/ingest/v1/entity", c.baseURL) - - payload, err := json.Marshal(entity) - if err != nil { - return fmt.Errorf("failed to marshal entity: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("api-key", c.apiKey) - req.Header.Set("api-secret", c.apiSecret) - - return c.executeWithRetry(req, entity.Type) -} - -func (c *ThreadWindsClient) executeWithRetry(req *http.Request, entityType string) error { - maxRetries := 3 - backoff := time.Second - - for attempt := 1; attempt <= maxRetries; attempt++ { - resp, err := c.httpClient.Do(req) - if err != nil { - catcher.Error("http request failed", err, map[string]any{ - "attempt": attempt, - "entity_type": entityType, - }) - if attempt < maxRetries { - time.Sleep(backoff) - backoff *= 2 - continue - } - return fmt.Errorf("failed after %d attempts: %w", maxRetries, err) - } - - defer resp.Body.Close() - - body, _ := io.ReadAll(resp.Body) - - if resp.StatusCode == http.StatusAccepted { - catcher.Info("entity ingested successfully", map[string]any{ - "entity_type": entityType, - "status_code": resp.StatusCode, - "response": string(body), - }) - return nil - } - - if resp.StatusCode >= 400 && resp.StatusCode < 500 { - return fmt.Errorf("client error %d: %s", resp.StatusCode, string(body)) - } - - if resp.StatusCode >= 500 && attempt < maxRetries { - catcher.Error("server error, retrying", fmt.Errorf("server error %d", resp.StatusCode), map[string]any{ - "attempt": attempt, - "entity_type": entityType, - "response": string(body), - }) - time.Sleep(backoff) - backoff *= 2 - continue - } - - return fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) - } - - return fmt.Errorf("max retries exceeded") -} - -func (c *ThreadWindsClient) IngestBatch(ctx context.Context, entityBatch []*entities.Entity) error { - successCount := 0 - errorCount := 0 - - for i, entity := range entityBatch { - select { - case <-ctx.Done(): - return fmt.Errorf("batch ingestion cancelled: %w (processed %d/%d)", ctx.Err(), successCount, len(entityBatch)) - default: - } - - err := c.ingestEntity(ctx, entity) - if err != nil { - errorCount++ - catcher.Error("failed to ingest entity", err, map[string]any{ - "entity_type": entity.Type, - "batch_index": i, - "success_count": successCount, - "error_count": errorCount, - }) - continue - } - successCount++ - - if i < len(entityBatch)-1 { - time.Sleep(100 * time.Millisecond) - } - } - - if errorCount > 0 { - return fmt.Errorf("batch completed with %d errors out of %d entities", errorCount, len(entityBatch)) - } - - return nil -} diff --git a/threadwinds-ingestion/internal/extractor/field_extractor.go b/threadwinds-ingestion/internal/extractor/field_extractor.go deleted file mode 100644 index b32f9ca36..000000000 --- a/threadwinds-ingestion/internal/extractor/field_extractor.go +++ /dev/null @@ -1,111 +0,0 @@ -package extractor - -import ( - "fmt" - "strings" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" -) - -type FieldExtractor struct{} - -func NewFieldExtractor() *FieldExtractor { - return &FieldExtractor{} -} - -func (e *FieldExtractor) ExtractFromAlert(alert *models.Alert) []*models.FlattenedField { - fields := make([]*models.FlattenedField, 0, 4) - - if alert.Source != nil { - fields = append(fields, e.extractFromHost(alert.Source, "alert.source")...) - } - - if alert.Destination != nil { - fields = append(fields, e.extractFromHost(alert.Destination, "alert.destination")...) - } - - catcher.Info("extracted fields from alert", map[string]any{ - "alert_id": alert.ID, - "field_count": len(fields), - }) - - return fields -} - -func (e *FieldExtractor) extractFromHost(host *models.Host, prefix string) []*models.FlattenedField { - fields := make([]*models.FlattenedField, 0, 5) - - if host.IP != "" && !isInvalidValue(host.IP) { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".ip", - Key: "ip", - Value: host.IP, - }) - } - - if host.Host != "" && !isInvalidValue(host.Host) { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".host", - Key: "hostname", - Value: host.Host, - }) - } - - if host.User != "" && !isInvalidValue(host.User) { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".user", - Key: "username", - Value: host.User, - }) - } - - if host.Port != 0 { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".port", - Key: "port", - Value: host.Port, - }) - } - - if host.ASN != 0 && !isInvalidValue(host.ASN) { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".asn", - Key: "asn", - Value: host.ASN, - }) - } - - return fields -} - -func isInvalidValue(value any) bool { - strValue := fmt.Sprintf("%v", value) - if strings.TrimSpace(strValue) == "" { - return true - } - - invalidValues := []string{ - "-", - "N/A", - "n/a", - "unknown", - "null", - "(null)", - "none", - "0", - "-1", - "0.0.0.0", - "255.255.255.255", - "127.0.0.1", - "localhost", - } - - for _, invalid := range invalidValues { - if strings.EqualFold(strValue, invalid) { - return true - } - } - - return false -} diff --git a/threadwinds-ingestion/internal/mapper/entity_mapper.go b/threadwinds-ingestion/internal/mapper/entity_mapper.go deleted file mode 100644 index 11601e7cd..000000000 --- a/threadwinds-ingestion/internal/mapper/entity_mapper.go +++ /dev/null @@ -1,126 +0,0 @@ -package mapper - -import ( - "fmt" - "strings" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/entities" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" -) - -type EntityMapper struct { - entityTypes map[string]bool -} - -func NewEntityMapper() *EntityMapper { - mapper := &EntityMapper{ - entityTypes: make(map[string]bool), - } - - for _, def := range entities.Definitions { - mapper.entityTypes[def.Type] = true - } - - catcher.Info("entity mapper initialized", map[string]any{ - "total_entity_types": len(mapper.entityTypes), - }) - - return mapper -} - -func (m *EntityMapper) MapFieldToEntityType(field *models.FlattenedField) (string, bool) { - leafKey := normalizeKey(field.Key) - - if m.entityTypes[leafKey] { - return leafKey, true - } - - return "", false -} - -func normalizeKey(key string) string { - key = strings.ToLower(key) - key = strings.ReplaceAll(key, "_", "-") - return key -} - -func (m *EntityMapper) BuildEntity(entityType string, value any, context EntityEnrichmentContext) (*entities.Entity, string, error) { - validatedValue, hash, err := entities.ValidateValue(value, entityType) - if err != nil { - return nil, "", fmt.Errorf("validation failed for type %s: %w", entityType, err) - } - - attrs := entities.Attributes{} - if !attrs.SetAttribute(entityType, validatedValue) { - return nil, "", fmt.Errorf("failed to set attribute for type %s", entityType) - } - - if context.Country != "" { - attrs.Country = &context.Country - } - if context.City != "" { - attrs.City = &context.City - } - if context.ASO != "" { - attrs.Aso = &context.ASO - } - if context.Latitude != nil { - attrs.Latitude = context.Latitude - } - if context.Longitude != nil { - attrs.Longitude = context.Longitude - } - if context.AccuracyRadius != nil { - attrs.AccuracyRadius = context.AccuracyRadius - } - - reputation := calculateReputation(context.Severity) - - tags := []string{ - "utmstack", - "incident-" + context.IncidentID, - } - if context.DataType != "" { - tags = append(tags, "datasource-"+context.DataType) - } - - entity := &entities.Entity{ - Type: entityType, - Attributes: attrs, - Reputation: reputation, - Tags: tags, - Associations: nil, - } - - entityID := fmt.Sprintf("%s-%s", entityType, hash) - - catcher.Info("entity built successfully", map[string]any{ - "entity_type": entityType, - "entity_id": entityID, - "reputation": reputation, - }) - - return entity, entityID, nil -} - -type EntityEnrichmentContext struct { - IncidentID string - Severity int - DataType string - Country string - City string - Latitude *float64 - Longitude *float64 - ASO string - AccuracyRadius *float64 -} - -func calculateReputation(severity int) int { - if severity >= 7 { - return -3 - } else if severity >= 4 { - return -1 - } - return 0 -} diff --git a/threadwinds-ingestion/internal/models/alert.go b/threadwinds-ingestion/internal/models/alert.go deleted file mode 100644 index 1ff41a2a9..000000000 --- a/threadwinds-ingestion/internal/models/alert.go +++ /dev/null @@ -1,30 +0,0 @@ -package models - -type Alert struct { - ID string `json:"id"` - Name string `json:"name"` - Timestamp string `json:"@timestamp"` - DataType string `json:"dataType"` - DataSource string `json:"dataSource"` - Severity int `json:"severity"` - Status int `json:"status"` - Source *Host `json:"source,omitempty"` - Destination *Host `json:"destination,omitempty"` - ASO string `json:"aso,omitempty"` - ASN int `json:"asn,omitempty"` -} - -type Host struct { - IP string `json:"ip,omitempty"` - Host string `json:"host,omitempty"` - User string `json:"user,omitempty"` - Port int `json:"port,omitempty"` - City string `json:"city,omitempty"` - Country string `json:"country,omitempty"` - Coordinates []float64 `json:"coordinates,omitempty"` - ASO string `json:"aso,omitempty"` - ASN int `json:"asn,omitempty"` - AccuracyRadius int `json:"accuracyRadius,omitempty"` - IsAnonymousProxy bool `json:"isAnonymousProxy,omitempty"` - IsSatelliteProvider bool `json:"isSatelliteProvider,omitempty"` -} diff --git a/threadwinds-ingestion/internal/models/event.go b/threadwinds-ingestion/internal/models/event.go deleted file mode 100644 index 7091bc879..000000000 --- a/threadwinds-ingestion/internal/models/event.go +++ /dev/null @@ -1,7 +0,0 @@ -package models - -type FlattenedField struct { - Path string - Key string - Value any -} diff --git a/threadwinds-ingestion/internal/models/incident.go b/threadwinds-ingestion/internal/models/incident.go deleted file mode 100644 index 3e513fd45..000000000 --- a/threadwinds-ingestion/internal/models/incident.go +++ /dev/null @@ -1,21 +0,0 @@ -package models - -import "time" - -type Incident struct { - ID int64 `json:"id"` - Name string `json:"incidentName"` - Description string `json:"incidentDescription"` - Status string `json:"incidentStatus"` - Severity int `json:"incidentSeverity"` - CreatedDate time.Time `json:"incidentCreatedDate"` -} - -type IncidentAlert struct { - ID int64 `json:"id"` - IncidentID int64 `json:"incidentId"` - AlertID string `json:"alertId"` - AlertName string `json:"alertName"` - AlertStatus int `json:"alertStatus"` - AlertSeverity int `json:"alertSeverity"` -} diff --git a/threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go b/threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go deleted file mode 100644 index b4d051d3f..000000000 --- a/threadwinds-ingestion/internal/scheduler/ingestion_scheduler.go +++ /dev/null @@ -1,349 +0,0 @@ -package scheduler - -import ( - "context" - "fmt" - "strings" - "sync" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/association" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/client" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/extractor" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/mapper" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/models" -) - -const ( - pollInterval = 5 * time.Minute - incidentRetentionPeriod = 48 * time.Hour - alertRetentionPeriod = 72 * time.Hour - cleanupInterval = 6 * time.Hour -) - -type IncidentState struct { - LastProcessedAt time.Time - ProcessedAlerts map[string]time.Time - TotalEntities int -} - -type IngestionScheduler struct { - cfg *config.TWConfig - backendClient *client.BackendClient - opensearchClient *client.OpenSearchClient - threadwindsClient *client.ThreadWindsClient - fieldExtractor *extractor.FieldExtractor - entityMapper *mapper.EntityMapper - associationBuilder *association.AssociationBuilder - processedIncidents map[int64]*IncidentState - mu sync.RWMutex -} - -func NewIngestionScheduler( - cfg *config.TWConfig, - backendClient *client.BackendClient, - opensearchClient *client.OpenSearchClient, - threadwindsClient *client.ThreadWindsClient, -) *IngestionScheduler { - return &IngestionScheduler{ - cfg: cfg, - backendClient: backendClient, - opensearchClient: opensearchClient, - threadwindsClient: threadwindsClient, - fieldExtractor: extractor.NewFieldExtractor(), - entityMapper: mapper.NewEntityMapper(), - associationBuilder: association.NewAssociationBuilder(), - processedIncidents: make(map[int64]*IncidentState), - } -} - -func (s *IngestionScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(pollInterval) - cleanupTicker := time.NewTicker(cleanupInterval) - defer ticker.Stop() - defer cleanupTicker.Stop() - - catcher.Info("ingestion scheduler started", map[string]any{ - "poll_interval": pollInterval, - "cleanup_interval": cleanupInterval, - }) - - s.runIngestionCycle(ctx) - - for { - select { - case <-ctx.Done(): - catcher.Info("scheduler stopped", nil) - return - case <-ticker.C: - s.runIngestionCycle(ctx) - case <-cleanupTicker.C: - s.cleanOldState() - } - } -} - -func (s *IngestionScheduler) runIngestionCycle(ctx context.Context) { - catcher.Info("starting ingestion cycle", nil) - startTime := time.Now() - - if err := s.updateThreadWindsCredentials(ctx); err != nil { - catcher.Error("failed to update ThreadWinds credentials from database", err, nil) - } - - cycleTimeout := time.Duration(float64(pollInterval) * 0.9) - cycleCtx, cancel := context.WithTimeout(ctx, cycleTimeout) - defer cancel() - - incidents, err := s.backendClient.GetRecentIncidents(cycleCtx) - if err != nil { - catcher.Error("failed to fetch incidents", err, nil) - return - } - - if len(incidents) == 0 { - catcher.Info("no recent incidents to process", nil) - return - } - - totalEntities := 0 - for i, incident := range incidents { - select { - case <-cycleCtx.Done(): - catcher.Info("cycle timeout or cancellation, stopping", map[string]any{ - "processed_incidents": i, - "total_incidents": len(incidents), - "reason": cycleCtx.Err().Error(), - }) - return - default: - } - - entitiesCount, err := s.processIncident(cycleCtx, incident) - if err != nil { - catcher.Error("failed to process incident", err, map[string]any{ - "incident_id": incident.ID, - "incident_name": incident.Name, - }) - continue - } - totalEntities += entitiesCount - } - - duration := time.Since(startTime) - catcher.Info("ingestion cycle completed", map[string]any{ - "duration_seconds": duration.Seconds(), - "incidents_processed": len(incidents), - "total_entities": totalEntities, - }) -} - -func (s *IngestionScheduler) processIncident(ctx context.Context, incident *models.Incident) (int, error) { - s.mu.Lock() - state, exists := s.processedIncidents[incident.ID] - if !exists { - state = &IncidentState{ - ProcessedAlerts: make(map[string]time.Time), - } - s.processedIncidents[incident.ID] = state - } - s.mu.Unlock() - - incidentAlerts, err := s.backendClient.GetIncidentAlerts(ctx, incident.ID) - if err != nil { - return 0, fmt.Errorf("failed to get incident alerts: %w", err) - } - - if len(incidentAlerts) == 0 { - return 0, nil - } - - newAlerts := s.filterNewAlerts(incidentAlerts, state) - if len(newAlerts) == 0 { - return 0, nil - } - - catcher.Info("processing incident with new alerts", map[string]any{ - "incident_id": incident.ID, - "new_alerts": len(newAlerts), - "total_alerts": len(incidentAlerts), - }) - - s.associationBuilder.ClearRegistry() - - for _, incidentAlert := range newAlerts { - err := s.processAlertWithAssociations(ctx, incidentAlert, incident) - if err != nil { - catcher.Error("failed to process alert", err, map[string]any{ - "alert_id": incidentAlert.AlertID, - "incident_id": incident.ID, - }) - continue - } - - s.mu.Lock() - state.ProcessedAlerts[incidentAlert.AlertID] = time.Now() - s.mu.Unlock() - } - - allEntities := s.associationBuilder.BuildAssociations() - - if len(allEntities) > 0 { - if err := s.threadwindsClient.IngestBatch(ctx, allEntities); err != nil { - return 0, fmt.Errorf("failed to ingest batch: %w", err) - } - } - - s.mu.Lock() - state.LastProcessedAt = time.Now() - state.TotalEntities += len(allEntities) - s.mu.Unlock() - - return len(allEntities), nil -} - -func (s *IngestionScheduler) processAlertWithAssociations(ctx context.Context, incidentAlert *models.IncidentAlert, incident *models.Incident) error { - alert, err := s.opensearchClient.GetAlertByID(ctx, incidentAlert.AlertID) - if err != nil { - return fmt.Errorf("failed to get alert: %w", err) - } - - alertFields := s.fieldExtractor.ExtractFromAlert(alert) - s.mapAndRegisterFieldsToEntities(alertFields, incident, alert) - - return nil -} - -func (s *IngestionScheduler) mapAndRegisterFieldsToEntities( - fields []*models.FlattenedField, - incident *models.Incident, - alert *models.Alert, -) { - for _, field := range fields { - entityType, matched := s.entityMapper.MapFieldToEntityType(field) - if !matched { - continue - } - - sourceField := "" - var hostContext *models.Host - if strings.Contains(field.Path, "source") { - sourceField = "source" - hostContext = alert.Source - } else if strings.Contains(field.Path, "destination") { - sourceField = "destination" - hostContext = alert.Destination - } - - enrichmentCtx := s.buildEnrichmentContext(incident, alert, hostContext) - entity, entityID, err := s.entityMapper.BuildEntity(entityType, field.Value, enrichmentCtx) - if err != nil { - catcher.Error("failed to build entity", err, map[string]any{ - "entity_type": entityType, - "field_path": field.Path, - }) - continue - } - - assocContext := association.AssociationContext{ - AlertID: alert.ID, - IncidentID: fmt.Sprintf("%d", incident.ID), - SourceField: sourceField, - } - s.associationBuilder.RegisterEntity(entity, entityID, field.Path, assocContext) - } -} - -func (s *IngestionScheduler) filterNewAlerts(incidentAlerts []*models.IncidentAlert, state *IncidentState) []*models.IncidentAlert { - s.mu.RLock() - defer s.mu.RUnlock() - - newAlerts := make([]*models.IncidentAlert, 0, len(incidentAlerts)) - for _, alert := range incidentAlerts { - if _, processed := state.ProcessedAlerts[alert.AlertID]; !processed { - newAlerts = append(newAlerts, alert) - } - } - return newAlerts -} - -func (s *IngestionScheduler) cleanOldState() { - s.mu.Lock() - defer s.mu.Unlock() - - incidentCutoff := time.Now().Add(-incidentRetentionPeriod) - alertCutoff := time.Now().Add(-alertRetentionPeriod) - - cleanedIncidents := 0 - cleanedAlerts := 0 - - for incidentID, state := range s.processedIncidents { - if state.LastProcessedAt.Before(incidentCutoff) { - delete(s.processedIncidents, incidentID) - cleanedIncidents++ - continue - } - - for alertID, processedAt := range state.ProcessedAlerts { - if processedAt.Before(alertCutoff) { - delete(state.ProcessedAlerts, alertID) - cleanedAlerts++ - } - } - } - - if cleanedIncidents > 0 || cleanedAlerts > 0 { - catcher.Info("state cleanup completed", map[string]any{ - "active_incidents": len(s.processedIncidents), - "cleaned_incidents": cleanedIncidents, - "cleaned_alerts": cleanedAlerts, - }) - } -} - -func (s *IngestionScheduler) buildEnrichmentContext(incident *models.Incident, alert *models.Alert, host *models.Host) mapper.EntityEnrichmentContext { - ctx := mapper.EntityEnrichmentContext{ - IncidentID: fmt.Sprintf("%d", incident.ID), - Severity: incident.Severity, - DataType: alert.DataType, - } - - if host != nil { - ctx.Country = host.Country - ctx.City = host.City - ctx.ASO = host.ASO - - if len(host.Coordinates) == 2 { - lat := host.Coordinates[0] - lon := host.Coordinates[1] - if lat != 0.0 || lon != 0.0 { - ctx.Latitude = &lat - ctx.Longitude = &lon - } - } - - if host.AccuracyRadius > 0 { - radius := float64(host.AccuracyRadius) - ctx.AccuracyRadius = &radius - } - } - - return ctx -} - -func (s *IngestionScheduler) updateThreadWindsCredentials(ctx context.Context) error { - config, err := s.backendClient.GetThreadWindsConfig(ctx) - if err != nil { - return fmt.Errorf("failed to get ThreadWinds credentials from backend: %w", err) - } - - if config.APIKey == "" || config.APISecret == "" { - return fmt.Errorf("ThreadWinds credentials are empty in backend configuration") - } - - s.threadwindsClient.UpdateCredentials(config.APIKey, config.APISecret) - - return nil -} diff --git a/threadwinds-ingestion/main.go b/threadwinds-ingestion/main.go deleted file mode 100644 index cade60c4b..000000000 --- a/threadwinds-ingestion/main.go +++ /dev/null @@ -1,128 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "syscall" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/threadwinds-ingestion/config" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/client" - "github.com/utmstack/UTMStack/threadwinds-ingestion/internal/scheduler" - "github.com/utmstack/UTMStack/threadwinds-ingestion/utils" -) - -func main() { - catcher.Info("Starting ThreadWinds Ingestion Service", nil) - - cfg, err := config.GetTWConfig() - if err != nil { - catcher.Error("failed to load configuration", err, nil) - os.Exit(1) - } - - postgresClient, err := client.NewPostgresClient(cfg) - if err != nil { - catcher.Error("failed to initialize postgres client", err, nil) - os.Exit(1) - } - - ctx := context.Background() - - adminEmail, err := postgresClient.WaitForValidAdminEmail(ctx, 60*time.Minute) - if err != nil { - catcher.Error("cannot start ThreadWinds Ingestion without valid admin email", err, nil) - os.Exit(1) - } - - catcher.Info("Valid admin email obtained", map[string]any{ - "admin_email": adminEmail, - }) - - cmClient := client.NewCustomersManagerClient(cfg) - backendClient := client.NewBackendClient(cfg) - opensearchClient, err := client.NewOpenSearchClient(cfg) - if err != nil { - catcher.Error("failed to initialize opensearch client", err, nil) - os.Exit(1) - } - - threadwindsClient := client.NewThreadWindsClient(cfg) - - twConfig, err := backendClient.GetThreadWindsConfig(ctx) - if err != nil { - catcher.Error("failed to check ThreadWinds configuration", err, nil) - os.Exit(1) - } - - if twConfig.APIKey == "" || twConfig.APISecret == "" { - catcher.Info("ThreadWinds not configured, will attempt registration with retry...", nil) - - var regResp *client.RegistrationResponse - - registerFunc := func() error { - currentEmail, emailErr := postgresClient.GetAdminEmail(ctx) - if emailErr != nil { - return catcher.Error("failed to get current admin email", emailErr, nil) - } - - catcher.Info("attempting ThreadWinds registration", map[string]any{ - "email": currentEmail.Email, - }) - - resp, err := cmClient.RegisterUserReporter(currentEmail.Email) - if err != nil { - return err - } - regResp = resp - return nil - } - - utils.InfiniteRetry(registerFunc, "ThreadWinds registration") - - catcher.Info("ThreadWinds registration successful", nil) - - err = backendClient.SaveThreadWindsCredentials(ctx, - regResp.APIKey, - regResp.APISecret, - twConfig.KeyID, - twConfig.SecretID) - if err != nil { - catcher.Error("failed to save ThreadWinds credentials", err, nil) - os.Exit(1) - } - - threadwindsClient.UpdateCredentials(regResp.APIKey, regResp.APISecret) - } else { - catcher.Info("ThreadWinds already configured", nil) - threadwindsClient.UpdateCredentials(twConfig.APIKey, twConfig.APISecret) - } - - ingestionScheduler := scheduler.NewIngestionScheduler( - cfg, - backendClient, - opensearchClient, - threadwindsClient, - ) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go ingestionScheduler.Start(ctx) - - sig := <-sigChan - catcher.Info("received shutdown signal, initiating graceful shutdown", map[string]any{ - "signal": sig.String(), - }) - - cancel() - - time.Sleep(5 * time.Second) - - catcher.Info("ThreadWinds Ingestion Service stopped", nil) -} diff --git a/threadwinds-ingestion/utils/aes.go b/threadwinds-ingestion/utils/aes.go deleted file mode 100644 index 3b355aa7b..000000000 --- a/threadwinds-ingestion/utils/aes.go +++ /dev/null @@ -1,10 +0,0 @@ -package utils - -import ( - "github.com/AtlasInsideCorp/AtlasInsideAES" -) - -func DecryptValue(encryptedValue string) (string, error) { - passphrase := Getenv("ENCRYPTION_KEY") - return AtlasInsideAES.AESDecrypt(encryptedValue, []byte(passphrase)) -} diff --git a/threadwinds-ingestion/utils/check.go b/threadwinds-ingestion/utils/check.go deleted file mode 100644 index 1f74df131..000000000 --- a/threadwinds-ingestion/utils/check.go +++ /dev/null @@ -1,48 +0,0 @@ -package utils - -import ( - "fmt" - "time" - - "github.com/threatwinds/go-sdk/catcher" -) - -const ( - retryInitialBackoff = 5 * time.Second - retryMaxBackoff = 2 * time.Minute - retryBackoffMultiplier = 2.0 - retryLogInterval = 10 -) - -func InfiniteRetry(f func() error, operationName string) { - attempt := 0 - currentBackoff := retryInitialBackoff - - catcher.Info(fmt.Sprintf("Starting %s with infinite retry and exponential backoff", operationName), map[string]any{ - "initial_backoff": retryInitialBackoff.String(), - "max_backoff": retryMaxBackoff.String(), - }) - - for { - attempt++ - err := f() - - if err == nil { - catcher.Info(fmt.Sprintf("%s completed successfully", operationName), map[string]any{ - "attempts": attempt, - }) - return - } - - if attempt == 1 || attempt%retryLogInterval == 0 { - _ = catcher.Error(fmt.Sprintf("%s failed, will retry indefinitely...", operationName), err, map[string]any{ - "attempt": attempt, - "next_retry_in": currentBackoff.String(), - }) - } - - time.Sleep(currentBackoff) - - currentBackoff = min(time.Duration(float64(currentBackoff)*retryBackoffMultiplier), retryMaxBackoff) - } -} diff --git a/threadwinds-ingestion/utils/env.go b/threadwinds-ingestion/utils/env.go deleted file mode 100644 index 779b3dc6a..000000000 --- a/threadwinds-ingestion/utils/env.go +++ /dev/null @@ -1,20 +0,0 @@ -package utils - -import ( - "os" - - "github.com/threatwinds/go-sdk/catcher" -) - -func Getenv(key string) string { - value, defined := os.LookupEnv(key) - if !defined { - catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) - os.Exit(1) - } - if (value == "") || (value == " ") { - catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) - os.Exit(1) - } - return value -} diff --git a/threadwinds-ingestion/utils/req.go b/threadwinds-ingestion/utils/req.go deleted file mode 100644 index 3bc51ca70..000000000 --- a/threadwinds-ingestion/utils/req.go +++ /dev/null @@ -1,46 +0,0 @@ -package utils - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" -) - -func DoReq[response any](url string, data []byte, method string, headers map[string]string) (response, int, error) { - var result response - - req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) - if err != nil { - return result, http.StatusInternalServerError, err - } - - for k, v := range headers { - req.Header.Add(k, v) - } - - client := &http.Client{} - - resp, err := client.Do(req) - if err != nil { - return result, http.StatusInternalServerError, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return result, http.StatusInternalServerError, err - } - - if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { - return result, resp.StatusCode, fmt.Errorf("while sending request to %s received status code: %d and response body: %s", url, resp.StatusCode, body) - } - - err = json.Unmarshal(body, &result) - if err != nil { - return result, http.StatusInternalServerError, err - } - - return result, resp.StatusCode, nil -} From fdf03ea0be46fa8fc0f9365c764db3beaf18b1a5 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 13 Jan 2026 10:35:02 -0500 Subject: [PATCH 17/21] ci: remove threadwinds-ingestion from deployment pipeline --- .github/workflows/v10-deployment-pipeline.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/v10-deployment-pipeline.yml b/.github/workflows/v10-deployment-pipeline.yml index 943c5d0fe..3e6c9c17c 100644 --- a/.github/workflows/v10-deployment-pipeline.yml +++ b/.github/workflows/v10-deployment-pipeline.yml @@ -249,15 +249,6 @@ jobs: image_name: sophos tag: ${{ needs.setup_deployment.outputs.tag }} - build_threadwinds_ingestion: - name: Build Threadwinds-Ingestion Microservice - needs: [validations,setup_deployment] - if: ${{ needs.setup_deployment.outputs.tag != '' }} - uses: ./.github/workflows/reusable-golang.yml - with: - image_name: threadwinds-ingestion - tag: ${{ needs.setup_deployment.outputs.tag }} - build_user_auditor: name: Build User-Auditor Microservice needs: [validations,setup_deployment] @@ -290,7 +281,6 @@ jobs: build_aws, build_backend, build_correlation, build_frontend, build_bitdefender, build_mutate, build_office365, build_log_auth_proxy, build_soc_ai, build_sophos, - build_threadwinds_ingestion, build_user_auditor, build_web_pdf ] if: ${{ needs.setup_deployment.outputs.tag != '' }} From ca99ad35f11733d0a5acdc438e4f8a34b935c309 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Tue, 13 Jan 2026 10:35:18 -0500 Subject: [PATCH 18/21] feat(installer): remove threadwinds-ingestion from stack configuration --- installer/types/compose.go | 32 -------------------------------- installer/types/stack.go | 1 - 2 files changed, 33 deletions(-) diff --git a/installer/types/compose.go b/installer/types/compose.go index c18efb1cb..49da51178 100644 --- a/installer/types/compose.go +++ b/installer/types/compose.go @@ -596,38 +596,6 @@ func (c *Compose) Populate(conf *Config, stack *StackConfig) *Compose { }, } - threadwindsIngestionMem := stack.ServiceResources["threadwinds-ingestion"].AssignedMemory - c.Services["threadwinds-ingestion"] = Service{ - Image: utils.Str("ghcr.io/utmstack/utmstack/threadwinds-ingestion:" + conf.Branch), - DependsOn: []string{ - "postgres", - "node1", - "backend", - }, - Environment: []string{ - "INTERNAL_KEY=" + conf.InternalKey, - "ENCRYPTION_KEY=" + conf.InternalKey, - "BACKEND_URL=http://backend:8080", - "ENV=" + conf.Branch, - "OPENSEARCH_HOST=node1", - "OPENSEARCH_PORT=9200", - "DB_HOST=postgres", - "DB_PORT=5432", - "DB_USER=postgres", - "DB_PASS=" + conf.Password, - "DB_NAME=utmstack", - }, - Logging: &dLogging, - Deploy: &Deploy{ - Placement: &pManager, - Resources: &Resources{ - Limits: &Res{ - Memory: utils.Str(fmt.Sprintf("%vM", threadwindsIngestionMem)), - }, - }, - }, - } - webPDFMem := stack.ServiceResources["web-pdf"].AssignedMemory c.Services["web-pdf"] = Service{ Image: utils.Str("ghcr.io/utmstack/utmstack/web-pdf:" + conf.Branch), diff --git a/installer/types/stack.go b/installer/types/stack.go index a4a32a837..490f3cfc2 100644 --- a/installer/types/stack.go +++ b/installer/types/stack.go @@ -39,7 +39,6 @@ var Services = []utils.ServiceConfig{ {Name: "socai", Priority: 3, MinMemory: 30, MaxMemory: 512}, {Name: "bitdefender", Priority: 3, MinMemory: 30, MaxMemory: 100}, {Name: "office365", Priority: 3, MinMemory: 30, MaxMemory: 100}, - {Name: "threadwinds-ingestion", Priority: 3, MinMemory: 50, MaxMemory: 256}, } func (s *StackConfig) Populate(c *Config) error { From 15caafbf8bdc15b95e378292868777815813ddf3 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Tue, 13 Jan 2026 09:41:17 -0600 Subject: [PATCH 19/21] Revert "feat(threatwinds): add configuration parameters for ThreatWinds integration" This reverts commit ab16ad05843b691da1c81a2b23ba9cf9e3ee751d. --- ...1_insert_threatwinds_credentials_section.xml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml index c0efaf6c9..47c95121b 100644 --- a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml +++ b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml @@ -15,22 +15,6 @@ - - - - - - - - - - - - - - - - @@ -45,7 +29,6 @@ - From be9defff42a64d3c9b0bbfcebfb006369a8a587d Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Tue, 13 Jan 2026 09:41:43 -0600 Subject: [PATCH 20/21] Revert "feat(threatwinds): add ThreatWinds credentials section and parameters to configuration" This reverts commit 3173db0aa7b26aa2fede00ab5ffa59088bf415a4. --- .../shared_types/enums/SectionType.java | 3 +- ...insert_threatwinds_credentials_section.xml | 45 ------------------- .../resources/config/liquibase/master.xml | 2 - 3 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml diff --git a/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java b/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java index a2a2ecb4f..f3aec586c 100644 --- a/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java +++ b/backend/src/main/java/com/park/utmstack/domain/shared_types/enums/SectionType.java @@ -5,7 +5,6 @@ public enum SectionType { EMAIL, TFA, ALERTS, - DATE_SETTINGS, - THREATWINDS_CREDENTIALS + DATE_SETTINGS } diff --git a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml b/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml deleted file mode 100644 index 47c95121b..000000000 --- a/backend/src/main/resources/config/liquibase/changelog/20251219001_insert_threatwinds_credentials_section.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 8f778706c..f6ce831a6 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -113,7 +113,5 @@ - - From a6117549c336d1a52bee9427085de160601fe744 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Fri, 15 May 2026 18:08:47 +0300 Subject: [PATCH 21/21] feat(filters/sophos_xg): add steps to accept new fields and version of sophos_xg --- filters/sophos/sophos_xg_firewall.conf | 149 ++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 4 deletions(-) diff --git a/filters/sophos/sophos_xg_firewall.conf b/filters/sophos/sophos_xg_firewall.conf index c51834263..da1ac71a4 100644 --- a/filters/sophos/sophos_xg_firewall.conf +++ b/filters/sophos/sophos_xg_firewall.conf @@ -1,6 +1,6 @@ filter { -# Sophos filter version 2.0.1 +# Sophos filter version 2.1.0 # Based on https://docs.sophos.com/nsg/sophos-firewall/17.5/PDF/SFOS_Logfile_Guide_17.5.pdf # and https://docs.sophos.com/nsg/sophos-firewall/18.5/PDF/SF%20syslog%20guide%2018.5.pdf # and https://docs.sophos.com/nsg/sophos-firewall/17.5/Help/en-us/webhelp/onlinehelp/nsg/sfos/concepts/LogMessages.html @@ -42,6 +42,7 @@ filter { gsub => ["device_name", '"', ""] gsub => ["log_type", '"', ""] gsub => ["log_component", '"', ""] + gsub => ["log_id", '"', ""] } if [log_type] and ([log_type] == "Firewall" or [log_type] == "Content Filtering" or [log_type] == "Event" or [log_type] == "WAF" or [log_type] == "System Health" or [log_type] == "IDP" @@ -133,7 +134,7 @@ filter { } } - if [logx][sophos][device] and [logx][sophos][device] == "SFW" { + if [logx][sophos][device] { if [msg] { #Fields from Firewall log_type grok { @@ -227,11 +228,123 @@ filter { ] } } + # New XGS fields - Firewall rules + grok { + match => { + "msg" => [ + "%{GREEDYDATA} fw_rule_name=%{QUOTEDSTRING:fw_rule_name} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} fw_rule_section=%{QUOTEDSTRING:fw_rule_section} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} nat_rule_name=%{QUOTEDSTRING:nat_rule_name} %{GREEDYDATA}" + ] + } + } + # New XGS fields - SD-WAN profile request + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_profile_id_request=%{NUMBER:sdwan_profile_id_request} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_profile_name_request=%{QUOTEDSTRING:sdwan_profile_name_request} %{GREEDYDATA}" + ] + } + } + # New XGS fields - SD-WAN profile reply + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_profile_id_reply=%{NUMBER:sdwan_profile_id_reply} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_profile_name_reply=%{QUOTEDSTRING:sdwan_profile_name_reply} %{GREEDYDATA}" + ] + } + } + # New XGS fields - Gateway request + grok { + match => { + "msg" => [ + "%{GREEDYDATA} gw_id_request=%{NUMBER:gw_id_request} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} gw_name_request=%{QUOTEDSTRING:gw_name_request} %{GREEDYDATA}" + ] + } + } + # New XGS fields - Gateway reply + grok { + match => { + "msg" => [ + "%{GREEDYDATA} gw_id_reply=%{NUMBER:gw_id_reply} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} gw_name_reply=%{QUOTEDSTRING:gw_name_reply} %{GREEDYDATA}" + ] + } + } + # New XGS fields - SD-WAN route request + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_route_id_request=%{NUMBER:sdwan_route_id_request} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_route_name_request=%{QUOTEDSTRING:sdwan_route_name_request} %{GREEDYDATA}" + ] + } + } + # New XGS fields - SD-WAN route reply + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_route_id_reply=%{NUMBER:sdwan_route_id_reply} %{GREEDYDATA}" + ] + } + } + grok { + match => { + "msg" => [ + "%{GREEDYDATA} sdwan_route_name_reply=%{QUOTEDSTRING:sdwan_route_name_reply} %{GREEDYDATA}" + ] + } + } #1.3.7 grok { match => { "msg" => [ - "%{GREEDYDATA} dst_mac=%{QUOTEDSTRING:dst_mac} %{GREEDYDATA}" + "%{GREEDYDATA} dst_mac=%{DATA:dst_mac} %{GREEDYDATA}" ] } } @@ -305,7 +418,7 @@ filter { grok { match => { "msg" => [ - "%{GREEDYDATA} src_mac=%{QUOTEDSTRING:src_mac} %{GREEDYDATA}" + "%{GREEDYDATA} src_mac=%{DATA:src_mac} %{GREEDYDATA}" ] } } @@ -534,6 +647,17 @@ filter { #1.3.7 gsub => ["dst_mac", '"', ""] + #New XGS fields + gsub => ["fw_rule_name", '"', ""] + gsub => ["fw_rule_section", '"', ""] + gsub => ["nat_rule_name", '"', ""] + gsub => ["sdwan_profile_name_request", '"', ""] + gsub => ["sdwan_profile_name_reply", '"', ""] + gsub => ["gw_name_request", '"', ""] + gsub => ["gw_name_reply", '"', ""] + gsub => ["sdwan_route_name_request", '"', ""] + gsub => ["sdwan_route_name_reply", '"', ""] + #New fields from Content Filtering log_type gsub => ["category", '"', ""] gsub => ["category_type", '"', ""] @@ -794,6 +918,23 @@ filter { #1.3.7 rename => { "[dst_mac]" => "[logx][sophos][dst_mac]" } + #New XGS fields + rename => { "[fw_rule_name]" => "[logx][sophos][fw_rule_name]" } + rename => { "[fw_rule_section]" => "[logx][sophos][fw_rule_section]" } + rename => { "[nat_rule_name]" => "[logx][sophos][nat_rule_name]" } + rename => { "[sdwan_profile_id_request]" => "[logx][sophos][sdwan_profile_id_request]" } + rename => { "[sdwan_profile_name_request]" => "[logx][sophos][sdwan_profile_name_request]" } + rename => { "[sdwan_profile_id_reply]" => "[logx][sophos][sdwan_profile_id_reply]" } + rename => { "[sdwan_profile_name_reply]" => "[logx][sophos][sdwan_profile_name_reply]" } + rename => { "[gw_id_request]" => "[logx][sophos][gw_id_request]" } + rename => { "[gw_name_request]" => "[logx][sophos][gw_name_request]" } + rename => { "[gw_id_reply]" => "[logx][sophos][gw_id_reply]" } + rename => { "[gw_name_reply]" => "[logx][sophos][gw_name_reply]" } + rename => { "[sdwan_route_id_request]" => "[logx][sophos][sdwan_route_id_request]" } + rename => { "[sdwan_route_name_request]" => "[logx][sophos][sdwan_route_name_request]" } + rename => { "[sdwan_route_id_reply]" => "[logx][sophos][sdwan_route_id_reply]" } + rename => { "[sdwan_route_name_reply]" => "[logx][sophos][sdwan_route_name_reply]" } + #New fields from Content Filtering log_type rename => { "[category]" => "[logx][sophos][category]" } rename => { "[category_type]" => "[logx][sophos][category_type]" }