From 3f12b74cc0f10544ae5fd5bc2a1e670e49c434fd Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Mon, 8 Jun 2026 23:26:53 +0200 Subject: [PATCH 1/3] IEBH-483: Remove unused LINEAGE_SERVICE env vars --- clusters/dev/apps/dataops/values.yaml | 1 - clusters/prod/apps/dataops/values.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/clusters/dev/apps/dataops/values.yaml b/clusters/dev/apps/dataops/values.yaml index c6a13cf..0a09474 100644 --- a/clusters/dev/apps/dataops/values.yaml +++ b/clusters/dev/apps/dataops/values.yaml @@ -34,7 +34,6 @@ dataops-service: REDIS_PORT: "6379" REDIS_DB: "0" WORKERS: "2" - LINEAGE_SERVICE: "http://lineage.utility:5064" QUEUE_SERVICE: "http://queue-producer.greenroom:6060" METADATA_SERVICE: "http://metadata.utility:5066" AUTH_SERVICE: "http://auth.utility:5061" diff --git a/clusters/prod/apps/dataops/values.yaml b/clusters/prod/apps/dataops/values.yaml index 219d949..0afbbc0 100644 --- a/clusters/prod/apps/dataops/values.yaml +++ b/clusters/prod/apps/dataops/values.yaml @@ -34,7 +34,6 @@ dataops-service: REDIS_PORT: "6379" REDIS_DB: "0" WORKERS: "2" - LINEAGE_SERVICE: "http://lineage.utility:5064" QUEUE_SERVICE: "http://queue-producer.greenroom:6060" METADATA_SERVICE: "http://metadata.utility:5066" AUTH_SERVICE: "http://auth.utility:5061" From 0ddcaad414cb43fed5366967e81b92517f55340a Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Tue, 9 Jun 2026 19:19:04 +0200 Subject: [PATCH 2/3] IEBH-483: Deploy lineage-event-handler in dev --- README.md | 2 + .../dev/apps/lineage-event-handler/Chart.yaml | 7 +++ .../lineage-event-handler/application.yaml | 28 +++++++++++ .../apps/lineage-event-handler/values.yaml | 48 +++++++++++++++++++ 4 files changed, 85 insertions(+) create mode 100644 clusters/dev/apps/lineage-event-handler/Chart.yaml create mode 100644 clusters/dev/apps/lineage-event-handler/application.yaml create mode 100644 clusters/dev/apps/lineage-event-handler/values.yaml diff --git a/README.md b/README.md index efdc9ec..afb0079 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,10 @@ This repo uses ArgoCD's app-of-apps pattern: a root Application (`root-app.yaml` | 8 | download-greenroom | Download service (greenroom ns) | | 8 | download-core | Download service (core ns) | | 8 | search | Search service (ES-backed) | +| 8 | atlas | Apache Atlas (lineage) | | 9 | kong | API gateway | | 9 | metadata-event-handler | Kafka→ES event indexer | +| 9 | lineage-event-handler | Kafka→Atlas event indexer | | 9 | kg-integration | EBRAINS Knowledge Graph integration | | 9 | audit-trail | | | 10 | bff | Backend-for-frontend (web) | diff --git a/clusters/dev/apps/lineage-event-handler/Chart.yaml b/clusters/dev/apps/lineage-event-handler/Chart.yaml new file mode 100644 index 0000000..469e22a --- /dev/null +++ b/clusters/dev/apps/lineage-event-handler/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: lineage-event-handler +version: 0.1.0 +dependencies: + - name: metadata-event-handler + version: "0.1.1" + repository: https://pilotdataplatform.github.io/helm-charts/ diff --git a/clusters/dev/apps/lineage-event-handler/application.yaml b/clusters/dev/apps/lineage-event-handler/application.yaml new file mode 100644 index 0000000..dbe42e8 --- /dev/null +++ b/clusters/dev/apps/lineage-event-handler/application.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: lineage-event-handler + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "9" +spec: + project: default + source: + repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git + targetRevision: main + path: clusters/dev/apps/lineage-event-handler + helm: + valueFiles: + - ../../registry.yaml + - ../../versions.yaml + - values.yaml + destination: + server: https://kubernetes.default.svc + namespace: utility + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/clusters/dev/apps/lineage-event-handler/values.yaml b/clusters/dev/apps/lineage-event-handler/values.yaml new file mode 100644 index 0000000..530761d --- /dev/null +++ b/clusters/dev/apps/lineage-event-handler/values.yaml @@ -0,0 +1,48 @@ +metadata-event-handler: + image: + repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/lineage_event_handler + pullPolicy: IfNotPresent + + fullnameOverride: lineage-event-handler + replicaCount: 1 + + container: + port: 5067 + + service: + type: ClusterIP + port: 5067 + targetPort: 5067 + + imagePullSecrets: + - name: docker-registry-secret + + extraEnv: + KAFKA_TOPICS: '["metadata.items.activity"]' + KAFKA_SERVICE: "kafka.utility:9092" + KAFKA_GROUP_ID: "lineage_consumer_group" + ATLAS_API: "http://atlas.utility:21000" + ATLAS_ENTITY_TYPE: "metadata_items" + ATLAS_ADMIN: "admin" + SEEK_TO_BEGINNING: true + + extraEnvYaml: + - name: ATLAS_PASSWD + valueFrom: + secretKeyRef: + name: atlas-credentials + key: atlas-password + + resources: + limits: + cpu: "1" + memory: 100Mi + requests: + cpu: 10m + memory: 10Mi + + updateStrategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 33% + type: RollingUpdate From 21a2507f649786e12e2c18905ae0e0d8af50adf4 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Tue, 9 Jun 2026 20:18:46 +0200 Subject: [PATCH 3/3] IEBH-483: Apply Copilot suggestions --- clusters/dev/apps/lineage-event-handler/Chart.yaml | 1 + clusters/dev/apps/lineage-event-handler/values.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clusters/dev/apps/lineage-event-handler/Chart.yaml b/clusters/dev/apps/lineage-event-handler/Chart.yaml index 469e22a..cd617e0 100644 --- a/clusters/dev/apps/lineage-event-handler/Chart.yaml +++ b/clusters/dev/apps/lineage-event-handler/Chart.yaml @@ -3,5 +3,6 @@ name: lineage-event-handler version: 0.1.0 dependencies: - name: metadata-event-handler + alias: lineage-event-handler version: "0.1.1" repository: https://pilotdataplatform.github.io/helm-charts/ diff --git a/clusters/dev/apps/lineage-event-handler/values.yaml b/clusters/dev/apps/lineage-event-handler/values.yaml index 530761d..6449782 100644 --- a/clusters/dev/apps/lineage-event-handler/values.yaml +++ b/clusters/dev/apps/lineage-event-handler/values.yaml @@ -1,4 +1,4 @@ -metadata-event-handler: +lineage-event-handler: image: repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/lineage_event_handler pullPolicy: IfNotPresent @@ -24,7 +24,7 @@ metadata-event-handler: ATLAS_API: "http://atlas.utility:21000" ATLAS_ENTITY_TYPE: "metadata_items" ATLAS_ADMIN: "admin" - SEEK_TO_BEGINNING: true + SEEK_TO_BEGINNING: "true" extraEnvYaml: - name: ATLAS_PASSWD