IEBH-483: Deploy lineage-event-handler service in dev

README.md

@@ -48,8 +48,10 @@ This repo uses ArgoCD's app-of-apps pattern: a root Application (`root-app.yaml`
 | 8 | download-greenroom | Download service (greenroom ns) |
 | 8 | download-core | Download service (core ns) |
 | 8 | search | Search service (ES-backed) |
+| 8 | atlas | Apache Atlas (lineage) |
 | 9 | kong | API gateway |
 | 9 | metadata-event-handler | Kafka→ES event indexer |
+| 9 | lineage-event-handler | Kafka→Atlas event indexer |
 | 9 | kg-integration | EBRAINS Knowledge Graph integration |
 | 9 | audit-trail | |
 | 10 | bff | Backend-for-frontend (web) |

clusters/dev/apps/dataops/values.yaml

@@ -34,7 +34,6 @@ dataops-service:
     REDIS_PORT: "6379"
     REDIS_DB: "0"
     WORKERS: "2"
-    LINEAGE_SERVICE: "http://lineage.utility:5064"
     QUEUE_SERVICE: "http://queue-producer.greenroom:6060"
     METADATA_SERVICE: "http://metadata.utility:5066"
     AUTH_SERVICE: "http://auth.utility:5061"

clusters/dev/apps/lineage-event-handler/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: lineage-event-handler
+version: 0.1.0
+dependencies:
+  - name: metadata-event-handler
+    alias: lineage-event-handler
+    version: "0.1.1"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/dev/apps/lineage-event-handler/application.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: lineage-event-handler
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "9"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/dev/apps/lineage-event-handler
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - ../../versions.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: utility
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/dev/apps/lineage-event-handler/values.yaml

@@ -0,0 +1,48 @@
+lineage-event-handler:
+  image:
+    repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/lineage_event_handler
+    pullPolicy: IfNotPresent
+
+  fullnameOverride: lineage-event-handler
+  replicaCount: 1
+
+  container:
+    port: 5067
+
+  service:
+    type: ClusterIP
+    port: 5067
+    targetPort: 5067
+
+  imagePullSecrets:
+    - name: docker-registry-secret
+
+  extraEnv:
+    KAFKA_TOPICS: '["metadata.items.activity"]'
+    KAFKA_SERVICE: "kafka.utility:9092"
+    KAFKA_GROUP_ID: "lineage_consumer_group"
+    ATLAS_API: "http://atlas.utility:21000"
+    ATLAS_ENTITY_TYPE: "metadata_items"
+    ATLAS_ADMIN: "admin"
+    SEEK_TO_BEGINNING: "true"
+
+  extraEnvYaml:
+    - name: ATLAS_PASSWD
+      valueFrom:
+        secretKeyRef:
+          name: atlas-credentials
+          key: atlas-password
+
+  resources:
+    limits:
+      cpu: "1"
+      memory: 100Mi
+    requests:
+      cpu: 10m
+      memory: 10Mi
+
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 33%
+    type: RollingUpdate

clusters/prod/apps/dataops/values.yaml

@@ -34,7 +34,6 @@ dataops-service:
     REDIS_PORT: "6379"
     REDIS_DB: "0"
     WORKERS: "2"
-    LINEAGE_SERVICE: "http://lineage.utility:5064"
     QUEUE_SERVICE: "http://queue-producer.greenroom:6060"
     METADATA_SERVICE: "http://metadata.utility:5066"
     AUTH_SERVICE: "http://auth.utility:5061"