diff --git a/README.md b/README.md index efdc9ec..afb0079 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,10 @@ This repo uses ArgoCD's app-of-apps pattern: a root Application (`root-app.yaml` | 8 | download-greenroom | Download service (greenroom ns) | | 8 | download-core | Download service (core ns) | | 8 | search | Search service (ES-backed) | +| 8 | atlas | Apache Atlas (lineage) | | 9 | kong | API gateway | | 9 | metadata-event-handler | Kafka→ES event indexer | +| 9 | lineage-event-handler | Kafka→Atlas event indexer | | 9 | kg-integration | EBRAINS Knowledge Graph integration | | 9 | audit-trail | | | 10 | bff | Backend-for-frontend (web) | diff --git a/clusters/dev/apps/dataops/values.yaml b/clusters/dev/apps/dataops/values.yaml index c6a13cf..0a09474 100644 --- a/clusters/dev/apps/dataops/values.yaml +++ b/clusters/dev/apps/dataops/values.yaml @@ -34,7 +34,6 @@ dataops-service: REDIS_PORT: "6379" REDIS_DB: "0" WORKERS: "2" - LINEAGE_SERVICE: "http://lineage.utility:5064" QUEUE_SERVICE: "http://queue-producer.greenroom:6060" METADATA_SERVICE: "http://metadata.utility:5066" AUTH_SERVICE: "http://auth.utility:5061" diff --git a/clusters/dev/apps/lineage-event-handler/Chart.yaml b/clusters/dev/apps/lineage-event-handler/Chart.yaml new file mode 100644 index 0000000..cd617e0 --- /dev/null +++ b/clusters/dev/apps/lineage-event-handler/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: lineage-event-handler +version: 0.1.0 +dependencies: + - name: metadata-event-handler + alias: lineage-event-handler + version: "0.1.1" + repository: https://pilotdataplatform.github.io/helm-charts/ diff --git a/clusters/dev/apps/lineage-event-handler/application.yaml b/clusters/dev/apps/lineage-event-handler/application.yaml new file mode 100644 index 0000000..dbe42e8 --- /dev/null +++ b/clusters/dev/apps/lineage-event-handler/application.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: lineage-event-handler + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "9" +spec: + project: default + source: + repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git + targetRevision: main + path: clusters/dev/apps/lineage-event-handler + helm: + valueFiles: + - ../../registry.yaml + - ../../versions.yaml + - values.yaml + destination: + server: https://kubernetes.default.svc + namespace: utility + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/clusters/dev/apps/lineage-event-handler/values.yaml b/clusters/dev/apps/lineage-event-handler/values.yaml new file mode 100644 index 0000000..6449782 --- /dev/null +++ b/clusters/dev/apps/lineage-event-handler/values.yaml @@ -0,0 +1,48 @@ +lineage-event-handler: + image: + repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/lineage_event_handler + pullPolicy: IfNotPresent + + fullnameOverride: lineage-event-handler + replicaCount: 1 + + container: + port: 5067 + + service: + type: ClusterIP + port: 5067 + targetPort: 5067 + + imagePullSecrets: + - name: docker-registry-secret + + extraEnv: + KAFKA_TOPICS: '["metadata.items.activity"]' + KAFKA_SERVICE: "kafka.utility:9092" + KAFKA_GROUP_ID: "lineage_consumer_group" + ATLAS_API: "http://atlas.utility:21000" + ATLAS_ENTITY_TYPE: "metadata_items" + ATLAS_ADMIN: "admin" + SEEK_TO_BEGINNING: "true" + + extraEnvYaml: + - name: ATLAS_PASSWD + valueFrom: + secretKeyRef: + name: atlas-credentials + key: atlas-password + + resources: + limits: + cpu: "1" + memory: 100Mi + requests: + cpu: 10m + memory: 10Mi + + updateStrategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 33% + type: RollingUpdate diff --git a/clusters/prod/apps/dataops/values.yaml b/clusters/prod/apps/dataops/values.yaml index 219d949..0afbbc0 100644 --- a/clusters/prod/apps/dataops/values.yaml +++ b/clusters/prod/apps/dataops/values.yaml @@ -34,7 +34,6 @@ dataops-service: REDIS_PORT: "6379" REDIS_DB: "0" WORKERS: "2" - LINEAGE_SERVICE: "http://lineage.utility:5064" QUEUE_SERVICE: "http://queue-producer.greenroom:6060" METADATA_SERVICE: "http://metadata.utility:5066" AUTH_SERVICE: "http://auth.utility:5061"