IEBH-483: Deploy atlas in prod

clusters/prod/apps/atlas/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: atlas
+version: 0.1.0
+dependencies:
+  - name: base-chart-hdc
+    version: "1.0.1"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/prod/apps/atlas/application.yaml

@@ -0,0 +1,27 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: atlas
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/prod/apps/atlas
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: utility
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/prod/apps/atlas/templates/external-secret.yaml

@@ -0,0 +1,17 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: atlas-credentials
+  namespace: {{ .Release.Namespace }}
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  target:
+    name: atlas-credentials
+  data:
+    - secretKey: atlas-password
+      remoteRef:
+        key: secret/data/atlas
+        property: atlas-password

clusters/prod/apps/atlas/templates/pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: atlas-storage
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    - ReadWriteOncePod
+  storageClassName: csi-cinder-high-speed
+  resources:
+    requests:
+      storage: 5Gi

clusters/prod/apps/atlas/values.yaml

@@ -0,0 +1,91 @@
+base-chart-hdc:
+  image:
+    repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-external/apache-atlas
+    tag: "2.1.0"
+    tagPrefix: "legacy"
+    pullPolicy: IfNotPresent
+
+  fullnameOverride: atlas
+
+  labels:
+    app: atlas
+    instance: atlas
+
+  replicaCount: 1
+
+  container:
+    name: atlas
+    command: [ "/bin/bash", "-c" ]
+    args:
+      - |
+        ADMIN_PASSWORD_HASH=$(echo -n "${ADMIN_PASSWORD}" | sha256sum | awk '{print $1}')
+        sed -i "s/^\(admin=ADMIN::\).*/\1${ADMIN_PASSWORD_HASH}/" /opt/apache-atlas-2.1.0/conf/users-credentials.properties
+
+        /opt/apache-atlas-2.1.0/bin/atlas_start.py &
+        tail -F /opt/apache-atlas-2.1.0/logs/application.log
+    ports:
+      - name: http
+        containerPort: 21000
+        protocol: TCP
+
+  service:
+    type: ClusterIP
+    ports:
+      - port: 21000
+        targetPort: 21000
+        protocol: TCP
+        name: http
+
+  imagePullSecrets:
+    - name: docker-registry-secret
+
+  extraEnv:
+    LOG4J_FORMAT_MSG_NO_LOOKUPS: "true"
+    MANAGE_LOCAL_HBASE: "true"
+    MANAGE_LOCAL_SOLR: "true"
+
+  extraEnvYaml:
+    - name: ADMIN_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: atlas-credentials
+          key: atlas-password
+
+  extraVolumes:
+    - name: atlas-storage
+      persistentVolumeClaim:
+        claimName: atlas-storage
+
+  extraVolumeMounts:
+    - name: atlas-storage
+      mountPath: /opt/apache-atlas-2.1.0/data
+      subPath: data
+    - name: atlas-storage
+      mountPath: /opt/apache-atlas-2.1.0/logs
+      subPath: logs
+
+  resources:
+    requests:
+      cpu: 10m
+      memory: 1024Mi
+    limits:
+      cpu: 500m
+      memory: 2048Mi
+
+  readinessProbe:
+    tcpSocket:
+      port: 21000
+    initialDelaySeconds: 60
+    periodSeconds: 10
+
+  livenessProbe:
+    failureThreshold: 10
+    tcpSocket:
+      port: 21000
+    initialDelaySeconds: 1800
+    periodSeconds: 30
+    successThreshold: 1
+    timeoutSeconds: 10
+
+  updateStrategy:
+    type: Recreate

clusters/prod/apps/audit-trail/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: audit-trail
+version: 0.1.0
+dependencies:
+  - name: audit-trail-service
+    version: "0.2.0"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/prod/apps/audit-trail/application.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: audit-trail
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "9"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/prod/apps/audit-trail
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - ../../versions.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: utility
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/prod/apps/audit-trail/templates/external-secret.yaml

@@ -0,0 +1,17 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: audit-trail-credentials
+  namespace: utility
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault
+  target:
+    name: audit-trail-credentials
+  data:
+    - secretKey: atlas-password
+      remoteRef:
+        key: secret/data/atlas
+        property: atlas-password

clusters/prod/apps/audit-trail/values.yaml

@@ -0,0 +1,61 @@
+audit-trail-service:
+  image:
+    repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/audit-trail
+    pullPolicy: IfNotPresent
+
+  fullnameOverride: audit-trail
+  replicaCount: 1
+
+  container:
+    port: 5077
+
+  service:
+    type: ClusterIP
+    port: 5077
+
+  imagePullSecrets:
+    - name: docker-registry-secret
+
+  extraEnv:
+    ATLAS_ADMIN: "admin"
+    ATLAS_HOST: "atlas.utility"
+    ATLAS_PORT: "21000"
+
+  extraEnvYaml:
+    - name: ATLAS_PASSWD
+      valueFrom:
+        secretKeyRef:
+          name: audit-trail-credentials
+          key: atlas-password
+
+  resources:
+    limits:
+      cpu: "1"
+      memory: 500Mi
+    requests:
+      cpu: 10m
+      memory: 50Mi
+
+  readinessProbe:
+    failureThreshold: 3
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    successThreshold: 1
+    tcpSocket:
+      port: 5077
+
+  livenessProbe:
+    failureThreshold: 3
+    httpGet:
+      path: /v1/health/
+      port: 5077
+      scheme: HTTP
+    periodSeconds: 10
+    successThreshold: 1
+    timeoutSeconds: 3
+
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 33%
+    type: RollingUpdate

clusters/prod/apps/bff/values.yaml

@@ -1,7 +1,7 @@
 base-chart-hdc:
   image:
     repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/bff-web
-    tag: "2.2.88"
+    tag: "2.2.90"
     tagPrefix: "bff"
     pullPolicy: IfNotPresent
   fullnameOverride: bff
@@ -66,7 +66,7 @@ base-chart-hdc:
     APPROVAL_SERVICE: "http://approval.utility:8000"
     SEARCH_SERVICE: "http://search.utility:5064"
     WORKSPACE_SERVICE: "http://workspace.utility:5068"
-    PROVENANCE_SERVICE: "http://localhost:5077"
+    PROVENANCE_SERVICE: "http://audit-trail.utility:5077"
     # MinIO
     MINIO_HOST: "minio.minio:9000"
     MINIO_BUCKET_ENCRYPTION: "true"

clusters/prod/apps/kg-integration/values.yaml

@@ -1,7 +1,7 @@
 base-chart:
   image:
     repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/kg-integration
-    tag: "1.3.34"
+    tag: "1.3.36"
     pullPolicy: IfNotPresent
   fullnameOverride: kg-integration
   replicaCount: 1

clusters/prod/apps/lineage-event-handler/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: lineage-event-handler
+version: 0.1.0
+dependencies:
+  - name: metadata-event-handler
+    alias: lineage-event-handler
+    version: "0.1.1"
+    repository: https://pilotdataplatform.github.io/helm-charts/

clusters/prod/apps/lineage-event-handler/application.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: lineage-event-handler
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "9"
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/PilotDataPlatform/pilot-hdc-platform-gitops.git
+    targetRevision: main
+    path: clusters/prod/apps/lineage-event-handler
+    helm:
+      valueFiles:
+        - ../../registry.yaml
+        - ../../versions.yaml
+        - values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: utility
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

clusters/prod/apps/lineage-event-handler/values.yaml

@@ -0,0 +1,51 @@
+lineage-event-handler:
+  image:
+    repository: n47w5524.c1.de1.container-registry.ovh.net/hdc-services-image/lineage_event_handler
+    pullPolicy: IfNotPresent
+
+  fullnameOverride: lineage-event-handler
+  replicaCount: 1
+
+  container:
+    port: 5067
+
+  service:
+    type: ClusterIP
+    port: 5067
+    targetPort: 5067
+
+  imagePullSecrets:
+    - name: docker-registry-secret
+
+  # appConfig: KAFKA_SERVICE is rendered by the chart template
+  appConfig:
+    KAFKA_SERVICE: "kafka.utility:9092"
+
+  extraEnv:
+    KAFKA_TOPICS: '["metadata.items.activity"]'
+    KAFKA_GROUP_ID: "lineage_consumer_group"
+    ATLAS_API: "http://atlas.utility:21000"
+    ATLAS_ENTITY_TYPE: "metadata_items"
+    ATLAS_ADMIN: "admin"
+    SEEK_TO_BEGINNING: "true"
+
+  extraEnvYaml:
+    - name: ATLAS_PASSWD
+      valueFrom:
+        secretKeyRef:
+          name: atlas-credentials
+          key: atlas-password
+
+  resources:
+    limits:
+      cpu: "1"
+      memory: 100Mi
+    requests:
+      cpu: 10m
+      memory: 10Mi
+
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 33%
+    type: RollingUpdate