Added grafana dashboard and prometheus exporter. Repository cleaning

core/agent-api-build.sh

@@ -9,7 +9,7 @@ echo "Copying connection tracker binaries"
 cp -r target/bpfel-unknown-none/release/conntracker conntracker
 
 # Run docker build
-docker build -f api/Dockerfile -t cortexflow-agent:0.0.1 .
+docker build -f api/Dockerfile -t cortexflow-agent:0.0.1 --provenance=false --sbom=false .
 
 # Cleanup
 echo "Cleaning building files"

core/common/src/otel_metrics.rs

@@ -42,25 +42,25 @@ impl Metrics {
     pub fn new(meter: &Meter) -> Self {
         // total events
         let events_total = meter
-            .u64_counter("cortexbrain_events_total")
+            .u64_counter("events_total")
             .with_description("Total number of eBPF events processed")
             .build();
 
         // total packets
         let packets_total = meter
-            .u64_counter("cortexbrain_packets_total")
+            .u64_counter("packets_total")
             .with_description("Total number of network events processed")
             .build();
 
         // socket drops
         let sk_drops = meter
-            .i64_gauge("cortexbrain_sk_drops")
+            .i64_gauge("sk_drops")
             .with_description("Socket drop count per event")
             .build();
 
         // socket errors
         let sk_err = meter
-            .i64_gauge("cortexbrain_sk_err")
+            .i64_gauge("sk_err")
             .with_description("Socket error count per event")
             .build();
 
@@ -72,7 +72,7 @@ impl Metrics {
 
         // timestamp microseconds grouped
         let ts_us = meter
-            .u64_histogram("cortexbrain_ts_us")
+            .u64_histogram("ts_us")
             .with_description("Distribution of timestamp values from eBPF events")
             .build();
 

core/src/components/identity/build-identity.sh

@@ -11,7 +11,7 @@ cp -r ../../../target/bpfel-unknown-none/release/conntracker conntracker
 cp -r ../../../common common 
 
 # Run docker build
-docker build -t identity:0.0.1 .
+docker build -t identity:0.0.1 --provenance=false --sbom=false .
 
 # Cleanup
 echo "Cleaning building files"

core/src/components/metrics/build-metrics.sh

@@ -11,7 +11,7 @@ cp -r ../../../target/bpfel-unknown-none/release/metrics_tracer metrics_tracer
 cp -r ../../../common common 
 
 # Run docker build
-docker build -t metrics:0.0.1 .
+docker build -t metrics:0.0.1 --provenance=false --sbom=false .
 
 # Cleanup
 echo "Cleaning building files"

core/src/testing/grafana.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+  namespace: cortexflow
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: cortexflow
+  labels:
+    app: grafana
+  name: grafana
+spec:
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      securityContext:
+        fsGroup: 472
+        supplementalGroups:
+          - 0
+      containers:
+        - name: grafana
+          image: grafana/grafana:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3000
+              name: http-grafana
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /robots.txt
+              port: 3000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 2
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3000
+            timeoutSeconds: 1
+          resources:
+            requests:
+              cpu: 250m
+              memory: 750Mi
+          volumeMounts:
+            - mountPath: /var/lib/grafana
+              name: grafana-pv
+      volumes:
+        - name: grafana-pv
+          persistentVolumeClaim:
+            claimName: grafana-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  namespace: cortexflow
+spec:
+  ports:
+    - port: 3000
+      protocol: TCP
+      targetPort: http-grafana
+  selector:
+    app: grafana
+  sessionAffinity: None
+  type: LoadBalancer

core/src/testing/otel_agent.yaml

@@ -19,11 +19,14 @@ data:
 
     exporters:
       otlp:
-        endpoint: otel-collector.cortexflow.svc.cluster.local:4317
+        endpoint: ${OTEL_COLLECTOR_SERVICE_HOST}:4317
         tls:
           insecure: true
       logging:
         loglevel: info   
+      prometheus:
+          endpoint: 0.0.0.0:8889
+          namespace: cortexbrain
 
     service:
       pipelines:
@@ -35,7 +38,7 @@ data:
           exporters: [otlp, logging]
         metrics:
           receivers: [otlp]
-          exporters: [otlp, logging]
+          exporters: [otlp, prometheus]
 
 ---
 apiVersion: apps/v1
@@ -79,6 +82,9 @@ spec:
         - containerPort: 4318
           hostPort: 4318
           protocol: TCP
+        - containerPort: 8889
+          hostPort: 8889
+          protocol: TCP
         env:
         - name: GOMEMLIMIT
           value: 400MiB
@@ -124,6 +130,10 @@ data:
     #    tls:
     #      insecure: true
       logging: {}  
+      prometheus:
+          endpoint: 0.0.0.0:8889
+          namespace: cortexbrain
+
 
     service:
       pipelines:
@@ -138,7 +148,7 @@ data:
         metrics: 
           receivers: [otlp]
           processors: [memory_limiter]
-          exporters: [logging]
+          exporters: [logging,prometheus]
 
 ---
 apiVersion: v1
@@ -161,8 +171,8 @@ spec:
     port: 4318
     targetPort: 4318
   - name: metrics
-    port: 8888
-    targetPort: 8888
+    port: 8889
+    targetPort: 8889
 
 ---
 apiVersion: apps/v1
@@ -201,7 +211,7 @@ spec:
         ports:
         - containerPort: 4317
         - containerPort: 4318
-        - containerPort: 8888
+        - containerPort: 8889
         env:
         - name: GOMEMLIMIT
           value: 1600MiB

core/src/testing/prometheus.yaml

@@ -0,0 +1,104 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheus-pvc
+  namespace: cortexflow
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: cortexflow
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 5s
+      evaluation_interval: 5s
+
+    scrape_configs:
+      - job_name: 'otel-collector'
+        static_configs:
+          - targets: ['otel-collector:8889']
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: cortexflow
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      securityContext:
+        fsGroup: 65534
+        runAsUser: 65534
+        runAsNonRoot: true
+      containers:
+        - name: prometheus
+          image: prom/prometheus:v2.51.2
+          args:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--storage.tsdb.path=/prometheus"
+            - "--web.enable-lifecycle"
+          ports:
+            - containerPort: 9090
+              name: web
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: 9090
+            initialDelaySeconds: 15
+            timeoutSeconds: 3
+          resources:
+            requests:
+              cpu: 250m
+              memory: 500Mi
+            limits:
+              cpu: 500m
+              memory: 1Gi
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/prometheus
+            - name: storage-volume
+              mountPath: /prometheus
+      volumes:
+        - name: config-volume
+          configMap:
+            name: prometheus-config
+        - name: storage-volume
+          persistentVolumeClaim:
+            claimName: prometheus-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: cortexflow
+spec:
+  ports:
+    - port: 9090
+      targetPort: 9090
+      name: web
+  selector:
+    app: prometheus
+  type: ClusterIP