feat(metrics): add OTEL Prometheus metrics and custom recorder (#111)

* feat(metrics): add OTEL Prometheus metrics and custom recorder

- Add internal/metrics package with OTEL Prometheus exporter and custom
  Recorder
- Instrument controllers and tokenmanager with metrics recording
- Update Helm chart to support metrics port/secure config and expose
  /metrics
- Add e2e and unit tests for metrics
- Update Go version and dependencies for OTEL support

* fix(helm): template image repository in deployment.yaml

Author: isometry

cmd/manager/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"os"
@@ -39,6 +40,7 @@ import (
 
 	githubv1 "github.com/isometry/github-token-manager/api/v1"
 	"github.com/isometry/github-token-manager/internal/controller"
+	"github.com/isometry/github-token-manager/internal/metrics"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -87,6 +89,8 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	ctx := ctrl.SetupSignalHandler()
+
 	if disableHTTP2 {
 		forceHTTP11 := func(c *tls.Config) {
 			setupLog.Info("disabling http/2")
@@ -191,16 +195,29 @@ func main() {
 		os.Exit(1)
 	}
 
+	metricsRecorder, err := metrics.Setup()
+	if err != nil {
+		setupLog.Error(err, "unable to set up metrics")
+		os.Exit(1)
+	}
+	defer func() {
+		if err := metricsRecorder.Shutdown(context.Background()); err != nil {
+			setupLog.Error(err, "shutting down meter provider")
+		}
+	}()
+
 	if err = (&controller.TokenReconciler{
-		Client: mgr.GetClient(),
+		Client:  mgr.GetClient(),
+		Metrics: metricsRecorder,
 		// Scheme: mgr.GetScheme(),
 		// Recorder: mgr.GetEventRecorderFor("token-controller"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Token")
 		os.Exit(1)
 	}
 	if err = (&controller.ClusterTokenReconciler{
-		Client: mgr.GetClient(),
+		Client:  mgr.GetClient(),
+		Metrics: metricsRecorder,
 		// Scheme: mgr.GetScheme(),
 		// Recorder: mgr.GetEventRecorderFor("clustertoken-controller"),
 	}).SetupWithManager(mgr); err != nil {
@@ -235,7 +252,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

config/rbac/role.yaml

@@ -4,51 +4,51 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
-  - apiGroups:
-      - ""
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - github.as-code.io
-    resources:
-      - clustertokens
-      - tokens
-    verbs:
-      - create
-      - delete
-      - get
-      - list
-      - patch
-      - update
-      - watch
-  - apiGroups:
-      - github.as-code.io
-    resources:
-      - clustertokens/finalizers
-      - tokens/finalizers
-    verbs:
-      - update
-  - apiGroups:
-      - github.as-code.io
-    resources:
-      - clustertokens/status
-      - tokens/status
-    verbs:
-      - get
-      - patch
-      - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - github.as-code.io
+  resources:
+  - clustertokens
+  - tokens
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - github.as-code.io
+  resources:
+  - clustertokens/finalizers
+  - tokens/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - github.as-code.io
+  resources:
+  - clustertokens/status
+  - tokens/status
+  verbs:
+  - get
+  - patch
+  - update

deploy/charts/github-token-manager/templates/deployment.yaml

@@ -43,16 +43,24 @@ spec:
       containers:
         - args:
             - --health-probe-bind-address=:8081
-            - --metrics-bind-address=127.0.0.1:8080
+            - --metrics-bind-address=:{{ .Values.metrics.listen.port }}
+            - --metrics-secure={{ .Values.metrics.secure }}
             - --leader-elect
+          {{- range $key, $value := $manager.extraArgs }}
+          {{- if kindIs "invalid" $value }}
+            - --{{ $key }}
+          {{- else }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          {{- end }}
           {{- with $manager.env }}
           env:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           image: {{ if (hasPrefix "sha256:" (default "" $manager.tag)) -}}
-              {{- printf "%s@%s" $manager.repository $manager.tag -}}
+              {{- printf "%s@%s" (tpl $manager.repository .) $manager.tag -}}
             {{- else -}}
-              {{- printf "%s:%s" $manager.repository (or $manager.tag $.Chart.AppVersion "latest") -}}
+              {{- printf "%s:%s" (tpl $manager.repository .) (or $manager.tag $.Chart.AppVersion "latest") -}}
             {{- end }}
           livenessProbe:
             httpGet:

deploy/charts/github-token-manager/templates/service.yaml

@@ -14,10 +14,12 @@ metadata:
     component: service
     {{- include "labels" . | nindent 4 }}
 spec:
-  {{- $service := .Values.metrics.service }}
   type: {{ .Values.metrics.service.type }}
   ports:
-    {{ $service.ports | toYaml | nindent 4 }}
+    - name: http-metrics
+      port: {{ .Values.metrics.listen.port }}
+      protocol: TCP
+      targetPort: {{ .Values.metrics.listen.port }}
   selector:
     {{- include "selectorLabels" . | nindent 4 }}
 {{- end }}

deploy/charts/github-token-manager/values.yaml

@@ -36,22 +36,18 @@ rbac:
 
 ## metrics:
 ##   enabled: true | false
-##   service
+##   listen:
+##     port: port number for --metrics-bind-address=:<port>
+##   secure: true | false (controls --metrics-secure flag; false = plain HTTP)
+##   service:
 ##     type: ClusterIP | NodePort | LoadBalancer | ExternalName
-##     ports: list of ports
-##       name: name of the port
-##       port: port number
-##       protocol: protocol
-##       targetPort: target port
 metrics:
   enabled: true
+  listen:
+    port: 8080
+  secure: false
   service:
     type: ClusterIP
-    ports:
-      - name: https
-        port: 8443
-        protocol: TCP
-        targetPort: https
 
 ## manager
 ##   repository: image repository
@@ -62,6 +58,11 @@ metrics:
 ##   nodeSelector: node selector (optional)
 ##   env: list of additional environment variables to set on the manager container
 ##   resources: manager container resource requests and limits
+## manager
+##   repository: image repository
+##   tag: image tag
+##   replicas: number of replicas
+##   extraArgs: map of additional CLI flags rendered as --key=value
 manager:
   repository: ghcr.io/isometry/github-token-manager
   tag: ~ # defaults to chart appVersion
@@ -72,6 +73,7 @@ manager:
   # additional environment variables to set on the controller container
   # e.g. `[{name: VAULT_ADDR, value: http://vault:8200}]`
   env: []
+  extraArgs: {}
   resources:
     limits:
       cpu: 500m