From 620c564b3ab6d235cf4fac7b370dc757e2a69c68 Mon Sep 17 00:00:00 2001 From: shcherbak Date: Fri, 14 Aug 2026 10:18:02 +0300 Subject: [PATCH] unittests --- README.md | 41 ++---- charts/keepup/Chart.yaml | 4 +- charts/keepup/templates/_config.yaml | 6 + charts/keepup/templates/confiig.yaml | 2 +- charts/keepup/templates/deployment.yaml | 14 +- charts/keepup/templates/httproute.yaml | 3 - charts/keepup/templates/ingress.yaml | 7 - charts/keepup/templates/seecret.yaml | 1 - charts/keepup/templates/service.yaml | 2 +- charts/keepup/values.yaml | 19 +++ docker/Dockerfile | 10 +- go.mod | 21 ++- go.sum | 54 ++++--- src/config/config.go | 33 ++--- src/handler/helmscrape.go | 38 ++++- src/handler/helmscrape_test.go | 130 +++++++++++++++++ src/handler/middleware.go | 179 ++++++------------------ src/handler/osrelease.go | 89 ------------ src/handler/packageversions.go | 54 ++++++- src/handler/packageversions_test.go | 131 +++++++++++++++++ src/handler/testhelpers_test.go | 14 ++ src/main.go | 24 +--- src/metrics/helmmetrics.go | 7 +- src/metrics/metrics.go | 62 -------- tests/end-to-end/example-002.json | 10 -- tests/end-to-end/run.sh | 3 - 26 files changed, 516 insertions(+), 442 deletions(-) create mode 100644 src/handler/helmscrape_test.go delete mode 100644 src/handler/osrelease.go create mode 100644 src/handler/packageversions_test.go create mode 100644 src/handler/testhelpers_test.go delete mode 100644 src/metrics/metrics.go delete mode 100644 tests/end-to-end/example-002.json diff --git a/README.md b/README.md index d8c226f..c417cbe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # keepup -A lightweight Prometheus exporter that collects infrastructure inventory pushed by remote agents - OS releases, package versions (with end-of-life enrichment), and Kubernetes/Helm deployments - and exposes it as metrics. +A lightweight Prometheus exporter that collects infrastructure inventory pushed by remote agents - package versions (with end-of-life enrichment) and Kubernetes/Helm deployments - and exposes it as metrics. `keepup` holds no state of its own: Redis is both the write buffer and the read source. Agents `PUT` JSON, `keepup` validates and stores it with a TTL, and Prometheus scrapes `/metrics` on demand. @@ -10,7 +10,6 @@ A lightweight Prometheus exporter that collects infrastructure inventory pushed - [Quick start](#quick-start) - [Configuration](#configuration) - [API](#api) - - [`PUT /os-release`](#put-os-release) - [`PUT /package-version`](#put-package-version) - [`PUT /helm-cluster`](#put-helm-cluster) - [Metrics](#metrics) @@ -22,18 +21,16 @@ A lightweight Prometheus exporter that collects infrastructure inventory pushed ``` agent(s) keepup Prometheus -┌─────────┐ PUT + token ┌───────────────────┐ scrape ┌────────────┐ -│ os-info │ ───────────────>│ handler ──▶ Redis │<────────│ /metrics │ -│ pkg-vers│ │ (TTL) │ │ │ -│ helm │ └───────────────────┘ └────────────┘ -└─────────┘ +----------- PUT + token --------------------- scrape -------------- +│ pkg-vers│ --------------->│ handler --> Redis │<---------│ /metrics | +│ helm │ │ (TTL) │ │ │ +----------- --------------------- -------------- ``` Every data domain follows the same shape: | Domain | Endpoint | Redis key | Metric | |---|---|---|---| -| OS release *(deprecated)* | `PUT /os-release` | SHA1 of `{data_center}-{host_ip}` | `os_release_info` | | Package versions | `PUT /package-version` | SHA1 of `{data_center}-{host_ip}-PACKAGE_UUID` | `package_version_info` | | Kubernetes / Helm | `PUT /helm-cluster` | SHA1 of `{cluster_name}` | `kubernetes_cluster_info` | @@ -58,7 +55,7 @@ docker build -f docker/Dockerfile -t keepup . The server listens on `LISTEN_PORT` (default `9101` in dev) and exposes: -- `PUT`/`GET /os-release`, `/package-version`, `/helm-cluster` - data ingestion & lookup (require `x-api-token`) +- `PUT`/`GET /package-version`, `/helm-cluster` - data ingestion & lookup (require `x-api-token`) - `GET /metrics` - Prometheus scrape endpoint (no auth) - `GET /healthcheck` - liveness probe @@ -80,23 +77,6 @@ Config is loaded from environment variables. If `APP_ENV` is unset, `keepup` loa All data endpoints require an `x-api-token` header matching `API_TOKEN`, and accept both `PUT` (insert) and `GET` (lookup by `id`). -### `PUT /os-release` - -> **Deprecated** - kept for backwards compatibility, no longer receiving new fields (e.g. `team`). Do not build new integrations against it. - -```jsonc -{ - "release": { - "os_id": "debian", - "version_codename": "bullseye", - "version": "11 (bullseye)", - "version_id": "11", - "data_center": "aaa", - "host_ip": "101.122.418.4" - } -} -``` - ### `PUT /package-version` ```jsonc @@ -135,13 +115,18 @@ Unlike the other two endpoints, the request body maps directly onto the stored s | Metric | Labels | |---|---| -| `os_release_info` *(deprecated)* | `id`, `os_id`, `version_codename`, `version`, `version_id`, `data_center`, `host_ip` | | `package_version_info` | `id`, `package_name`, `current_version`, `current_version_eof`, `newest_version`, `expired`, `data_center`, `host_ip`, `team` | | `kubernetes_cluster_info` | `id`, `cluster_name`, `kube_version`, `chart_name`, `chart_version`, `chart_namespace`, `team` | ## Testing -There are no unit tests - only an end-to-end shell script that exercises all three endpoints against a running server: +Unit tests cover the handler package against an in-process fake Redis ([`miniredis`](https://github.com/alicebob/miniredis)) - no external services required: + +```bash +go test ./... +``` + +An end-to-end shell script also exercises both endpoints against a running server: ```bash # start the server first (see Quick start), then: diff --git a/charts/keepup/Chart.yaml b/charts/keepup/Chart.yaml index 7164a13..0d724a2 100644 --- a/charts/keepup/Chart.yaml +++ b/charts/keepup/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: keepup description: eondoflife.date version tracker service -version: 1.7.0 -appVersion: 1.6.0 +version: 1.7.1 +appVersion: 1.7.0 diff --git a/charts/keepup/templates/_config.yaml b/charts/keepup/templates/_config.yaml index d70f9f6..4dbae88 100644 --- a/charts/keepup/templates/_config.yaml +++ b/charts/keepup/templates/_config.yaml @@ -5,6 +5,12 @@ name: keepup-config key: APP_ENV +- name: LISTEN_PORT + valueFrom: + configMapKeyRef: + name: keepup-config + key: LISTEN_PORT + - name: REDIS_ADDR valueFrom: configMapKeyRef: diff --git a/charts/keepup/templates/confiig.yaml b/charts/keepup/templates/confiig.yaml index c63346b..0113697 100644 --- a/charts/keepup/templates/confiig.yaml +++ b/charts/keepup/templates/confiig.yaml @@ -2,9 +2,9 @@ kind: ConfigMap apiVersion: v1 metadata: name: keepup-config - namespace: keepup data: APP_ENV: {{ .Values.appEnv | quote }} + LISTEN_PORT: {{ .Values.listenPort | quote }} REDIS_ADDR: {{ .Values.redisAddr | quote }} REDIS_PORT: {{ .Values.redisPort | quote }} REDIS_DBNO: {{ .Values.redisDbNo | quote }} diff --git a/charts/keepup/templates/deployment.yaml b/charts/keepup/templates/deployment.yaml index 06a0f76..ffbc063 100644 --- a/charts/keepup/templates/deployment.yaml +++ b/charts/keepup/templates/deployment.yaml @@ -22,6 +22,9 @@ spec: {{- toYaml .Values.annotations | nindent 8 }} {{- end }} spec: + {{- with .Values.podSecurityContext }} + securityContext: {{ toYaml . | nindent 8 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} @@ -35,15 +38,24 @@ spec: - name: keepup-redis-container imagePullPolicy: {{ .Values.redis.pullPolicy }} image: {{ .Values.redis.image | quote }} + {{- with .Values.redis.securityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} {{ end }} - name: keepup-main-container imagePullPolicy: {{ .Values.main.pullPolicy }} image: "{{ .Values.main.image }}:{{ .Values.main.tag | default .Chart.AppVersion }}" env: {{ include "project.config" . | indent 12 }} + {{- with .Values.main.securityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.listenPort | int }} readinessProbe: httpGet: path: /healthcheck - port: 80 + port: http initialDelaySeconds: 10 periodSeconds: 60 diff --git a/charts/keepup/templates/httproute.yaml b/charts/keepup/templates/httproute.yaml index 9c69c14..d86dd10 100644 --- a/charts/keepup/templates/httproute.yaml +++ b/charts/keepup/templates/httproute.yaml @@ -15,9 +15,6 @@ spec: {{- end }} rules: - matches: - - path: - type: PathPrefix - value: /os-release - path: type: PathPrefix value: /helm-cluster diff --git a/charts/keepup/templates/ingress.yaml b/charts/keepup/templates/ingress.yaml index 21178f7..ceec90e 100644 --- a/charts/keepup/templates/ingress.yaml +++ b/charts/keepup/templates/ingress.yaml @@ -15,13 +15,6 @@ spec: - host: {{ .Values.ingress.host }} http: paths: - - path: /os-release - pathType: ImplementationSpecific - backend: - service: - name: keepup-service - port: - number: 80 - path: /helm-cluster pathType: ImplementationSpecific backend: diff --git a/charts/keepup/templates/seecret.yaml b/charts/keepup/templates/seecret.yaml index 683c276..25107a1 100644 --- a/charts/keepup/templates/seecret.yaml +++ b/charts/keepup/templates/seecret.yaml @@ -2,7 +2,6 @@ kind: Secret apiVersion: v1 metadata: name: keepup-seecret - namespace: keepup data: API_TOKEN: {{ .Values.apiToken | b64enc }} type: Opaque diff --git a/charts/keepup/templates/service.yaml b/charts/keepup/templates/service.yaml index 31f3ec5..03d7e9f 100644 --- a/charts/keepup/templates/service.yaml +++ b/charts/keepup/templates/service.yaml @@ -14,4 +14,4 @@ spec: - name: http protocol: TCP port: 80 - targetPort: 80 + targetPort: http diff --git a/charts/keepup/values.yaml b/charts/keepup/values.yaml index ec8f9e2..544fe58 100644 --- a/charts/keepup/values.yaml +++ b/charts/keepup/values.yaml @@ -1,14 +1,32 @@ replicas: 1 +podSecurityContext: + seccompProfile: + type: RuntimeDefault + main: image: "ghcr.io/code-tool/keepup" tag: "" pullPolicy: "IfNotPresent" + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] redis: enabled: true image: "redis:8.2.8" pullPolicy: "IfNotPresent" + # the official redis image's built-in non-root user + securityContext: + runAsNonRoot: true + runAsUser: 999 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] ingress: enabled: true @@ -50,6 +68,7 @@ tolerations: [] apiToken: '' appEnv: prod +listenPort: '9101' redisAddr: 127.0.0.1 redisPort: '6379' redisDbNo: '7' diff --git a/docker/Dockerfile b/docker/Dockerfile index 3491fcd..36d5362 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ FROM golang:1.25.12-trixie AS builder -ARG BUILD_VERSION='v1.6.0' +ARG BUILD_VERSION='v1.7.0' ENV LISTEN_PORT=9101 WORKDIR /opt/keepup/ COPY go.mod ./ @@ -9,13 +9,17 @@ RUN go mod download RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.buildVersion=${BUILD_VERSION}" -a -installsuffix cgo -o keepup src/main.go FROM debian:trixie +ENV LISTEN_PORT=9101 WORKDIR /opt/keepup/ RUN apt-get update \ && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ - && rm -rf /var/cache/apt/archives/* -COPY --from=builder /opt/keepup/keepup . + && rm -rf /var/cache/apt/archives/* \ + && groupadd --system --gid 1000 keepup \ + && useradd --system --uid 1000 --gid keepup --no-create-home keepup +COPY --from=builder --chown=keepup:keepup /opt/keepup/keepup . +USER 1000:1000 EXPOSE ${LISTEN_PORT} CMD ["/opt/keepup/keepup"] diff --git a/go.mod b/go.mod index 274f777..3de5736 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,24 @@ module keepup -go 1.25 +go 1.25.0 require ( + github.com/alicebob/miniredis/v2 v2.38.0 github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 - github.com/prometheus/client_golang v1.23.2 - github.com/redis/go-redis/v9 v9.17.2 - github.com/golang/protobuf v1.5.4 + github.com/prometheus/client_golang v1.24.1 + github.com/redis/go-redis/v9 v9.22.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.19.2 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/sys v0.40.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/sys v0.47.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/go.sum b/go.sum index 25e1d73..4b936a3 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alicebob/miniredis/v2 v2.38.0 h1:nZAzCR+Lj+Vxk4ZXzm2NuKq2O33RXj1XxJ2e2uP9jiw= +github.com/alicebob/miniredis/v2 v2.38.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -6,53 +8,49 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= -github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= -github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/config/config.go b/src/config/config.go index 721a1a8..918942f 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -9,13 +9,13 @@ import ( ) type Config struct { - APP_ENV string - API_TOKEN string - LISTEN_PORT string - REDIS_ADDR string - REDIS_PORT string - REDIS_DBNO string - TTL_SECONDS string + APP_ENV string `env:"APP_ENV"` + API_TOKEN string `env:"API_TOKEN"` + LISTEN_PORT string `env:"LISTEN_PORT"` + REDIS_ADDR string `env:"REDIS_ADDR"` + REDIS_PORT string `env:"REDIS_PORT"` + REDIS_DBNO string `env:"REDIS_DBNO"` + TTL_SECONDS string `env:"TTL_SECONDS"` } var config *Config @@ -26,26 +26,23 @@ func GetConfig() Config { func loadEnvFile() { log.Println("Loading .env file.") - err := godotenv.Load(".env") - if err != nil { + if err := godotenv.Load(".env"); err != nil { panic("Error loading .env file.") } } func init() { - config = &Config{} - _, found := os.LookupEnv("APP_ENV") - if !found { + if _, found := os.LookupEnv("APP_ENV"); !found { loadEnvFile() } - _, found = os.LookupEnv("LISTEN_PORT") - if !found { - os.Setenv("LISTEN_PORT", "80") + if _, found := os.LookupEnv("LISTEN_PORT"); !found { + os.Setenv("LISTEN_PORT", "9101") } + + config = &Config{} refl := reflect.ValueOf(config).Elem() - numFields := refl.NumField() - for i := 0; i < numFields; i++ { - envName := refl.Type().Field(i).Name + for i := 0; i < refl.NumField(); i++ { + envName := refl.Type().Field(i).Tag.Get("env") envVal, found := os.LookupEnv(envName) if !found { panic("Environment [" + envName + "] not found.") diff --git a/src/handler/helmscrape.go b/src/handler/helmscrape.go index 4f821f9..9a19585 100644 --- a/src/handler/helmscrape.go +++ b/src/handler/helmscrape.go @@ -70,27 +70,55 @@ func (c *KubernetesClusters) RetrieveCluster(id uuid.UUID, ctx context.Context, } func (c *KubernetesClusters) ScanClusters(ctx context.Context, con *redis.Client) (KubernetesClusters, error) { - var clusters = KubernetesClusters{ + clusters := KubernetesClusters{ Items: make(map[uuid.UUID]KubernetesCluster), } + var uids []uuid.UUID + var keys []string iter := con.Scan(ctx, 0, "*", 0).Iterator() for iter.Next(ctx) { uid, err := uuid.Parse(iter.Val()) if err != nil { if iter.Val() != "eol_cache:all_packages" { log.Printf("Cannot parse UUID: %s, %v", iter.Val(), err) - continue } continue } - - clusters.Items[uid], _ = c.RetrieveCluster(uid, ctx, con) + uids = append(uids, uid) + keys = append(keys, iter.Val()) } - if err := iter.Err(); err != nil { log.Printf("Error scanning clusters: %v", err) + return clusters, err + } + if len(keys) == 0 { + return clusters, nil + } + + values, err := con.MGet(ctx, keys...).Result() + if err != nil { + log.Printf("Error fetching clusters: %v", err) + return clusters, err + } + for i, val := range values { + if val == nil { + // Key expired between SCAN and MGET. + continue + } + str, ok := val.(string) + if !ok { + log.Printf("Unexpected value type for key %s", keys[i]) + continue + } + var cluster KubernetesCluster + if err := json.Unmarshal([]byte(str), &cluster); err != nil { + log.Printf("Can't unmarshal cluster %s: %v", keys[i], err) + continue + } + clusters.Items[uids[i]] = cluster } + return clusters, nil } diff --git a/src/handler/helmscrape_test.go b/src/handler/helmscrape_test.go new file mode 100644 index 0000000..6efd908 --- /dev/null +++ b/src/handler/helmscrape_test.go @@ -0,0 +1,130 @@ +package handler + +import ( + "context" + "testing" + + "github.com/google/uuid" +) + +func TestUUIDFromClusterName_Deterministic(t *testing.T) { + a := UUIDFromClusterName("minikube") + b := UUIDFromClusterName("minikube") + if a != b { + t.Fatalf("expected same input to produce the same UUID, got %s and %s", a, b) + } +} + +func TestUUIDFromClusterName_DifferentInputsDiffer(t *testing.T) { + a := UUIDFromClusterName("cluster-a") + b := UUIDFromClusterName("cluster-b") + if a == b { + t.Fatalf("expected different cluster names to produce different UUIDs, both were %s", a) + } +} + +func TestClusterInsertAndRetrieve(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &KubernetesClusters{Items: make(map[uuid.UUID]KubernetesCluster)} + + id, err := c.InsertClusterData(KubernetesCluster{ClusterName: "minikube", KubeVersion: "1.30"}, ctx, con, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + stored, err := c.RetrieveCluster(id, ctx, con) + if err != nil { + t.Fatalf("unexpected error retrieving inserted cluster: %v", err) + } + if stored.ClusterName != "minikube" { + t.Errorf("expected cluster_name %q, got %q", "minikube", stored.ClusterName) + } +} + +func TestClusterRetrieve_UnmarshalFailureOnCorruptData(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &KubernetesClusters{Items: make(map[uuid.UUID]KubernetesCluster)} + + id := uuid.New() + if err := con.Set(ctx, id.String(), "not-json", 0).Err(); err != nil { + t.Fatalf("failed to seed corrupt value: %v", err) + } + + if _, err := c.RetrieveCluster(id, ctx, con); err != ErrClusterMarshalFailed { + t.Fatalf("expected ErrClusterMarshalFailed for corrupt data, got %v", err) + } +} + +func TestClusterScan_ReturnsAllInsertedClusters(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &KubernetesClusters{Items: make(map[uuid.UUID]KubernetesCluster)} + + first, err := c.InsertClusterData(KubernetesCluster{ClusterName: "cluster-a"}, ctx, con, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + second, err := c.InsertClusterData(KubernetesCluster{ClusterName: "cluster-b"}, ctx, con, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + result, err := c.ScanClusters(ctx, con) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Items) != 2 { + t.Fatalf("expected 2 items, got %d", len(result.Items)) + } + if _, ok := result.Items[first]; !ok { + t.Errorf("expected scan to contain first inserted cluster %s", first) + } + if _, ok := result.Items[second]; !ok { + t.Errorf("expected scan to contain second inserted cluster %s", second) + } +} + +func TestClusterScan_SkipsCorruptAndNonUUIDEntries(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &KubernetesClusters{Items: make(map[uuid.UUID]KubernetesCluster)} + + good, err := c.InsertClusterData(KubernetesCluster{ClusterName: "cluster-a"}, ctx, con, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + corrupt := uuid.New() + if err := con.Set(ctx, corrupt.String(), "not-json", 0).Err(); err != nil { + t.Fatalf("failed to seed corrupt value: %v", err) + } + if err := con.Set(ctx, "eol_cache:all_packages", "{}", 0).Err(); err != nil { + t.Fatalf("failed to seed eol cache key: %v", err) + } + + result, err := c.ScanClusters(ctx, con) + if err != nil { + t.Fatalf("unexpected error from scan: %v", err) + } + if len(result.Items) != 1 { + t.Fatalf("expected scan to skip corrupt/non-uuid entries and return 1 item, got %d", len(result.Items)) + } + if _, ok := result.Items[good]; !ok { + t.Errorf("expected scan to still contain the valid cluster %s", good) + } +} + +func TestClusterScan_EmptyDatabase(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &KubernetesClusters{Items: make(map[uuid.UUID]KubernetesCluster)} + + result, err := c.ScanClusters(ctx, con) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Items) != 0 { + t.Fatalf("expected no items in an empty database, got %d", len(result.Items)) + } +} diff --git a/src/handler/middleware.go b/src/handler/middleware.go index 4e3d663..83b8734 100644 --- a/src/handler/middleware.go +++ b/src/handler/middleware.go @@ -34,22 +34,6 @@ type IDDocumentPackage struct { ID uuid.UUID `json:"id"` } -type OsReleasesMiddleware struct { - OsReleases *OsReleases - Client *redis.Client - Context context.Context - ApiToken string - TTL int -} - -type ReleaseDocument struct { - OsRelease OsRelease `json:"release"` -} - -type IDDocument struct { - ID uuid.UUID `json:"id"` -} - type KubernetesClusterMiddleware struct { Clusters *KubernetesClusters Client *redis.Client @@ -66,100 +50,39 @@ type IDClusterDocument struct { ID uuid.UUID `json:"id"` } -func (s *OsReleasesMiddleware) HandleOsRelease(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get("x-api-token") - if subtle.ConstantTimeCompare([]byte(token), []byte(s.ApiToken)) != 1 { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusForbidden) - io.WriteString(w, "FORBIDDEN") - return - } else { - switch strings.ToUpper(r.Method) { - case "GET": - w.Header().Set("Content-Type", "application/json") - s.handleGetByID(w, r) - case "PUT": - w.Header().Set("Content-Type", "application/json") - s.handleInsert(w, r) - } - } -} - -func (s *OsReleasesMiddleware) handleInsert(w http.ResponseWriter, r *http.Request) { - var req ReleaseDocument - var res IDDocument - r.Body = http.MaxBytesReader(w, r.Body, 1<<20) - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - id, e := s.OsReleases.Insert(req.OsRelease, s.Context, s.Client, s.TTL) - if e != nil { - http.Error(w, e.Error(), http.StatusBadRequest) - return - } - res = IDDocument{ID: id} - err = json.NewEncoder(w).Encode(res) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - -} - -func (s *OsReleasesMiddleware) handleGetByID(w http.ResponseWriter, r *http.Request) { - var req IDDocument - r.Body = http.MaxBytesReader(w, r.Body, 1<<20) - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - rel, err := s.OsReleases.Retrieve(req.ID, s.Context, s.Client) - if err == ErrIDNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - res := ReleaseDocument{OsRelease: rel} - err = json.NewEncoder(w).Encode(res) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *OsReleasesMiddleware) HandlePuppet(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Content-Type", "application/octet-stream") - w.Header().Add("Content-Transfer-Encoding", "binary") - w.Header().Add("Cache-Control", "private") - w.Header().Add("Content-Disposition", "attachment; filename=puppet.tar.bz2") - http.ServeFile(w, r, "./static/puppet.tar.bz2") -} - -func (s *OsReleasesMiddleware) HandleAnsible(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Content-Type", "application/octet-stream") - w.Header().Add("Content-Transfer-Encoding", "binary") - w.Header().Add("Cache-Control", "private") - w.Header().Add("Content-Disposition", "attachment; filename=ansible.tar.bz2") - http.ServeFile(w, r, "./static/ansible.tar.bz2") -} - -func (s *OsReleasesMiddleware) HealthCheck(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - io.WriteString(w, "OK") -} - func FlushBufferOnShutdown(shutdownWaiter *sync.WaitGroup) { // TODO: cleanup logic shutdownWaiter.Done() } +func methodNotAllowedResponse(w http.ResponseWriter) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusMethodNotAllowed) + io.WriteString(w, "METHOD NOT ALLOWED") +} + +// withAuth checks the x-api-token header against apiToken, then dispatches +// to the handler registered for the request method in methods. Every +// dispatched handler responds with application/json. +func withAuth(apiToken string, methods map[string]http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("x-api-token") + if subtle.ConstantTimeCompare([]byte(token), []byte(apiToken)) != 1 { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, "FORBIDDEN") + return + } + handlerFunc, ok := methods[strings.ToUpper(r.Method)] + if !ok { + methodNotAllowedResponse(w) + return + } + w.Header().Set("Content-Type", "application/json") + handlerFunc(w, r) + } +} + func (p *PackageVersionsHandler) handleInsertPackages(w http.ResponseWriter, r *http.Request) { var req PackageDocument var res IDDocumentPackage @@ -242,46 +165,23 @@ func (p *PackageVersionsHandler) handleGetPackages(w http.ResponseWriter, r *htt } } -func (s *PackageVersionsHandler) HandlePackage(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get("x-api-token") - if subtle.ConstantTimeCompare([]byte(token), []byte(s.ApiToken)) != 1 { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusForbidden) - io.WriteString(w, "FORBIDDEN") - return - } else { - switch strings.ToUpper(r.Method) { - case "GET": - w.Header().Set("Content-Type", "application/json") - s.handleGetPackages(w, r) - case "PUT": - w.Header().Set("Content-Type", "application/json") - s.handleInsertPackages(w, r) - } - } +func (s *PackageVersionsHandler) Handler() http.HandlerFunc { + return withAuth(s.ApiToken, map[string]http.HandlerFunc{ + "GET": s.handleGetPackages, + "PUT": s.handleInsertPackages, + }) } -// TODO -// Simplife new endpoint handling logic. Maybe define common handler for all endpoints. -func (s *KubernetesClusterMiddleware) HandleKubernetesCluster(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get("x-api-token") - if subtle.ConstantTimeCompare([]byte(token), []byte(s.ApiToken)) != 1 { - http.Error(w, "FORBIDDEN", http.StatusForbidden) - return - } - - switch strings.ToUpper(r.Method) { - case "GET": - w.Header().Set("Content-Type", "application/json") - s.handleGetClusterByID(w, r) - case "PUT": - w.Header().Set("Content-Type", "application/json") - s.handleInsertCluster(w, r) - } +func (s *KubernetesClusterMiddleware) Handler() http.HandlerFunc { + return withAuth(s.ApiToken, map[string]http.HandlerFunc{ + "GET": s.handleGetClusterByID, + "PUT": s.handleInsertCluster, + }) } func (s *KubernetesClusterMiddleware) handleInsertCluster(w http.ResponseWriter, r *http.Request) { var cluster KubernetesCluster + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) body, err := io.ReadAll(r.Body) if err != nil { log.Println("Failed to read request body:", err) @@ -327,7 +227,6 @@ func (s *KubernetesClusterMiddleware) handleGetClusterByID(w http.ResponseWriter } res := ClusterDocument{Cluster: cluster} - w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(res); err != nil { http.Error(w, "Failed to encode response", http.StatusInternalServerError) return diff --git a/src/handler/osrelease.go b/src/handler/osrelease.go deleted file mode 100644 index 48cdc31..0000000 --- a/src/handler/osrelease.go +++ /dev/null @@ -1,89 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log" - "time" - - "github.com/google/uuid" - "github.com/redis/go-redis/v9" -) - -type OsRelease struct { - ID uuid.UUID `json:"id"` // /sys/devices/virtual/dmi/id/product_uuid - OsId string `json:"os_id"` // ID=debian - VersionCodename string `json:"version_codename"` // VERSION_CODENAME=bullseye - Version string `json:"version"` // VERSION="11 (bullseye)" - VersionId string `json:"version_id"` // VERSION_ID="11" - DataCenter string `json:"data_center"` // team or dc name - HostIP string `json:"host_ip"` // host ip - UpdatedAt string `json:"updated_at"` // unix timestamp -} - -type OsReleases struct { - Items map[uuid.UUID]OsRelease -} - -var ( - ErrInsertFailed = errors.New("Insert failed") - ErrMarshalFailed = errors.New("Marshal failed") - ErrIDNotFound = errors.New("Id not found") - ErrNoKeysFound = errors.New("No keys found") -) - -func (c *OsReleases) Insert(rel OsRelease, ctx context.Context, con *redis.Client, ttl int) (uuid.UUID, error) { - rel.ID = UUIDFromDcAndIP(rel.DataCenter, rel.HostIP) - rel.UpdatedAt = fmt.Sprint(time.Now().Unix()) - srt, err := json.Marshal(rel) - if err != nil { - return rel.ID, ErrMarshalFailed - } - result, err := con.Set(ctx, fmt.Sprint(rel.ID), srt, time.Duration(ttl)*time.Second).Result() - if err != nil { - return rel.ID, ErrInsertFailed - } - log.Printf("Creating %s: %s", rel.ID, result) - return rel.ID, nil -} - -func (c *OsReleases) Retrieve(id uuid.UUID, ctx context.Context, con *redis.Client) (OsRelease, error) { - result, err := con.Get(ctx, fmt.Sprint(id)).Result() - if err != nil { - return OsRelease{}, ErrIDNotFound - } - rel := OsRelease{} - json.Unmarshal([]byte(result), &rel) - return rel, nil -} - -func (c *OsReleases) Scan(ctx context.Context, con *redis.Client) (OsReleases, error) { - var rels = OsReleases{ - Items: make(map[uuid.UUID]OsRelease), - } - iter := con.Scan(ctx, 0, "*", 0).Iterator() - for iter.Next(ctx) { - uid, err := uuid.Parse(iter.Val()) - if err != nil { - if iter.Val() != "eol_cache:all_packages" { - log.Printf("Can't parse uuid: %s, %v", iter.Val(), err) - continue - } else { - continue - } - } else { - rels.Items[uid], _ = c.Retrieve(uid, ctx, con) - } - } - if err := iter.Err(); err != nil { - log.Printf("Error scanning Redis keys: %v", err) - return rels, err - } - return rels, nil -} - -func UUIDFromDcAndIP(dc string, ip string) uuid.UUID { - return uuid.NewSHA1(uuid.NameSpaceDNS, []byte(fmt.Sprintf("%s-%s", dc, ip))) -} diff --git a/src/handler/packageversions.go b/src/handler/packageversions.go index 80caa22..926b7b0 100644 --- a/src/handler/packageversions.go +++ b/src/handler/packageversions.go @@ -142,18 +142,52 @@ func (c *PackageVersionss) Retrieve(id uuid.UUID, ctx context.Context, con *redi } func (c *PackageVersionss) Scan(ctx context.Context, con *redis.Client) (PackageVersionss, error) { - var pkgs = PackageVersionss{ + pkgs := PackageVersionss{ Items: make(map[uuid.UUID]PackageVersions), } + var uids []uuid.UUID + var keys []string iter := con.Scan(ctx, 0, "*", 0).Iterator() for iter.Next(ctx) { uid, err := uuid.Parse(iter.Val()) if err != nil { continue } - pkgs.Items[uid], _ = c.Retrieve(uid, ctx, con) + uids = append(uids, uid) + keys = append(keys, iter.Val()) } + if err := iter.Err(); err != nil { + log.Printf("Error scanning Redis keys: %v", err) + return pkgs, err + } + if len(keys) == 0 { + return pkgs, nil + } + + values, err := con.MGet(ctx, keys...).Result() + if err != nil { + log.Printf("Error fetching Redis keys: %v", err) + return pkgs, err + } + for i, val := range values { + if val == nil { + // Key expired between SCAN and MGET. + continue + } + str, ok := val.(string) + if !ok { + log.Printf("Unexpected value type for key %s", keys[i]) + continue + } + var pkg PackageVersions + if err := json.Unmarshal([]byte(str), &pkg); err != nil { + log.Printf("Can't unmarshal package %s: %v", keys[i], err) + continue + } + pkgs.Items[uids[i]] = pkg + } + return pkgs, nil } @@ -293,18 +327,24 @@ func getEOLData(ctx context.Context, con *redis.Client, packageName string) ([]E } func isVersionExpired(current, newest string) bool { - parseVersion := func(version string) (int, int) { + parseVersion := func(label, version string) (int, int) { segments := strings.Split(version, ".") - major, _ := strconv.Atoi(segments[0]) + major, err := strconv.Atoi(segments[0]) + if err != nil { + log.Printf("Can't parse %s version major segment %q: %v", label, version, err) + } minor := 0 if len(segments) > 1 { - minor, _ = strconv.Atoi(segments[1]) + minor, err = strconv.Atoi(segments[1]) + if err != nil { + log.Printf("Can't parse %s version minor segment %q: %v", label, version, err) + } } return major, minor } - currentMajor, currentMinor := parseVersion(current) - newestMajor, newestMinor := parseVersion(newest) + currentMajor, currentMinor := parseVersion("current", current) + newestMajor, newestMinor := parseVersion("newest", newest) if currentMajor < newestMajor { return true diff --git a/src/handler/packageversions_test.go b/src/handler/packageversions_test.go new file mode 100644 index 0000000..e478f93 --- /dev/null +++ b/src/handler/packageversions_test.go @@ -0,0 +1,131 @@ +package handler + +import ( + "context" + "testing" + + "github.com/google/uuid" +) + +func noopQuery(name string) (string, string, error) { + return "unknown", "", nil +} + +func TestUUIDFromDcAndIPPackage_Deterministic(t *testing.T) { + a := UUIDFromDcAndIPPackage("dc1", "10.0.0.1") + b := UUIDFromDcAndIPPackage("dc1", "10.0.0.1") + if a != b { + t.Fatalf("expected same input to produce the same UUID, got %s and %s", a, b) + } +} + +func TestPackageVersionsInsertAndRetrieve(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &PackageVersionss{Items: make(map[uuid.UUID]PackageVersions)} + + pkg := PackageVersions{ + DataCenterPkg: "dc1", + HostIPPkg: "10.0.0.1", + Packages: map[string]PackageDetail{ + "redis": {CurrentVersion: "7.0.15"}, + }, + } + + id, err := c.Insert(pkg, ctx, con, noopQuery, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + stored, err := c.Retrieve(id, ctx, con) + if err != nil { + t.Fatalf("unexpected error retrieving inserted package: %v", err) + } + if _, ok := stored.Packages["redis"]; !ok { + t.Errorf("expected stored packages to contain %q", "redis") + } +} + +func TestPackageVersionsRetrieve_UnmarshalFailureOnCorruptData(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &PackageVersionss{Items: make(map[uuid.UUID]PackageVersions)} + + id := uuid.New() + if err := con.Set(ctx, id.String(), "not-json", 0).Err(); err != nil { + t.Fatalf("failed to seed corrupt value: %v", err) + } + + if _, err := c.Retrieve(id, ctx, con); err != ErrMarshalFailedPackage { + t.Fatalf("expected ErrMarshalFailedPackage for corrupt data, got %v", err) + } +} + +func TestPackageVersionsScan_ReturnsAllInsertedPackages(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &PackageVersionss{Items: make(map[uuid.UUID]PackageVersions)} + + first, err := c.Insert(PackageVersions{DataCenterPkg: "dc1", HostIPPkg: "10.0.0.1"}, ctx, con, noopQuery, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + second, err := c.Insert(PackageVersions{DataCenterPkg: "dc1", HostIPPkg: "10.0.0.2"}, ctx, con, noopQuery, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + result, err := c.Scan(ctx, con) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Items) != 2 { + t.Fatalf("expected 2 items, got %d", len(result.Items)) + } + if _, ok := result.Items[first]; !ok { + t.Errorf("expected scan to contain first inserted package %s", first) + } + if _, ok := result.Items[second]; !ok { + t.Errorf("expected scan to contain second inserted package %s", second) + } +} + +func TestPackageVersionsScan_SkipsCorruptEntryWithoutFailingWholeScan(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &PackageVersionss{Items: make(map[uuid.UUID]PackageVersions)} + + good, err := c.Insert(PackageVersions{DataCenterPkg: "dc1", HostIPPkg: "10.0.0.1"}, ctx, con, noopQuery, 60) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + corrupt := uuid.New() + if err := con.Set(ctx, corrupt.String(), "not-json", 0).Err(); err != nil { + t.Fatalf("failed to seed corrupt value: %v", err) + } + + result, err := c.Scan(ctx, con) + if err != nil { + t.Fatalf("unexpected error from scan: %v", err) + } + if len(result.Items) != 1 { + t.Fatalf("expected scan to skip the corrupt entry and return 1 item, got %d", len(result.Items)) + } + if _, ok := result.Items[good]; !ok { + t.Errorf("expected scan to still contain the valid package %s", good) + } +} + +func TestPackageVersionsScan_EmptyDatabase(t *testing.T) { + ctx := context.Background() + con := newTestClient(t) + c := &PackageVersionss{Items: make(map[uuid.UUID]PackageVersions)} + + result, err := c.Scan(ctx, con) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Items) != 0 { + t.Fatalf("expected no items in an empty database, got %d", len(result.Items)) + } +} diff --git a/src/handler/testhelpers_test.go b/src/handler/testhelpers_test.go new file mode 100644 index 0000000..3eacab0 --- /dev/null +++ b/src/handler/testhelpers_test.go @@ -0,0 +1,14 @@ +package handler + +import ( + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func newTestClient(t *testing.T) *redis.Client { + t.Helper() + mr := miniredis.RunT(t) + return redis.NewClient(&redis.Options{Addr: mr.Addr()}) +} diff --git a/src/main.go b/src/main.go index c32c868..86c4186 100644 --- a/src/main.go +++ b/src/main.go @@ -26,7 +26,6 @@ var ( server *http.Server shutdownWaiter sync.WaitGroup PackageHandler *handler.PackageVersionsHandler - osReleaseHandler *handler.OsReleasesMiddleware kubeClusterHandler *handler.KubernetesClusterMiddleware buildVersion string ) @@ -48,16 +47,6 @@ func main() { log.Fatalf("Can't configure TTL_SECONDS: %v", err) } - osReleaseHandler = &handler.OsReleasesMiddleware{ - OsReleases: &handler.OsReleases{ - Items: make(map[uuid.UUID]handler.OsRelease), - }, - Context: ctx, - Client: con, - ApiToken: config.GetConfig().API_TOKEN, - TTL: ttlSeconds, - } - PackageHandler = &handler.PackageVersionsHandler{ PackageVersions: &handler.PackageVersionss{ Items: make(map[uuid.UUID]handler.PackageVersions), @@ -78,9 +67,6 @@ func main() { TTL: ttlSeconds, } - osReleaseCollector := metrics.OsReleaseCollector{ - RelInfo: osReleaseHandler, - } packageCollector := metrics.PackageVersionsCollector{ PackageInfo: PackageHandler, } @@ -90,7 +76,6 @@ func main() { } prometheus.MustRegister(packageCollector) - prometheus.MustRegister(osReleaseCollector) prometheus.MustRegister(HelmCollector) shutdownWaiter.Add(1) @@ -118,17 +103,14 @@ func configureServer() { func initSignalHandler() { sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL) + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT) go handleSignal(server, sigChan) } func initRouting() { http.Handle("/metrics", promhttp.Handler()) - //http.HandleFunc("/static/puppet.tar.bz2", osReleaseHandler.HandlePuppet) - //http.HandleFunc("/static/ansible.tar.bz2", osReleaseHandler.HandleAnsible) - http.HandleFunc("/os-release", osReleaseHandler.HandleOsRelease) - http.HandleFunc("/package-version", PackageHandler.HandlePackage) - http.HandleFunc("/helm-cluster", kubeClusterHandler.HandleKubernetesCluster) + http.HandleFunc("/package-version", PackageHandler.Handler()) + http.HandleFunc("/helm-cluster", kubeClusterHandler.Handler()) http.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte("OK")) diff --git a/src/metrics/helmmetrics.go b/src/metrics/helmmetrics.go index a9c9f67..77e6688 100644 --- a/src/metrics/helmmetrics.go +++ b/src/metrics/helmmetrics.go @@ -3,6 +3,7 @@ package metrics import ( "fmt" "keepup/src/handler" + "log" "github.com/prometheus/client_golang/prometheus" ) @@ -42,7 +43,11 @@ func (kc KubernetesClusterCollector) Describe(ch chan<- *prometheus.Desc) { func (kc KubernetesClusterCollector) Collect(ch chan<- prometheus.Metric) { - clusters, _ := kc.ClusterInfo.Clusters.ScanClusters(kc.ClusterInfo.Context, kc.ClusterInfo.Client) + clusters, err := kc.ClusterInfo.Clusters.ScanClusters(kc.ClusterInfo.Context, kc.ClusterInfo.Client) + if err != nil { + log.Printf("Failed to scan clusters: %v", err) + return + } for id, cluster := range clusters.Items { diff --git a/src/metrics/metrics.go b/src/metrics/metrics.go deleted file mode 100644 index d60f12e..0000000 --- a/src/metrics/metrics.go +++ /dev/null @@ -1,62 +0,0 @@ -package metrics - -import ( - "fmt" - "keepup/src/handler" - - "github.com/prometheus/client_golang/prometheus" -) - -var ( - ID = "id" - OsId = "os_id" - OsVersionCodename = "version_codename" - OsVersion = "version" - OsVersionId = "version_id" - DataCenter = "data_center" - HostIP = "host_ip" - osReleaseMetricValue = float64(1) - osReleseMetricDesc = prometheus.NewDesc( - "os_release_info", - "count of OS by version", - []string{ - ID, - OsId, - OsVersionCodename, - OsVersion, - OsVersionId, - DataCenter, - HostIP, - }, nil, - ) -) - -type OsReleaseCollector struct { - RelInfo *handler.OsReleasesMiddleware -} - -func (cc OsReleaseCollector) Describe(ch chan<- *prometheus.Desc) { - prometheus.DescribeByCollect(cc, ch) -} - -func (cc OsReleaseCollector) Collect(ch chan<- prometheus.Metric) { - rels, _ := cc.RelInfo.OsReleases.Scan(cc.RelInfo.Context, cc.RelInfo.Client) - - for id, rel := range rels.Items { - if rel.Version == "" { - continue - } - ch <- prometheus.MustNewConstMetric( - osReleseMetricDesc, - prometheus.CounterValue, - osReleaseMetricValue, - fmt.Sprint(id), - rel.OsId, - rel.VersionCodename, - rel.Version, - rel.VersionId, - rel.DataCenter, - rel.HostIP, - ) - } -} diff --git a/tests/end-to-end/example-002.json b/tests/end-to-end/example-002.json deleted file mode 100644 index fe1ec5a..0000000 --- a/tests/end-to-end/example-002.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "release": { - "os_id": "debian", - "version_codename": "bullseye", - "version": "11 (bullseye)", - "version_id": "11", - "data_center": "qqq", - "host_ip": "5.45.79.30" - } -} diff --git a/tests/end-to-end/run.sh b/tests/end-to-end/run.sh index 2484068..2d107b5 100755 --- a/tests/end-to-end/run.sh +++ b/tests/end-to-end/run.sh @@ -5,14 +5,11 @@ set -e echo "=== PUT test data ===" curl -XPUT -H "x-api-token: secret" http://127.0.0.1:9101/package-version -d @example-001.json -curl -XPUT -H "x-api-token: secret" http://127.0.0.1:9101/os-release -d @example-002.json curl -XPUT -H "x-api-token: secret" http://127.0.0.1:9101/helm-cluster -d @example-003.json echo "=== GET test data ===" curl -X GET -H "x-api-token: secret" -s http://127.0.0.1:9101/package-version -d '{"id":"91015d87-2c51-5601-b337-1414f2b5496a"}' | grep debian -curl -X GET -H "x-api-token: secret" -s http://127.0.0.1:9101/os-release -d '{"id":"8b00021e-af61-546e-a0c1-1038bc422d39"}' | grep bullseye curl -X GET -H "x-api-token: secret" -s http://127.0.0.1:9101/helm-cluster -d '{"id":"688c14fe-9b83-5887-ba6c-f4fa310adc63"}' | grep minikube -curl -X GET -s http://127.0.0.1:9101/metrics | grep 'os' curl -X GET -s http://127.0.0.1:9101/metrics | grep 'package_version' curl -X GET -s http://127.0.0.1:9101/metrics | grep 'kubernetes_cluster'