Merge pull request #2 from ebpfdev/prometheus-endpoint

Prometheus endpoint

Author: klpx

Dockerfile

@@ -22,5 +22,7 @@ LABEL org.opencontainers.image.source=https://github.com/ebpfdev/dev-agent
 WORKDIR /app
 COPY --from=0 /build/dev-agent ./
 
-CMD ["/app/dev-agent"]
+EXPOSE 8080
+
+ENTRYPOINT ["/app/dev-agent"]
 

README.md

@@ -6,18 +6,66 @@ This agent provides access to system's eBPF-programs and maps to perform remote
 ## GraphQL server
 
 ```shell
-sudo ./phydev server
+sudo ./phydev server [--help]
 ```
 
 GraphQL interface: [http://localhost:8080/](http://localhost:8080/)
 Schema: [pkg/graph/schema.graphqls](pkg/graph/schema.graphqls)
 
 ![GraphQL interface example](docs/graphql-example.png)
 
+### Prometheus endpoint
 
+Metrics scrape endpoint for Prometheus: [http://localhost:8080/metrics](http://localhost:8080/metrics)
+
+* program metrics:
+  * `devagent_ebpf_prog_count` - number of eBPF programs by `type`
+  * runtime metrics only available with `sysctl -w kernel.bpf_stats_enabled=1`:
+    * `devagent_ebpf_prog_run_count` - number of times an eBPF program has been run (by `id`, `name`, `tag`, `type`)
+    * `devagent_ebpf_prog_run_time` - total time spent running eBPF programs (by `id`, `name`, `tag`, `type`)
+* map metrics:
+  * `devagent_ebpf_map_count` - number of eBPF maps by `type`
+  * if map export is configured (see below):
+    * `devagent_ebpf_map_entry_count` - number of entries in an eBPF map (by `id`, `name`, `type`)
+    * `devagent_ebpf_map_entry_value` - value of an eBPF map entry (by `key`, `cpu`, `id`, `name`, `type`)
+
+You can find example of Grafana dashboard in [grafana-ebpf-dashboard.json](./grafana-ebpf-dashboard.json):
+![grafana dashboard with program metrics](docs/grafana-ebpf.png)
+
+#### Configuring map export
+
+As an example, I'm running this [bpftrace](https://github.com/iovisor/bpftrace) program:
+```shell
+sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @SYSCALLNUM[comm] = count(); }'
+```
+
+You could see the name of created map - `AT_SYSCALLNUM`, and the map content in [ebpf-explorer](https://github.com/ebpfdev/explorer):
+![exbpf explorer showing AT_SYSCALNUM page](docs/explorer-syscallnum.png)
+
+By default, dev-agent doesn't export map entries to Prometheus, as it may introduce some performance issues.
+
+Instead, you could set an option `--etm -:AT_SYSCALLNUM:string` when running server, which will suggest agent which map entries to expose in /metrics.
+
+For this HASH_PER_CPU map, it will export 2 metrics:
+```text
+# HELP devagent_ebpf_map_entry_count Number of entries in an eBPF map
+# TYPE devagent_ebpf_map_entry_count gauge
+devagent_ebpf_map_entry_count{id="25",name="AT_SYSCALLNUM",type="PerCPUHash"} 764
+# HELP devagent_ebpf_map_entry_value Value of an eBPF map entry
+# TYPE devagent_ebpf_map_entry_value gauge
+devagent_ebpf_map_entry_value{cpu="0",id="25",key="(anacron)",name="AT_SYSCALLNUM",type="PerCPUHash"} 0
+devagent_ebpf_map_entry_value{cpu="0",id="25",key="(fprintd)",name="AT_SYSCALLNUM",type="PerCPUHash"} 0
+```
+
+This is how it may look in Grafana (top 10 processes doing most of syscalls):
+![Grafana showing top 10 processes doing most of syscalls](docs/grafana-syscallnum.png)
+
+Run `./phydev server --help` for more details on this flag.
 
 ## CLI commands
 
+These are just for debugging purpose, use [bpftool](https://github.com/libbpf/bpftool) instead
+
 List loaded eBPF programs:
 
 ```shell
@@ -44,6 +92,13 @@ ID      Name    FD      Type    Flags   IsPinned        KeySize ValueSize
 63      open_at_args    29      Hash    0       false   8       128     1024
 ```
 
+## Docker
+
+Instead of `./phydev`, use docker command:
+```shell
+docker run -ti --rm --privileged -p 8080:8080 ghcr.io/ebpfdev/dev-agent:v0.0.1 /app/dev-agent server
+```
+
 # Development
 
 ## Build

cmd/dev-agent/commands/commands.go

@@ -12,7 +12,9 @@ import (
 func App() *cli.App {
 	logger := log.Logger.Level(zerolog.InfoLevel)
 	progsRepo := progs.NewWatcher(logger, 1*time.Second)
-	mapsRepo := maps.NewWatcher(logger, 1*time.Second)
+	mapsRepo := maps.NewWatcher(&maps.WatcherOpts{
+		RefreshInterval: 1 * time.Second,
+	}, logger)
 	progsCommands := &ProgsCommands{
 		ProgsRepo: progsRepo,
 	}
@@ -39,16 +41,38 @@ func App() *cli.App {
 				Name: "server",
 				Flags: []cli.Flag{
 					&cli.StringFlag{
-						Name:  "path-prefix",
-						Usage: "path prefix for the web ui to access the server",
-						Value: "/",
+						Name:     "path-prefix",
+						Category: "Server",
+						Usage:    "path prefix for the web ui to access the server",
+						Value:    "/",
 					},
 					&cli.BoolFlag{
 						Name:  "skip-welcome",
 						Usage: "skip welcome message",
 					},
+					&cli.MultiStringFlag{
+						Target: &cli.StringSliceFlag{
+							Name:     "entries-to-metrics",
+							Category: "Metrics",
+							Usage: "(experimental, api may change)\n\tConfigure which map entries should be exposed as metrics, " +
+								"in the format: id_start-id_end:metric_name_regexp:key_format.\n\t" +
+								"Example: '-:.+:string' to export any map with non-empty name while treating key as string.\n\t" +
+								"or '10-:.*:hex' to export any map after ID 10 with key represented in HEX format\n\t" +
+								"Available key formats: string, number, hex\n\t" +
+								"If a map matches multiple entries, the first one is used.",
+							Aliases: []string{"etm"},
+						},
+					},
 				},
 				Action: func(c *cli.Context) error {
+					for _, etm := range c.StringSlice("entries-to-metrics") {
+						etmConfig, err := maps.ParseMapExportConfiguration(etm)
+						if err != nil {
+							return err
+						}
+						mapsRepo.AddExportConfig(etmConfig)
+					}
+
 					return serverCommands.ServerStart(&ServerStartOptions{
 						PathPrefix:  c.String("path-prefix"),
 						SkipWelcome: c.Bool("skip-welcome"),

cmd/dev-agent/commands/server.go

@@ -8,6 +8,8 @@ import (
 	"github.com/ebpfdev/dev-agent/pkg/ebpf/progs"
 	"github.com/ebpfdev/dev-agent/pkg/graph"
 	"github.com/ebpfdev/dev-agent/pkg/graph/generated"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/cors"
 	"log"
 	"net/http"
@@ -33,9 +35,14 @@ func (sc *ServerCommands) ServerStart(options *ServerStartOptions) error {
 		port = defaultPort
 	}
 
+	registry := prometheus.NewRegistry()
+
 	sc.ProgsRepo.Run(context.Background())
 	sc.MapsRepo.Run(context.Background())
 
+	sc.ProgsRepo.RegisterMetrics(registry)
+	sc.MapsRepo.RegisterMetrics(registry)
+
 	resolver := &graph.Resolver{
 		ProgsRepository: sc.ProgsRepo,
 		MapsRepository:  sc.MapsRepo,
@@ -47,6 +54,11 @@ func (sc *ServerCommands) ServerStart(options *ServerStartOptions) error {
 
 	mux.Handle("/", playground.Handler("GraphQL playground", options.PathPrefix+"query"))
 	mux.Handle("/query", srv)
+	mux.Handle("/metrics", promhttp.HandlerFor(
+		registry,
+		promhttp.HandlerOpts{
+			EnableOpenMetrics: true,
+		}))
 
 	if !options.SkipWelcome {
 		log.Printf("connect to http://localhost:%s/ for GraphQL playground", port)

docs/explorer-syscallnum.png

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/99designs/gqlgen v0.17.31
 	github.com/cilium/ebpf v0.10.0
+	github.com/prometheus/client_golang v1.15.1
 	github.com/rs/cors v1.9.0
 	github.com/rs/zerolog v1.29.1
 	github.com/urfave/cli/v2 v2.25.3
@@ -13,17 +14,25 @@ require (
 
 require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/tools v0.6.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

docs/grafana-ebpf.png

@@ -7,6 +7,10 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cilium/ebpf v0.10.0 h1:nk5HPMeoBXtOzbkZBWym+ZWq1GIiHUsBFXxwewXAHLQ=
 github.com/cilium/ebpf v0.10.0/go.mod h1:DPiVdY/kT534dgc9ERmvP8mWA+9gvwgKfRvk4nNWnoE=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -19,6 +23,12 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -36,11 +46,21 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
+github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
 github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
@@ -62,6 +82,7 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -72,12 +93,17 @@ golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=