diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index cb6824180..5bae2eaa1 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -253,7 +253,11 @@ func setupTest(t *testing.T, ns string) *testContext { rdb := redis.NewClusterClient(&redis.ClusterOptions{ Addrs: []string{mr.Addr()}, }) - persistence := ateredis.NewPersistence(rdb) + persistence, err := ateredis.NewPersistence(rdb) + if err != nil { + mr.Close() + t.Fatalf("ateredis.NewPersistence: %v", err) + } // 2. Initialize Clientsets using global cfg k8sClient, err := kubernetes.NewForConfig(cfg) @@ -291,7 +295,12 @@ func setupTest(t *testing.T, ns string) *testContext { substrateInformerFactory.WaitForCacheSync(ctx.Done()) // 4. Initialize Service - wc := workercache.New(persistence, 5*time.Minute) + wc, err := workercache.New(persistence, 5*time.Minute) + if err != nil { + cancel() + mr.Close() + t.Fatalf("workercache.New: %v", err) + } if err := wc.Start(ctx); err != nil { cancel() mr.Close() diff --git a/cmd/ateapi/internal/store/ateredis/ateredis.go b/cmd/ateapi/internal/store/ateredis/ateredis.go index 025516a50..5dca999ed 100644 --- a/cmd/ateapi/internal/store/ateredis/ateredis.go +++ b/cmd/ateapi/internal/store/ateredis/ateredis.go @@ -56,6 +56,9 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) @@ -74,16 +77,25 @@ type redisClient interface { // Persistence is a service that stores information about applications in Redis. type Persistence struct { - rdb redisClient + rdb redisClient + metricWorkerPubsubMessages metric.Int64Counter } var _ store.Interface = (*Persistence)(nil) // NewPersistence creates a new Persistence. -func NewPersistence(redisClient *redis.ClusterClient) *Persistence { - return &Persistence{ - rdb: redisClient, +func NewPersistence(redisClient *redis.ClusterClient) (*Persistence, error) { + publishMessages, err := otel.Meter("ateredis").Int64Counter( + "persistence.worker.pubsub.messages", + metric.WithUnit("{message}"), + metric.WithDescription("Total worker-changes pub/sub PUBLISH attempts.")) + if err != nil { + return nil, fmt.Errorf("create persistence.worker.pubsub.messages counter: %w", err) } + return &Persistence{ + rdb: redisClient, + metricWorkerPubsubMessages: publishMessages, + }, nil } func actorDBKey(id string) string { @@ -128,6 +140,9 @@ func (s *Persistence) publishWorkerEvent(ctx context.Context, eventType store.Wo } if err := s.rdb.Publish(ctx, workerPubSubChannel, payload).Err(); err != nil { slog.ErrorContext(ctx, "worker event publish failed", slog.Any("err", err)) + s.metricWorkerPubsubMessages.Add(ctx, 1, metric.WithAttributes(attribute.String("error.type", "_OTHER"))) + } else { + s.metricWorkerPubsubMessages.Add(ctx, 1) } } diff --git a/cmd/ateapi/internal/store/ateredis/ateredis_test.go b/cmd/ateapi/internal/store/ateredis/ateredis_test.go index 779612e6c..cf0686691 100644 --- a/cmd/ateapi/internal/store/ateredis/ateredis_test.go +++ b/cmd/ateapi/internal/store/ateredis/ateredis_test.go @@ -42,7 +42,11 @@ func setupTest(t *testing.T) (*miniredis.Miniredis, *Persistence, context.Contex rdb := redis.NewClusterClient(&redis.ClusterOptions{ Addrs: []string{mr.Addr()}, }) - return mr, &Persistence{rdb: rdb}, context.Background() + p, err := NewPersistence(rdb) + if err != nil { + t.Fatalf("NewPersistence: %v", err) + } + return mr, p, context.Background() } func TestGetActor_NotFound(t *testing.T) { diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index e7f46111d..a431221a9 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -37,7 +37,10 @@ func SetupTestStore(t *testing.T) (store.Interface, func()) { Addrs: []string{mr.Addr()}, }) - persistence := ateredis.NewPersistence(rdb) + persistence, err := ateredis.NewPersistence(rdb) + if err != nil { + t.Fatalf("ateredis.NewPersistence: %v", err) + } cleanup := func() { mr.Close() diff --git a/cmd/ateapi/internal/workercache/workercache.go b/cmd/ateapi/internal/workercache/workercache.go index f945faa6c..1e67bbb75 100644 --- a/cmd/ateapi/internal/workercache/workercache.go +++ b/cmd/ateapi/internal/workercache/workercache.go @@ -28,13 +28,13 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" "k8s.io/apimachinery/pkg/util/wait" ) // Cache maintains an in-memory snapshot of all workers. -// -// TODO: add metrics — at minimum a gauge for worker count, a counter for -// resync events, and a counter for failed PUBLISH operations (in ateredis). type Cache struct { store store.Interface relistInterval time.Duration @@ -43,17 +43,53 @@ type Cache struct { workers map[string]*ateapipb.Worker ready atomic.Bool + + metricWorkerCount metric.Int64Gauge + metricNotReadyDuration metric.Float64Counter + metricResyncs metric.Int64Counter + metricRelists metric.Int64Counter } // New creates a Cache backed by a given store. relistInterval controls how // often the cache performs a full ListWorkers to recover from state drifts // caused by missing WorkerWatch events. -func New(store store.Interface, relistInterval time.Duration) *Cache { - return &Cache{ - store: store, +func New(s store.Interface, relistInterval time.Duration) (*Cache, error) { + c := &Cache{ + store: s, relistInterval: relistInterval, workers: make(map[string]*ateapipb.Worker), } + m := otel.Meter("workercache") + var err error + c.metricWorkerCount, err = m.Int64Gauge( + "cache.worker.count", + metric.WithUnit("{worker}"), + metric.WithDescription("Current number of workers in the cache.")) + if err != nil { + return nil, fmt.Errorf("create cache.worker.count gauge failed: %w", err) + } + c.metricNotReadyDuration, err = m.Float64Counter( + "cache.not_ready.duration", + metric.WithUnit("s"), + metric.WithDescription("Total time the worker cache spent not ready, between a watch disconnect and a successful resync.")) + if err != nil { + return nil, fmt.Errorf("create cache.not_ready.duration counter failed: %w", err) + } + c.metricResyncs, err = m.Int64Counter( + "cache.resyncs", + metric.WithUnit("{resync}"), + metric.WithDescription("Total full resyncs triggered by watch disconnects.")) + if err != nil { + return nil, fmt.Errorf("create cache.resyncs counter failed: %w", err) + } + c.metricRelists, err = m.Int64Counter( + "cache.relists", + metric.WithUnit("{relist}"), + metric.WithDescription("Total relists (initial, resync, and periodic).")) + if err != nil { + return nil, fmt.Errorf("create cache.relists counter failed: %w", err) + } + return c, nil } // Start syncs the cache synchronously, then spawns a background goroutine @@ -101,6 +137,7 @@ func (c *Cache) sync(ctx context.Context) (*store.WorkerWatch, error) { func (c *Cache) relist(ctx context.Context) error { workers, err := c.store.ListWorkers(ctx) if err != nil { + c.metricRelists.Add(ctx, 1, metric.WithAttributes(attribute.String("error.type", "_OTHER"))) return fmt.Errorf("ListWorkers: %w", err) } newMap := make(map[string]*ateapipb.Worker, len(workers)) @@ -109,8 +146,11 @@ func (c *Cache) relist(ctx context.Context) error { } c.mu.Lock() c.workers = newMap + count := int64(len(newMap)) c.mu.Unlock() - slog.InfoContext(ctx, "worker cache synced", slog.Int("count", len(newMap))) + c.metricWorkerCount.Record(ctx, count) + c.metricRelists.Add(ctx, 1) + slog.InfoContext(ctx, "worker cache synced", slog.Int("count", int(count))) return nil } @@ -122,6 +162,7 @@ func (c *Cache) watchEvents(ctx context.Context, watch *store.WorkerWatch) { case event, ok := <-watch.Events: if !ok { c.ready.Store(false) + notReadySince := time.Now() watch.Close() if ctx.Err() != nil { return @@ -132,8 +173,9 @@ func (c *Cache) watchEvents(ctx context.Context, watch *store.WorkerWatch) { return // context cancelled } c.ready.Store(true) + c.metricNotReadyDuration.Add(ctx, time.Since(notReadySince).Seconds()) } else { - c.applyEvent(event) + c.applyEvent(ctx, event) } case <-ticker.C: if err := c.relist(ctx); err != nil { @@ -148,6 +190,7 @@ func (c *Cache) watchEvents(ctx context.Context, watch *store.WorkerWatch) { } func (c *Cache) resync(ctx context.Context) *store.WorkerWatch { + c.metricResyncs.Add(ctx, 1) backoff := wait.Backoff{ Duration: time.Second, Factor: 2.0, @@ -167,10 +210,9 @@ func (c *Cache) resync(ctx context.Context) *store.WorkerWatch { return watch } -func (c *Cache) applyEvent(event store.WorkerEvent) { +func (c *Cache) applyEvent(ctx context.Context, event store.WorkerEvent) { key := workerKey(event.Worker) c.mu.Lock() - defer c.mu.Unlock() switch event.Type { case store.WorkerEventDeleted: delete(c.workers, key) @@ -180,6 +222,9 @@ func (c *Cache) applyEvent(event store.WorkerEvent) { c.workers[key] = event.Worker } } + count := int64(len(c.workers)) + c.mu.Unlock() + c.metricWorkerCount.Record(ctx, count) } func workerKey(w *ateapipb.Worker) string { diff --git a/cmd/ateapi/internal/workercache/workercache_test.go b/cmd/ateapi/internal/workercache/workercache_test.go index 6f53f36ab..1fc0b8b69 100644 --- a/cmd/ateapi/internal/workercache/workercache_test.go +++ b/cmd/ateapi/internal/workercache/workercache_test.go @@ -26,12 +26,16 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/wait" ) func TestCache_NotReadyBeforeStart(t *testing.T) { - c := workercache.New(newFakeStore(), time.Hour) + c := newWorkerCache(t, newFakeStore(), time.Hour) _, err := c.Workers() if err == nil { t.Fatal("expected error from Workers before Start, got nil") @@ -39,12 +43,11 @@ func TestCache_NotReadyBeforeStart(t *testing.T) { } func TestCache_SyncsOnStart(t *testing.T) { + ctx := t.Context() w1 := makeWorker("ns", "pod1", 1) w2 := makeWorker("ns", "pod2", 1) - c := workercache.New(newFakeStore(w1, w2), time.Hour) - ctx := t.Context() - + c := newWorkerCache(t, newFakeStore(w1, w2), time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } @@ -59,10 +62,9 @@ func TestCache_SyncsOnStart(t *testing.T) { } func TestCache_CreatedEvent(t *testing.T) { - fs := newFakeStore() - c := workercache.New(fs, time.Hour) ctx := t.Context() - + fs := newFakeStore() + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } @@ -82,11 +84,10 @@ func TestCache_CreatedEvent(t *testing.T) { } func TestCache_UpdatedEvent_NewerVersionApplied(t *testing.T) { + ctx := t.Context() w := makeWorker("ns", "pod1", 1) fs := newFakeStore(w) - c := workercache.New(fs, time.Hour) - ctx := t.Context() - + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } @@ -107,11 +108,10 @@ func TestCache_UpdatedEvent_NewerVersionApplied(t *testing.T) { } func TestCache_UpdatedEvent_OlderVersionIgnored(t *testing.T) { + ctx := t.Context() w := makeWorker("ns", "pod1", 5) fs := newFakeStore(w) - c := workercache.New(fs, time.Hour) - ctx := t.Context() - + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } @@ -137,11 +137,10 @@ func TestCache_UpdatedEvent_OlderVersionIgnored(t *testing.T) { } func TestCache_DeletedEvent(t *testing.T) { + ctx := t.Context() w := makeWorker("ns", "pod1", 1) fs := newFakeStore(w) - c := workercache.New(fs, time.Hour) - ctx := t.Context() - + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } @@ -158,11 +157,10 @@ func TestCache_DeletedEvent(t *testing.T) { } func TestCache_Disconnect_ResyncsWithFreshSnapshot(t *testing.T) { + ctx := t.Context() w1 := makeWorker("ns", "pod1", 1) fs := newFakeStore(w1) - c := workercache.New(fs, time.Hour) - ctx := t.Context() - + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } @@ -172,7 +170,6 @@ func TestCache_Disconnect_ResyncsWithFreshSnapshot(t *testing.T) { fs.setWorkers(w1, w2) fs.disconnect() - // After resync the cache should reflect the updated snapshot. eventually(t, func() bool { workers, err := c.Workers() return err == nil && len(workers) == 2 @@ -185,15 +182,13 @@ func TestCache_Disconnect_ResyncsWithFreshSnapshot(t *testing.T) { } func TestCache_MultipleDisconnects(t *testing.T) { - fs := newFakeStore() - c := workercache.New(fs, time.Hour) ctx := t.Context() - + fs := newFakeStore() + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } - // Disconnect three times, each time adding a worker to the snapshot. for i := range 3 { pod := makeWorker("ns", string(rune('a'+i)), 1) fs.setWorkers(append(fs.workers[:i], pod)...) @@ -208,11 +203,12 @@ func TestCache_MultipleDisconnects(t *testing.T) { } func TestCache_WatchClosedOnListWorkersFailure(t *testing.T) { + ctx := t.Context() fs := newFakeStore() fs.listErr = errors.New("valkey unavailable") - c := workercache.New(fs, time.Hour) + c := newWorkerCache(t, fs, time.Hour) - if err := c.Start(t.Context()); err == nil { + if err := c.Start(ctx); err == nil { t.Fatal("expected Start to fail when ListWorkers errors") } @@ -225,9 +221,9 @@ func TestCache_WatchClosedOnListWorkersFailure(t *testing.T) { } func TestCache_WatchClosedOnShutdown(t *testing.T) { - fs := newFakeStore() - c := workercache.New(fs, time.Hour) ctx, cancel := context.WithCancel(t.Context()) + fs := newFakeStore() + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) @@ -243,15 +239,14 @@ func TestCache_WatchClosedOnShutdown(t *testing.T) { } func TestCache_WatchClosedOnDisconnectAndShutdown(t *testing.T) { - fs := newFakeStore() - c := workercache.New(fs, time.Hour) ctx, cancel := context.WithCancel(t.Context()) + fs := newFakeStore() + c := newWorkerCache(t, fs, time.Hour) if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } - // Disconnect: the old watch should be closed and a new one opened. fs.disconnect() eventually(t, func() bool { fs.mu.Lock() @@ -259,7 +254,6 @@ func TestCache_WatchClosedOnDisconnectAndShutdown(t *testing.T) { return fs.closes == 1 }, 2*time.Second) - // Shutdown: the new watch should also be closed. cancel() eventually(t, func() bool { fs.mu.Lock() @@ -268,17 +262,16 @@ func TestCache_WatchClosedOnDisconnectAndShutdown(t *testing.T) { }, 2*time.Second) } -func TestCache_Relist_RecoversFromMissedCreate(t *testing.T) { +func TestCache_Relist_PicksUpSilentCreate(t *testing.T) { + ctx := t.Context() w1 := makeWorker("ns", "pod1", 1) fs := newFakeStore(w1) - c := workercache.New(fs, 10*time.Millisecond) + c := newWorkerCache(t, fs, 10*time.Millisecond) - if err := c.Start(t.Context()); err != nil { + if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } - // Add a worker directly to the store without sending a watch event, - // simulating a silent PUBLISH failure. w2 := makeWorker("ns", "pod2", 1) fs.setWorkers(w1, w2) @@ -293,18 +286,17 @@ func TestCache_Relist_RecoversFromMissedCreate(t *testing.T) { } } -func TestCache_Relist_RecoversFromMissedDelete(t *testing.T) { +func TestCache_Relist_PicksUpSilentDelete(t *testing.T) { + ctx := t.Context() w1 := makeWorker("ns", "pod1", 1) w2 := makeWorker("ns", "pod2", 1) fs := newFakeStore(w1, w2) - c := workercache.New(fs, 10*time.Millisecond) + c := newWorkerCache(t, fs, 10*time.Millisecond) - if err := c.Start(t.Context()); err != nil { + if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } - // Remove a worker from the store without a watch event, - // simulating a silent PUBLISH failure on delete. fs.setWorkers(w1) eventually(t, func() bool { @@ -319,23 +311,21 @@ func TestCache_Relist_RecoversFromMissedDelete(t *testing.T) { } func TestCache_Relist_FailureIsNonFatal(t *testing.T) { + ctx := t.Context() w1 := makeWorker("ns", "pod1", 1) fs := newFakeStore(w1) - c := workercache.New(fs, 10*time.Millisecond) + c := newWorkerCache(t, fs, 10*time.Millisecond) - if err := c.Start(t.Context()); err != nil { + if err := c.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } - // Make ListWorkers fail to simulate a transient Valkey error. fs.mu.Lock() fs.listErr = errors.New("valkey unavailable") fs.mu.Unlock() - // Wait long enough for at least one relist attempt. time.Sleep(50 * time.Millisecond) - // Clear the error; the cache should still be usable with the old snapshot. fs.mu.Lock() fs.listErr = nil fs.mu.Unlock() @@ -349,14 +339,99 @@ func TestCache_Relist_FailureIsNonFatal(t *testing.T) { } } +func TestCache_Metrics_WorkersGauge(t *testing.T) { + ctx := t.Context() + reader := newTestProvider(t) + + w1 := makeWorker("ns", "pod1", 1) + w2 := makeWorker("ns", "pod2", 1) + fs := newFakeStore(w1, w2) + c := newWorkerCache(t, fs, time.Hour) + + if err := c.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + + m, ok := collectMetric(t, reader, "cache.worker.count") + if !ok { + t.Fatal("cache.worker.count not found") + } + metricdatatest.AssertEqual(t, metricdata.Metrics{ + Name: "cache.worker.count", + Description: "Current number of workers in the cache.", + Unit: "{worker}", + Data: metricdata.Gauge[int64]{ + DataPoints: []metricdata.DataPoint[int64]{{Value: 2}}, + }, + }, m, metricdatatest.IgnoreTimestamp()) + + fs.send(store.WorkerEvent{ + Type: store.WorkerEventDeleted, + Worker: &ateapipb.Worker{WorkerNamespace: "ns", WorkerPod: "pod1"}, + }) + eventually(t, func() bool { + m, ok := collectMetric(t, reader, "cache.worker.count") + if !ok { + return false + } + g := m.Data.(metricdata.Gauge[int64]) + return len(g.DataPoints) > 0 && g.DataPoints[len(g.DataPoints)-1].Value == 1 + }, 2*time.Second) +} + +func TestCache_Metrics_ResyncsCounter(t *testing.T) { + ctx := t.Context() + reader := newTestProvider(t) + + fs := newFakeStore() + c := newWorkerCache(t, fs, time.Hour) + if err := c.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + + fs.disconnect() + + eventually(t, func() bool { + m, ok := collectMetric(t, reader, "cache.resyncs") + if !ok { + return false + } + sum := m.Data.(metricdata.Sum[int64]) + return len(sum.DataPoints) > 0 && sum.DataPoints[0].Value == 1 + }, 2*time.Second) +} + +func TestCache_Metrics_NotReadyDurationCounter(t *testing.T) { + ctx := t.Context() + reader := newTestProvider(t) + + fs := newFakeStore() + c := newWorkerCache(t, fs, time.Hour) + if err := c.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + + fs.disconnect() + + eventually(t, func() bool { + m, ok := collectMetric(t, reader, "cache.not_ready.duration") + if !ok { + return false + } + sum := m.Data.(metricdata.Sum[float64]) + return len(sum.DataPoints) > 0 && sum.DataPoints[0].Value > 0 + }, 2*time.Second) +} + +// fakeStore is a test double for store.Interface. type fakeStore struct { store.Interface mu sync.Mutex workers []*ateapipb.Worker watchCh chan store.WorkerEvent - listErr error // if set, ListWorkers returns it - closes int // number of times a returned watch was Closed + listErr error + closes int } func newFakeStore(workers ...*ateapipb.Worker) *fakeStore { @@ -416,7 +491,6 @@ func makeWorker(namespace, pod string, version int64) *ateapipb.Worker { } } -// workerSortOpt compares workers ignoring ordering. var workerSortOpt = cmpopts.SortSlices(func(a, b *ateapipb.Worker) bool { if a.GetWorkerNamespace() != b.GetWorkerNamespace() { return a.GetWorkerNamespace() < b.GetWorkerNamespace() @@ -424,7 +498,6 @@ var workerSortOpt = cmpopts.SortSlices(func(a, b *ateapipb.Worker) bool { return a.GetWorkerPod() < b.GetWorkerPod() }) -// eventually polls condition every 10ms until it returns true or timeout elapses. func eventually(t *testing.T, condition func() bool, timeout time.Duration) { t.Helper() err := wait.PollUntilContextTimeout(t.Context(), 10*time.Millisecond, timeout, true, func(context.Context) (bool, error) { @@ -434,3 +507,42 @@ func eventually(t *testing.T, condition func() bool, timeout time.Duration) { t.Fatal("condition not met within timeout") } } + +// newWorkerCache creates a Cache, failing the test if New returns an error. +func newWorkerCache(t *testing.T, fs *fakeStore, relistInterval time.Duration) *workercache.Cache { + t.Helper() + c, err := workercache.New(fs, relistInterval) + if err != nil { + t.Fatalf("workercache.New: %v", err) + } + return c +} + +// newTestProvider installs a fresh ManualReader as the global MeterProvider for +// this test. Because New creates instruments from otel.GetMeterProvider() at +// call time (not package-init time), each test gets isolated metric state. +func newTestProvider(t *testing.T) *sdkmetric.ManualReader { + t.Helper() + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(mp) + t.Cleanup(func() { mp.Shutdown(context.Background()) }) + return reader +} + +// collectMetric collects all metrics from reader and returns the one named name. +func collectMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) (metricdata.Metrics, bool) { + t.Helper() + var rm metricdata.ResourceMetrics + if err := reader.Collect(t.Context(), &rm); err != nil { + t.Fatalf("collect metrics: %v", err) + } + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return m, true + } + } + } + return metricdata.Metrics{}, false +} diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 165561617..431590f20 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -110,9 +110,15 @@ func main() { serverboot.Fatal(ctx, "Failed to build server credentials", err) } - redisPersistence := ateredis.NewPersistence(redisClient) + redisPersistence, err := ateredis.NewPersistence(redisClient) + if err != nil { + serverboot.Fatal(ctx, "Failed to create Redis persistence", err) + } - workerCache := workercache.New(redisPersistence, 5*time.Minute) + workerCache, err := workercache.New(redisPersistence, 5*time.Minute) + if err != nil { + serverboot.Fatal(ctx, "Failed to create worker cache", err) + } if err := workerCache.Start(ctx); err != nil { serverboot.Fatal(ctx, "Failed to seed worker cache", err) } diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/README.md b/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/README.md new file mode 100644 index 000000000..969ffde50 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/README.md @@ -0,0 +1,3 @@ +# SDK Metric data test + +[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest)](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest) diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/assertion.go b/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/assertion.go new file mode 100644 index 000000000..7935d9e99 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/assertion.go @@ -0,0 +1,270 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package metricdatatest provides testing functionality for use with the +// metricdata package. +package metricdatatest // import "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + +import ( + "fmt" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// Datatypes are the concrete data-types the metricdata package provides. +type Datatypes interface { + metricdata.DataPoint[float64] | + metricdata.DataPoint[int64] | + metricdata.Gauge[float64] | + metricdata.Gauge[int64] | + metricdata.Histogram[float64] | + metricdata.Histogram[int64] | + metricdata.HistogramDataPoint[float64] | + metricdata.HistogramDataPoint[int64] | + metricdata.Extrema[int64] | + metricdata.Extrema[float64] | + metricdata.Metrics | + metricdata.ResourceMetrics | + metricdata.ScopeMetrics | + metricdata.Sum[float64] | + metricdata.Sum[int64] | + metricdata.Exemplar[float64] | + metricdata.Exemplar[int64] | + metricdata.ExponentialHistogram[float64] | + metricdata.ExponentialHistogram[int64] | + metricdata.ExponentialHistogramDataPoint[float64] | + metricdata.ExponentialHistogramDataPoint[int64] | + metricdata.ExponentialBucket | + metricdata.Summary | + metricdata.SummaryDataPoint | + metricdata.QuantileValue + + // Interface types are not allowed in union types, therefore the + // Aggregation and Value type from metricdata are not included here. +} + +// TestingT is an interface that implements [testing.T], but without the +// private method of [testing.TB], so other testing packages can rely on it as +// well. +// The methods in this interface must match the [testing.TB] interface. +type TestingT interface { + Helper() + // DO NOT CHANGE: any modification will not be backwards compatible and + // must never be done outside of a new major release. + + Error(...any) + // DO NOT CHANGE: any modification will not be backwards compatible and + // must never be done outside of a new major release. +} + +type config struct { + ignoreTimestamp bool + ignoreExemplars bool + ignoreValue bool +} + +func newConfig(opts []Option) config { + var cfg config + for _, opt := range opts { + cfg = opt.apply(cfg) + } + return cfg +} + +// Option allows for fine grain control over how AssertEqual operates. +type Option interface { + apply(cfg config) config +} + +type fnOption func(cfg config) config + +func (fn fnOption) apply(cfg config) config { + return fn(cfg) +} + +// IgnoreTimestamp disables checking if timestamps are different. +func IgnoreTimestamp() Option { + return fnOption(func(cfg config) config { + cfg.ignoreTimestamp = true + return cfg + }) +} + +// IgnoreExemplars disables checking if Exemplars are different. +func IgnoreExemplars() Option { + return fnOption(func(cfg config) config { + cfg.ignoreExemplars = true + return cfg + }) +} + +// IgnoreValue disables checking if values are different. This can be +// useful for non-deterministic values, like measured durations. +// +// This will ignore the value and trace information for Exemplars; +// the buckets, zero count, scale, sum, max, min, and counts of +// ExponentialHistogramDataPoints; the buckets, sum, count, max, +// and min of HistogramDataPoints; the value of DataPoints. +func IgnoreValue() Option { + return fnOption(func(cfg config) config { + cfg.ignoreValue = true + return cfg + }) +} + +// AssertEqual asserts that the two concrete data-types from the metricdata +// package are equal. +func AssertEqual[T Datatypes](t TestingT, expected, actual T, opts ...Option) bool { + t.Helper() + + cfg := newConfig(opts) + + // Generic types cannot be type asserted. Use an interface instead. + aIface := any(actual) + + var r []string + switch e := any(expected).(type) { + case metricdata.Exemplar[int64]: + r = equalExemplars(e, aIface.(metricdata.Exemplar[int64]), cfg) + case metricdata.Exemplar[float64]: + r = equalExemplars(e, aIface.(metricdata.Exemplar[float64]), cfg) + case metricdata.DataPoint[int64]: + r = equalDataPoints(e, aIface.(metricdata.DataPoint[int64]), cfg) + case metricdata.DataPoint[float64]: + r = equalDataPoints(e, aIface.(metricdata.DataPoint[float64]), cfg) + case metricdata.Gauge[int64]: + r = equalGauges(e, aIface.(metricdata.Gauge[int64]), cfg) + case metricdata.Gauge[float64]: + r = equalGauges(e, aIface.(metricdata.Gauge[float64]), cfg) + case metricdata.Histogram[float64]: + r = equalHistograms(e, aIface.(metricdata.Histogram[float64]), cfg) + case metricdata.Histogram[int64]: + r = equalHistograms(e, aIface.(metricdata.Histogram[int64]), cfg) + case metricdata.HistogramDataPoint[float64]: + r = equalHistogramDataPoints(e, aIface.(metricdata.HistogramDataPoint[float64]), cfg) + case metricdata.HistogramDataPoint[int64]: + r = equalHistogramDataPoints(e, aIface.(metricdata.HistogramDataPoint[int64]), cfg) + case metricdata.Extrema[int64]: + r = equalExtrema(e, aIface.(metricdata.Extrema[int64]), cfg) + case metricdata.Extrema[float64]: + r = equalExtrema(e, aIface.(metricdata.Extrema[float64]), cfg) + case metricdata.Metrics: + r = equalMetrics(e, aIface.(metricdata.Metrics), cfg) + case metricdata.ResourceMetrics: + r = equalResourceMetrics(e, aIface.(metricdata.ResourceMetrics), cfg) + case metricdata.ScopeMetrics: + r = equalScopeMetrics(e, aIface.(metricdata.ScopeMetrics), cfg) + case metricdata.Sum[int64]: + r = equalSums(e, aIface.(metricdata.Sum[int64]), cfg) + case metricdata.Sum[float64]: + r = equalSums(e, aIface.(metricdata.Sum[float64]), cfg) + case metricdata.ExponentialHistogram[float64]: + r = equalExponentialHistograms(e, aIface.(metricdata.ExponentialHistogram[float64]), cfg) + case metricdata.ExponentialHistogram[int64]: + r = equalExponentialHistograms(e, aIface.(metricdata.ExponentialHistogram[int64]), cfg) + case metricdata.ExponentialHistogramDataPoint[float64]: + r = equalExponentialHistogramDataPoints(e, aIface.(metricdata.ExponentialHistogramDataPoint[float64]), cfg) + case metricdata.ExponentialHistogramDataPoint[int64]: + r = equalExponentialHistogramDataPoints(e, aIface.(metricdata.ExponentialHistogramDataPoint[int64]), cfg) + case metricdata.ExponentialBucket: + r = equalExponentialBuckets(e, aIface.(metricdata.ExponentialBucket), cfg) + case metricdata.Summary: + r = equalSummary(e, aIface.(metricdata.Summary), cfg) + case metricdata.SummaryDataPoint: + r = equalSummaryDataPoint(e, aIface.(metricdata.SummaryDataPoint), cfg) + case metricdata.QuantileValue: + r = equalQuantileValue(e, aIface.(metricdata.QuantileValue), cfg) + default: + // We control all types passed to this, panic to signal developers + // early they changed things in an incompatible way. + panic(fmt.Sprintf("unknown types: %T", expected)) + } + + if len(r) > 0 { + t.Error(r) + return false + } + return true +} + +// AssertAggregationsEqual asserts that two Aggregations are equal. +func AssertAggregationsEqual(t TestingT, expected, actual metricdata.Aggregation, opts ...Option) bool { + t.Helper() + + cfg := newConfig(opts) + if r := equalAggregations(expected, actual, cfg); len(r) > 0 { + t.Error(r) + return false + } + return true +} + +// AssertHasAttributes asserts that all Datapoints or HistogramDataPoints have all passed attrs. +func AssertHasAttributes[T Datatypes](t TestingT, actual T, attrs ...attribute.KeyValue) bool { + t.Helper() + + var reasons []string + + switch e := any(actual).(type) { + case metricdata.Exemplar[int64]: + reasons = hasAttributesExemplars(e, attrs...) + case metricdata.Exemplar[float64]: + reasons = hasAttributesExemplars(e, attrs...) + case metricdata.DataPoint[int64]: + reasons = hasAttributesDataPoints(e, attrs...) + case metricdata.DataPoint[float64]: + reasons = hasAttributesDataPoints(e, attrs...) + case metricdata.Gauge[int64]: + reasons = hasAttributesGauge(e, attrs...) + case metricdata.Gauge[float64]: + reasons = hasAttributesGauge(e, attrs...) + case metricdata.Sum[int64]: + reasons = hasAttributesSum(e, attrs...) + case metricdata.Sum[float64]: + reasons = hasAttributesSum(e, attrs...) + case metricdata.HistogramDataPoint[int64]: + reasons = hasAttributesHistogramDataPoints(e, attrs...) + case metricdata.HistogramDataPoint[float64]: + reasons = hasAttributesHistogramDataPoints(e, attrs...) + case metricdata.Extrema[int64], metricdata.Extrema[float64]: + // Nothing to check. + case metricdata.Histogram[int64]: + reasons = hasAttributesHistogram(e, attrs...) + case metricdata.Histogram[float64]: + reasons = hasAttributesHistogram(e, attrs...) + case metricdata.Metrics: + reasons = hasAttributesMetrics(e, attrs...) + case metricdata.ScopeMetrics: + reasons = hasAttributesScopeMetrics(e, attrs...) + case metricdata.ResourceMetrics: + reasons = hasAttributesResourceMetrics(e, attrs...) + case metricdata.ExponentialHistogram[int64]: + reasons = hasAttributesExponentialHistogram(e, attrs...) + case metricdata.ExponentialHistogram[float64]: + reasons = hasAttributesExponentialHistogram(e, attrs...) + case metricdata.ExponentialHistogramDataPoint[int64]: + reasons = hasAttributesExponentialHistogramDataPoints(e, attrs...) + case metricdata.ExponentialHistogramDataPoint[float64]: + reasons = hasAttributesExponentialHistogramDataPoints(e, attrs...) + case metricdata.ExponentialBucket: + // Nothing to check. + case metricdata.Summary: + reasons = hasAttributesSummary(e, attrs...) + case metricdata.SummaryDataPoint: + reasons = hasAttributesSummaryDataPoint(e, attrs...) + case metricdata.QuantileValue: + // Nothing to check. + default: + // We control all types passed to this, panic to signal developers + // early they changed things in an incompatible way. + panic(fmt.Sprintf("unknown types: %T", actual)) + } + + if len(reasons) > 0 { + t.Error(reasons) + return false + } + + return true +} diff --git a/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/comparisons.go b/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/comparisons.go new file mode 100644 index 000000000..0cd0f2372 --- /dev/null +++ b/vendor/go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest/comparisons.go @@ -0,0 +1,888 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package metricdatatest // import "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + +import ( + "bytes" + "fmt" + "reflect" + "slices" + "strings" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// equalResourceMetrics returns reasons ResourceMetrics are not equal. If they +// are equal, the returned reasons will be empty. +// +// The ScopeMetrics each ResourceMetrics contains are compared based on +// containing the same ScopeMetrics, not the order they are stored in. +func equalResourceMetrics(a, b metricdata.ResourceMetrics, cfg config) (reasons []string) { + if !a.Resource.Equal(b.Resource) { + reasons = append(reasons, notEqualStr("Resources", a.Resource, b.Resource)) + } + + r := diffSlices( + a.ScopeMetrics, + b.ScopeMetrics, + func(sm metricdata.ScopeMetrics) string { + return fmt.Sprintf("Scope %q", sm.Scope.Name) + }, + func(a, b metricdata.ScopeMetrics) []string { + return equalScopeMetrics(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "ResourceMetrics ScopeMetrics not equal:\n"+r) + } + return reasons +} + +// equalScopeMetrics returns reasons ScopeMetrics are not equal. If they are +// equal, the returned reasons will be empty. +// +// The Metrics each ScopeMetrics contains are compared based on containing the +// same Metrics, not the order they are stored in. +func equalScopeMetrics(a, b metricdata.ScopeMetrics, cfg config) (reasons []string) { + if a.Scope != b.Scope { + reasons = append(reasons, notEqualStr("Scope", a.Scope, b.Scope)) + } + + r := diffSlices( + a.Metrics, + b.Metrics, + func(m metricdata.Metrics) string { + return fmt.Sprintf("Metric %q", m.Name) + }, + func(a, b metricdata.Metrics) []string { + return equalMetrics(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "ScopeMetrics Metrics not equal:\n"+r) + } + return reasons +} + +// equalMetrics returns reasons Metrics are not equal. If they are equal, the +// returned reasons will be empty. +func equalMetrics(a, b metricdata.Metrics, cfg config) (reasons []string) { + if a.Name != b.Name { + reasons = append(reasons, notEqualStr("Name", a.Name, b.Name)) + } + if a.Description != b.Description { + reasons = append(reasons, notEqualStr("Description", a.Description, b.Description)) + } + if a.Unit != b.Unit { + reasons = append(reasons, notEqualStr("Unit", a.Unit, b.Unit)) + } + + r := equalAggregations(a.Data, b.Data, cfg) + if len(r) > 0 { + reasons = append(reasons, "Metrics Data not equal:") + reasons = append(reasons, r...) + } + return reasons +} + +// equalAggregations returns reasons a and b are not equal. If they are equal, +// the returned reasons will be empty. +func equalAggregations(a, b metricdata.Aggregation, cfg config) (reasons []string) { + if a == nil || b == nil { + if a != b { + return []string{notEqualStr("Aggregation", a, b)} + } + return reasons + } + + if reflect.TypeOf(a) != reflect.TypeOf(b) { + return []string{fmt.Sprintf("Aggregation types not equal:\nexpected: %T\nactual: %T", a, b)} + } + + switch v := a.(type) { + case metricdata.Gauge[int64]: + r := equalGauges(v, b.(metricdata.Gauge[int64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "Gauge[int64] not equal:") + reasons = append(reasons, r...) + } + case metricdata.Gauge[float64]: + r := equalGauges(v, b.(metricdata.Gauge[float64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "Gauge[float64] not equal:") + reasons = append(reasons, r...) + } + case metricdata.Sum[int64]: + r := equalSums(v, b.(metricdata.Sum[int64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "Sum[int64] not equal:") + reasons = append(reasons, r...) + } + case metricdata.Sum[float64]: + r := equalSums(v, b.(metricdata.Sum[float64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "Sum[float64] not equal:") + reasons = append(reasons, r...) + } + case metricdata.Histogram[int64]: + r := equalHistograms(v, b.(metricdata.Histogram[int64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "Histogram not equal:") + reasons = append(reasons, r...) + } + case metricdata.Histogram[float64]: + r := equalHistograms(v, b.(metricdata.Histogram[float64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "Histogram not equal:") + reasons = append(reasons, r...) + } + case metricdata.ExponentialHistogram[int64]: + r := equalExponentialHistograms(v, b.(metricdata.ExponentialHistogram[int64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "ExponentialHistogram not equal:") + reasons = append(reasons, r...) + } + case metricdata.ExponentialHistogram[float64]: + r := equalExponentialHistograms(v, b.(metricdata.ExponentialHistogram[float64]), cfg) + if len(r) > 0 { + reasons = append(reasons, "ExponentialHistogram not equal:") + reasons = append(reasons, r...) + } + case metricdata.Summary: + r := equalSummary(v, b.(metricdata.Summary), cfg) + if len(r) > 0 { + reasons = append(reasons, "Summary not equal:") + reasons = append(reasons, r...) + } + default: + reasons = append(reasons, fmt.Sprintf("Aggregation of unknown types %T", a)) + } + return reasons +} + +// equalGauges returns reasons Gauges are not equal. If they are equal, the +// returned reasons will be empty. +// +// The DataPoints each Gauge contains are compared based on containing the +// same DataPoints, not the order they are stored in. +func equalGauges[N int64 | float64](a, b metricdata.Gauge[N], cfg config) (reasons []string) { + r := diffSlices( + a.DataPoints, + b.DataPoints, + func(dp metricdata.DataPoint[N]) string { + return fmt.Sprintf("DataPoint [%v]", dp.Attributes.Encoded(attribute.DefaultEncoder())) + }, + func(a, b metricdata.DataPoint[N]) []string { + return equalDataPoints(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Gauge DataPoints not equal:\n"+r) + } + return reasons +} + +// equalSums returns reasons Sums are not equal. If they are equal, the +// returned reasons will be empty. +// +// The DataPoints each Sum contains are compared based on containing the same +// DataPoints, not the order they are stored in. +func equalSums[N int64 | float64](a, b metricdata.Sum[N], cfg config) (reasons []string) { + if a.Temporality != b.Temporality { + reasons = append(reasons, notEqualStr("Temporality", a.Temporality, b.Temporality)) + } + if a.IsMonotonic != b.IsMonotonic { + reasons = append(reasons, notEqualStr("IsMonotonic", a.IsMonotonic, b.IsMonotonic)) + } + + r := diffSlices( + a.DataPoints, + b.DataPoints, + func(dp metricdata.DataPoint[N]) string { + return fmt.Sprintf("DataPoint [%v]", dp.Attributes.Encoded(attribute.DefaultEncoder())) + }, + func(a, b metricdata.DataPoint[N]) []string { + return equalDataPoints(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Sum DataPoints not equal:\n"+r) + } + return reasons +} + +// equalHistograms returns reasons Histograms are not equal. If they are +// equal, the returned reasons will be empty. +// +// The DataPoints each Histogram contains are compared based on containing the +// same HistogramDataPoint, not the order they are stored in. +func equalHistograms[N int64 | float64](a, b metricdata.Histogram[N], cfg config) (reasons []string) { + if a.Temporality != b.Temporality { + reasons = append(reasons, notEqualStr("Temporality", a.Temporality, b.Temporality)) + } + + r := diffSlices( + a.DataPoints, + b.DataPoints, + func(dp metricdata.HistogramDataPoint[N]) string { + return fmt.Sprintf("HistogramDataPoint [%v]", dp.Attributes.Encoded(attribute.DefaultEncoder())) + }, + func(a, b metricdata.HistogramDataPoint[N]) []string { + return equalHistogramDataPoints(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Histogram DataPoints not equal:\n"+r) + } + return reasons +} + +// equalDataPoints returns reasons DataPoints are not equal. If they are +// equal, the returned reasons will be empty. +func equalDataPoints[N int64 | float64]( + a, b metricdata.DataPoint[N], + cfg config, +) (reasons []string) { // nolint: revive // Intentional internal control flag + if !a.Attributes.Equals(&b.Attributes) { + reasons = append(reasons, notEqualStr( + "Attributes", + a.Attributes.Encoded(attribute.DefaultEncoder()), + b.Attributes.Encoded(attribute.DefaultEncoder()), + )) + } + + if !cfg.ignoreTimestamp { + if !a.StartTime.Equal(b.StartTime) { + reasons = append(reasons, notEqualStr("StartTime", a.StartTime.UnixNano(), b.StartTime.UnixNano())) + } + if !a.Time.Equal(b.Time) { + reasons = append(reasons, notEqualStr("Time", a.Time.UnixNano(), b.Time.UnixNano())) + } + } + + if !cfg.ignoreValue { + if a.Value != b.Value { + reasons = append(reasons, notEqualStr("Value", a.Value, b.Value)) + } + } + + if !cfg.ignoreExemplars { + r := diffSlices( + a.Exemplars, + b.Exemplars, + func(_ metricdata.Exemplar[N]) string { + return "Exemplar" + }, + func(a, b metricdata.Exemplar[N]) []string { + return equalExemplars(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Exemplars not equal:\n"+r) + } + } + return reasons +} + +// equalHistogramDataPoints returns reasons HistogramDataPoints are not equal. +// If they are equal, the returned reasons will be empty. +func equalHistogramDataPoints[N int64 | float64]( + a, b metricdata.HistogramDataPoint[N], + cfg config, +) (reasons []string) { // nolint: revive // Intentional internal control flag + if !a.Attributes.Equals(&b.Attributes) { + reasons = append(reasons, notEqualStr( + "Attributes", + a.Attributes.Encoded(attribute.DefaultEncoder()), + b.Attributes.Encoded(attribute.DefaultEncoder()), + )) + } + if !cfg.ignoreTimestamp { + if !a.StartTime.Equal(b.StartTime) { + reasons = append(reasons, notEqualStr("StartTime", a.StartTime.UnixNano(), b.StartTime.UnixNano())) + } + if !a.Time.Equal(b.Time) { + reasons = append(reasons, notEqualStr("Time", a.Time.UnixNano(), b.Time.UnixNano())) + } + } + if !cfg.ignoreValue { + if a.Count != b.Count { + reasons = append(reasons, notEqualStr("Count", a.Count, b.Count)) + } + if !slices.Equal(a.Bounds, b.Bounds) { + reasons = append(reasons, notEqualStr("Bounds", a.Bounds, b.Bounds)) + } + if !slices.Equal(a.BucketCounts, b.BucketCounts) { + reasons = append(reasons, notEqualStr("BucketCounts", a.BucketCounts, b.BucketCounts)) + } + if !eqExtrema(a.Min, b.Min) { + reasons = append(reasons, notEqualStr("Min", a.Min, b.Min)) + } + if !eqExtrema(a.Max, b.Max) { + reasons = append(reasons, notEqualStr("Max", a.Max, b.Max)) + } + if a.Sum != b.Sum { + reasons = append(reasons, notEqualStr("Sum", a.Sum, b.Sum)) + } + } + if !cfg.ignoreExemplars { + r := diffSlices( + a.Exemplars, + b.Exemplars, + func(_ metricdata.Exemplar[N]) string { + return "Exemplar" + }, + func(a, b metricdata.Exemplar[N]) []string { + return equalExemplars(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Exemplars not equal:\n"+r) + } + } + return reasons +} + +// equalExponentialHistograms returns reasons exponential Histograms are not equal. If they are +// equal, the returned reasons will be empty. +// +// The DataPoints each Histogram contains are compared based on containing the +// same HistogramDataPoint, not the order they are stored in. +func equalExponentialHistograms[N int64 | float64]( + a, b metricdata.ExponentialHistogram[N], + cfg config, +) (reasons []string) { + if a.Temporality != b.Temporality { + reasons = append(reasons, notEqualStr("Temporality", a.Temporality, b.Temporality)) + } + + r := diffSlices( + a.DataPoints, + b.DataPoints, + func(dp metricdata.ExponentialHistogramDataPoint[N]) string { + return fmt.Sprintf("ExponentialHistogramDataPoint [%v]", dp.Attributes.Encoded(attribute.DefaultEncoder())) + }, + func(a, b metricdata.ExponentialHistogramDataPoint[N]) []string { + return equalExponentialHistogramDataPoints(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Histogram DataPoints not equal:\n"+r) + } + return reasons +} + +// equalExponentialHistogramDataPoints returns reasons HistogramDataPoints are not equal. +// If they are equal, the returned reasons will be empty. +func equalExponentialHistogramDataPoints[N int64 | float64]( + a, b metricdata.ExponentialHistogramDataPoint[N], + cfg config, +) (reasons []string) { // nolint: revive // Intentional internal control flag + if !a.Attributes.Equals(&b.Attributes) { + reasons = append(reasons, notEqualStr( + "Attributes", + a.Attributes.Encoded(attribute.DefaultEncoder()), + b.Attributes.Encoded(attribute.DefaultEncoder()), + )) + } + if !cfg.ignoreTimestamp { + if !a.StartTime.Equal(b.StartTime) { + reasons = append(reasons, notEqualStr("StartTime", a.StartTime.UnixNano(), b.StartTime.UnixNano())) + } + if !a.Time.Equal(b.Time) { + reasons = append(reasons, notEqualStr("Time", a.Time.UnixNano(), b.Time.UnixNano())) + } + } + if !cfg.ignoreValue { + if a.Count != b.Count { + reasons = append(reasons, notEqualStr("Count", a.Count, b.Count)) + } + if !eqExtrema(a.Min, b.Min) { + reasons = append(reasons, notEqualStr("Min", a.Min, b.Min)) + } + if !eqExtrema(a.Max, b.Max) { + reasons = append(reasons, notEqualStr("Max", a.Max, b.Max)) + } + if a.Sum != b.Sum { + reasons = append(reasons, notEqualStr("Sum", a.Sum, b.Sum)) + } + + if a.Scale != b.Scale { + reasons = append(reasons, notEqualStr("Scale", a.Scale, b.Scale)) + } + if a.ZeroCount != b.ZeroCount { + reasons = append(reasons, notEqualStr("ZeroCount", a.ZeroCount, b.ZeroCount)) + } + + r := equalExponentialBuckets(a.PositiveBucket, b.PositiveBucket, cfg) + if len(r) > 0 { + reasons = append(reasons, r...) + } + r = equalExponentialBuckets(a.NegativeBucket, b.NegativeBucket, cfg) + if len(r) > 0 { + reasons = append(reasons, r...) + } + } + if !cfg.ignoreExemplars { + r := diffSlices( + a.Exemplars, + b.Exemplars, + func(_ metricdata.Exemplar[N]) string { + return "Exemplar" + }, + func(a, b metricdata.Exemplar[N]) []string { + return equalExemplars(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Exemplars not equal:\n"+r) + } + } + return reasons +} + +func equalExponentialBuckets(a, b metricdata.ExponentialBucket, _ config) (reasons []string) { + if a.Offset != b.Offset { + reasons = append(reasons, notEqualStr("Offset", a.Offset, b.Offset)) + } + if !slices.Equal(a.Counts, b.Counts) { + reasons = append(reasons, notEqualStr("Counts", a.Counts, b.Counts)) + } + return reasons +} + +func equalSummary(a, b metricdata.Summary, cfg config) (reasons []string) { + r := diffSlices( + a.DataPoints, + b.DataPoints, + func(dp metricdata.SummaryDataPoint) string { + return fmt.Sprintf("SummaryDataPoint [%v]", dp.Attributes.Encoded(attribute.DefaultEncoder())) + }, + func(a, b metricdata.SummaryDataPoint) []string { + return equalSummaryDataPoint(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, "Summary DataPoints not equal:\n"+r) + } + return reasons +} + +func equalSummaryDataPoint(a, b metricdata.SummaryDataPoint, cfg config) (reasons []string) { + if !a.Attributes.Equals(&b.Attributes) { + reasons = append(reasons, notEqualStr( + "Attributes", + a.Attributes.Encoded(attribute.DefaultEncoder()), + b.Attributes.Encoded(attribute.DefaultEncoder()), + )) + } + if !cfg.ignoreTimestamp { + if !a.StartTime.Equal(b.StartTime) { + reasons = append(reasons, notEqualStr("StartTime", a.StartTime.UnixNano(), b.StartTime.UnixNano())) + } + if !a.Time.Equal(b.Time) { + reasons = append(reasons, notEqualStr("Time", a.Time.UnixNano(), b.Time.UnixNano())) + } + } + if !cfg.ignoreValue { + if a.Count != b.Count { + reasons = append(reasons, notEqualStr("Count", a.Count, b.Count)) + } + if a.Sum != b.Sum { + reasons = append(reasons, notEqualStr("Sum", a.Sum, b.Sum)) + } + r := diffSlices( + a.QuantileValues, + b.QuantileValues, + func(qv metricdata.QuantileValue) string { + return fmt.Sprintf("QuantileValue %v", qv.Quantile) + }, + func(a, b metricdata.QuantileValue) []string { + return equalQuantileValue(a, b, cfg) + }, + ) + if r != "" { + reasons = append(reasons, r) + } + } + return reasons +} + +func equalQuantileValue(a, b metricdata.QuantileValue, _ config) (reasons []string) { + if a.Quantile != b.Quantile { + reasons = append(reasons, notEqualStr("Quantile", a.Quantile, b.Quantile)) + } + if a.Value != b.Value { + reasons = append(reasons, notEqualStr("Value", a.Value, b.Value)) + } + return reasons +} + +func notEqualStr(prefix string, expected, actual any) string { + return fmt.Sprintf("%s not equal:\nexpected: %v\nactual: %v", prefix, expected, actual) +} + +func equalExtrema[N int64 | float64](a, b metricdata.Extrema[N], _ config) (reasons []string) { + if !eqExtrema(a, b) { + reasons = append(reasons, notEqualStr("Extrema", a, b)) + } + return reasons +} + +func eqExtrema[N int64 | float64](a, b metricdata.Extrema[N]) bool { + aV, aOk := a.Value() + bV, bOk := b.Value() + + if !aOk || !bOk { + return aOk == bOk + } + return aV == bV +} + +func equalKeyValue(a, b attribute.KeyValue) bool { + if a.Key != b.Key { + return false + } + if a.Value.Type() != b.Value.Type() { + return false + } + switch a.Value.Type() { + case attribute.BOOL: + if a.Value.AsBool() != b.Value.AsBool() { + return false + } + case attribute.INT64: + if a.Value.AsInt64() != b.Value.AsInt64() { + return false + } + case attribute.FLOAT64: + if a.Value.AsFloat64() != b.Value.AsFloat64() { + return false + } + case attribute.STRING: + if a.Value.AsString() != b.Value.AsString() { + return false + } + case attribute.BOOLSLICE: + if ok := slices.Equal(a.Value.AsBoolSlice(), b.Value.AsBoolSlice()); !ok { + return false + } + case attribute.INT64SLICE: + if ok := slices.Equal(a.Value.AsInt64Slice(), b.Value.AsInt64Slice()); !ok { + return false + } + case attribute.FLOAT64SLICE: + if ok := slices.Equal(a.Value.AsFloat64Slice(), b.Value.AsFloat64Slice()); !ok { + return false + } + case attribute.STRINGSLICE: + if ok := slices.Equal(a.Value.AsStringSlice(), b.Value.AsStringSlice()); !ok { + return false + } + case attribute.EMPTY: + default: + // We control all types passed to this, panic to signal developers + // early they changed things in an incompatible way. + panic(fmt.Sprintf("unknown attribute value type: %s", a.Value.Type())) + } + return true +} + +func equalExemplars[N int64 | float64](a, b metricdata.Exemplar[N], cfg config) (reasons []string) { + if !slices.EqualFunc(a.FilteredAttributes, b.FilteredAttributes, equalKeyValue) { + reasons = append(reasons, notEqualStr("FilteredAttributes", a.FilteredAttributes, b.FilteredAttributes)) + } + if !cfg.ignoreTimestamp { + if !a.Time.Equal(b.Time) { + reasons = append(reasons, notEqualStr("Time", a.Time.UnixNano(), b.Time.UnixNano())) + } + } + if !cfg.ignoreValue { + if a.Value != b.Value { + reasons = append(reasons, notEqualStr("Value", a.Value, b.Value)) + } + } + if !slices.Equal(a.SpanID, b.SpanID) { + reasons = append(reasons, notEqualStr("SpanID", a.SpanID, b.SpanID)) + } + if !slices.Equal(a.TraceID, b.TraceID) { + reasons = append(reasons, notEqualStr("TraceID", a.TraceID, b.TraceID)) + } + return reasons +} + +func diffSlices[T any](a, b []T, formatContext func(T) string, compare func(T, T) []string) string { + visited := make([]bool, len(b)) + var extraA []T + var extraB []T + + for i := range a { + found := false + for j := range b { + if visited[j] { + continue + } + if len(compare(a[i], b[j])) == 0 { + visited[j] = true + found = true + break + } + } + if !found { + extraA = append(extraA, a[i]) + } + } + + for j := range b { + if visited[j] { + continue + } + extraB = append(extraB, b[j]) + } + + if len(extraA) == 0 && len(extraB) == 0 { + return "" + } + + var msg bytes.Buffer + minLen := min(len(extraB), len(extraA)) + + for i := range minLen { + reasons := compare(extraA[i], extraB[i]) + _, _ = msg.WriteString(formatContext(extraA[i]) + ":\n") + for _, reason := range reasons { + // Indent reasons + lines := strings.SplitSeq(reason, "\n") + for line := range lines { + if line != "" { + _, _ = msg.WriteString("\t" + line + "\n") + } + } + } + } + + formatter := func(v T) string { + return fmt.Sprintf("%#v", v) + } + if len(extraA) > minLen { + _, _ = msg.WriteString("missing expected values:\n") + for i := minLen; i < len(extraA); i++ { + _, _ = msg.WriteString(formatter(extraA[i]) + "\n") + } + } + if len(extraB) > minLen { + _, _ = msg.WriteString("unexpected additional values:\n") + for i := minLen; i < len(extraB); i++ { + _, _ = msg.WriteString(formatter(extraB[i]) + "\n") + } + } + + return msg.String() +} + +func missingAttrStr(name string) string { + return "missing attribute " + name +} + +func hasAttributesExemplars[T int64 | float64]( + exemplar metricdata.Exemplar[T], + attrs ...attribute.KeyValue, +) (reasons []string) { + s := attribute.NewSet(exemplar.FilteredAttributes...) + for _, attr := range attrs { + val, ok := s.Value(attr.Key) + if !ok { + reasons = append(reasons, missingAttrStr(string(attr.Key))) + continue + } + if val != attr.Value { + reasons = append(reasons, notEqualStr(string(attr.Key), attr.Value.Emit(), val.Emit())) + } + } + return reasons +} + +func hasAttributesDataPoints[T int64 | float64]( + dp metricdata.DataPoint[T], + attrs ...attribute.KeyValue, +) (reasons []string) { + for _, attr := range attrs { + val, ok := dp.Attributes.Value(attr.Key) + if !ok { + reasons = append(reasons, missingAttrStr(string(attr.Key))) + continue + } + if val != attr.Value { + reasons = append(reasons, notEqualStr(string(attr.Key), attr.Value.Emit(), val.Emit())) + } + } + return reasons +} + +func hasAttributesGauge[T int64 | float64](gauge metricdata.Gauge[T], attrs ...attribute.KeyValue) (reasons []string) { + for n, dp := range gauge.DataPoints { + reas := hasAttributesDataPoints(dp, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("gauge datapoint %d attributes:\n", n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesSum[T int64 | float64](sum metricdata.Sum[T], attrs ...attribute.KeyValue) (reasons []string) { + for n, dp := range sum.DataPoints { + reas := hasAttributesDataPoints(dp, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("sum datapoint %d attributes:\n", n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesHistogramDataPoints[T int64 | float64]( + dp metricdata.HistogramDataPoint[T], + attrs ...attribute.KeyValue, +) (reasons []string) { + for _, attr := range attrs { + val, ok := dp.Attributes.Value(attr.Key) + if !ok { + reasons = append(reasons, missingAttrStr(string(attr.Key))) + continue + } + if val != attr.Value { + reasons = append(reasons, notEqualStr(string(attr.Key), attr.Value.Emit(), val.Emit())) + } + } + return reasons +} + +func hasAttributesHistogram[T int64 | float64]( + histogram metricdata.Histogram[T], + attrs ...attribute.KeyValue, +) (reasons []string) { + for n, dp := range histogram.DataPoints { + reas := hasAttributesHistogramDataPoints(dp, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("histogram datapoint %d attributes:\n", n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesExponentialHistogramDataPoints[T int64 | float64]( + dp metricdata.ExponentialHistogramDataPoint[T], + attrs ...attribute.KeyValue, +) (reasons []string) { + for _, attr := range attrs { + val, ok := dp.Attributes.Value(attr.Key) + if !ok { + reasons = append(reasons, missingAttrStr(string(attr.Key))) + continue + } + if val != attr.Value { + reasons = append(reasons, notEqualStr(string(attr.Key), attr.Value.Emit(), val.Emit())) + } + } + return reasons +} + +func hasAttributesExponentialHistogram[T int64 | float64]( + histogram metricdata.ExponentialHistogram[T], + attrs ...attribute.KeyValue, +) (reasons []string) { + for n, dp := range histogram.DataPoints { + reas := hasAttributesExponentialHistogramDataPoints(dp, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("histogram datapoint %d attributes:\n", n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesAggregation(agg metricdata.Aggregation, attrs ...attribute.KeyValue) (reasons []string) { + switch agg := agg.(type) { + case metricdata.Gauge[int64]: + reasons = hasAttributesGauge(agg, attrs...) + case metricdata.Gauge[float64]: + reasons = hasAttributesGauge(agg, attrs...) + case metricdata.Sum[int64]: + reasons = hasAttributesSum(agg, attrs...) + case metricdata.Sum[float64]: + reasons = hasAttributesSum(agg, attrs...) + case metricdata.Histogram[int64]: + reasons = hasAttributesHistogram(agg, attrs...) + case metricdata.Histogram[float64]: + reasons = hasAttributesHistogram(agg, attrs...) + case metricdata.ExponentialHistogram[int64]: + reasons = hasAttributesExponentialHistogram(agg, attrs...) + case metricdata.ExponentialHistogram[float64]: + reasons = hasAttributesExponentialHistogram(agg, attrs...) + case metricdata.Summary: + reasons = hasAttributesSummary(agg, attrs...) + default: + reasons = []string{fmt.Sprintf("unknown aggregation %T", agg)} + } + return reasons +} + +func hasAttributesMetrics(metrics metricdata.Metrics, attrs ...attribute.KeyValue) (reasons []string) { + reas := hasAttributesAggregation(metrics.Data, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("Metric %s:\n", metrics.Name)) + reasons = append(reasons, reas...) + } + return reasons +} + +func hasAttributesScopeMetrics(sm metricdata.ScopeMetrics, attrs ...attribute.KeyValue) (reasons []string) { + for n, metrics := range sm.Metrics { + reas := hasAttributesMetrics(metrics, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("ScopeMetrics %s Metrics %d:\n", sm.Scope.Name, n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesResourceMetrics(rm metricdata.ResourceMetrics, attrs ...attribute.KeyValue) (reasons []string) { + for n, sm := range rm.ScopeMetrics { + reas := hasAttributesScopeMetrics(sm, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("ResourceMetrics ScopeMetrics %d:\n", n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesSummary(summary metricdata.Summary, attrs ...attribute.KeyValue) (reasons []string) { + for n, dp := range summary.DataPoints { + reas := hasAttributesSummaryDataPoint(dp, attrs...) + if len(reas) > 0 { + reasons = append(reasons, fmt.Sprintf("summary datapoint %d attributes:\n", n)) + reasons = append(reasons, reas...) + } + } + return reasons +} + +func hasAttributesSummaryDataPoint(dp metricdata.SummaryDataPoint, attrs ...attribute.KeyValue) (reasons []string) { + for _, attr := range attrs { + val, ok := dp.Attributes.Value(attr.Key) + if !ok { + reasons = append(reasons, missingAttrStr(string(attr.Key))) + continue + } + if val != attr.Value { + reasons = append(reasons, notEqualStr(string(attr.Key), attr.Value.Emit(), val.Emit())) + } + } + return reasons +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 64eefdfb4..d6c9c4a4d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -764,6 +764,7 @@ go.opentelemetry.io/otel/sdk/metric/internal/aggregate go.opentelemetry.io/otel/sdk/metric/internal/observ go.opentelemetry.io/otel/sdk/metric/internal/reservoir go.opentelemetry.io/otel/sdk/metric/metricdata +go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest # go.opentelemetry.io/otel/trace v1.43.0 ## explicit; go 1.25.0 go.opentelemetry.io/otel/trace