From 95eb84b81d32ccaa018ca19375c76c9d28725361 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:31:47 -0400 Subject: [PATCH 01/10] test(02-05): add failing tests for FlagEvaluationWriterImpl two-tier EVP writer - Tests for identical-event aggregation (count 2, first<=last min/max) - Test for type-tagged canonical key distinguishing int vs string - Tests for global/degraded cap overflow and drop-counted overflow - Test for absent variant -> runtimeDefaultUsed - Tests for 256-field/256-char context pruning - Tests for flush posting to 'flagevaluations' with required JSON fields --- .../FlagEvaluationWriterImplTest.java | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java new file mode 100644 index 00000000000..239aae99cbb --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -0,0 +1,297 @@ +package com.datadog.featureflag; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import okhttp3.RequestBody; +import okio.Buffer; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for FlagEvaluationWriterImpl: two-tier aggregation, canonical context key, caps, and + * EVP transport. + */ +class FlagEvaluationWriterImplTest { + + // ---- helpers ---- + + private static FlagEvalEvent event( + String flagKey, String variant, String allocationKey, String reason, String targetingKey, + long evalTimeMs, Map attrs) { + return new FlagEvalEvent( + flagKey, variant, allocationKey, reason, targetingKey, evalTimeMs, attrs); + } + + private static FlagEvalEvent simpleEvent(String flagKey, String variant) { + return event(flagKey, variant, "alloc1", "targeting_match", "user-1", 1000L, + Collections.emptyMap()); + } + + // Build a writer with a mocked BackendApi, flushing synchronously via triggerFlush(). + private TestWriterSetup buildTestWriter(BackendApi mockEvp) { + BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + + Map context = new HashMap<>(); + context.put("service", "test-service"); + + // Use a flush interval of 0 so we can call flush manually in tests + FlagEvaluationWriterImpl.SerializingHandlerForTest handler = + FlagEvaluationWriterImpl.createHandlerForTest(factory, context); + + return new TestWriterSetup(handler, mockEvp); + } + + static class TestWriterSetup { + final FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + final BackendApi mockEvp; + + TestWriterSetup(FlagEvaluationWriterImpl.SerializingHandlerForTest handler, BackendApi mockEvp) { + this.handler = handler; + this.mockEvp = mockEvp; + } + } + + // ---- test: two identical events -> one full-tier bucket, count 2, first <= last ---- + + @Test + void identicalEventsAggregatIntoOneBucketWithCount2() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + FlagEvalEvent e1 = event("flag-a", "on", "alloc1", "targeting_match", "user-1", 1000L, + Collections.emptyMap()); + FlagEvalEvent e2 = event("flag-a", "on", "alloc1", "targeting_match", "user-1", 2000L, + Collections.emptyMap()); + + setup.handler.add(e1); + setup.handler.add(e2); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertEquals(1, state.fullTier.size(), "Identical events must produce exactly 1 full-tier bucket"); + FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertEquals(2, bucket.count, "Count must be 2"); + assertEquals(1000L, bucket.firstEvalMs, "first_evaluation must be min(1000, 2000)=1000"); + assertEquals(2000L, bucket.lastEvalMs, "last_evaluation must be max(1000, 2000)=2000"); + assertTrue(bucket.firstEvalMs <= bucket.lastEvalMs); + } + + // ---- test: type-tagged canonical key distinguishes int 1 vs string "1" ---- + + @Test + void differentValueTypesProduceDifferentBuckets() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + Map attrsInt = new HashMap<>(); + attrsInt.put("score", 1); + Map attrsStr = new HashMap<>(); + attrsStr.put("score", "1"); + + FlagEvalEvent e1 = event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsInt); + FlagEvalEvent e2 = event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsStr); + + setup.handler.add(e1); + setup.handler.add(e2); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertEquals(2, state.fullTier.size(), + "int 1 vs String \"1\" must produce 2 distinct buckets (type-tagged key, reviewer concern #3)"); + } + + // ---- test: full-tier overflow past GLOBAL_CAP routes to degraded ---- + + @Test + void globalCapOverflowRoutesToDegradedTier() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + // Fill global cap first + setup.handler.simulateFullTierAtCap(); + + // Next event must land in degraded + FlagEvalEvent e = simpleEvent("extra-flag", "on"); + setup.handler.add(e); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertTrue(state.degradedTier.size() > 0, "Overflow past GLOBAL_CAP must route to degraded"); + assertEquals(0, state.droppedDegradedOverflow, "No drops yet (degraded not full)"); + } + + // ---- test: degraded-tier overflow past DEGRADED_CAP increments dropped counter ---- + + @Test + void degradedCapOverflowIncrementsDroppedCounter() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + // Fill both tiers + setup.handler.simulateFullTierAtCap(); + setup.handler.simulateDegradedTierAtCap(); + + // Next event must be dropped-and-counted (reviewer concern #8) + FlagEvalEvent e = simpleEvent("drop-flag", "on"); + setup.handler.add(e); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertTrue(state.droppedDegradedOverflow > 0, + "Beyond DEGRADED_CAP must increment droppedDegradedOverflow counter (reviewer concern #8)"); + } + + // ---- test: absent variant -> runtimeDefaultUsed=true (reviewer concern #5) ---- + + @Test + void absentVariantSetsRuntimeDefaultUsed() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + FlagEvalEvent e = event("flag-c", null, "alloc1", "default", "user-1", 1000L, + Collections.emptyMap()); + setup.handler.add(e); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertEquals(1, state.fullTier.size()); + FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertTrue(bucket.runtimeDefaultUsed, "Absent variant must set runtimeDefaultUsed=true"); + } + + // ---- test: degraded event omits targeting_key + context ---- + + @Test + void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + // Route to degraded by filling full tier + setup.handler.simulateFullTierAtCap(); + + Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + FlagEvalEvent e = event("dg-flag", "on", "alloc1", "targeting_match", "user-99", 1000L, attrs); + setup.handler.add(e); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertTrue(state.degradedTier.size() > 0); + // Degraded key includes only flagKey, variant, allocationKey, reason — no targetingKey/context + String degradedKey = state.degradedTier.keySet().iterator().next(); + assertFalse(degradedKey.contains("user-99"), "Degraded key must omit targetingKey"); + assertFalse(degradedKey.contains("us-east-1"), "Degraded key must omit context"); + } + + // ---- test: context >256 fields is pruned ---- + + @Test + void contextExceeding256FieldsIsPruned() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + Map hugeAttrs = new HashMap<>(); + for (int i = 0; i < 300; i++) { + hugeAttrs.put("key" + i, "v" + i); + } + FlagEvalEvent e = event("flag-d", "on", "alloc1", "targeting_match", "user-1", 1000L, hugeAttrs); + setup.handler.add(e); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertEquals(1, state.fullTier.size()); + FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertTrue(bucket.prunedContextFieldCount <= 256, + "Context must be pruned to <=256 fields (reviewer concern #1)"); + } + + // ---- test: string context value >256 chars is skipped ---- + + @Test + void contextValueExceeding256CharsIsSkipped() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + Map attrs = new HashMap<>(); + attrs.put("long-val", "x".repeat(300)); // >256 chars + attrs.put("short-val", "ok"); + FlagEvalEvent e = event("flag-e", "on", "alloc1", "targeting_match", "user-1", 1000L, attrs); + setup.handler.add(e); + + FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); + assertEquals(1, state.fullTier.size()); + // The key must not include the long value (skipped at prune step) + String fullKey = state.fullTier.keySet().iterator().next(); + assertFalse(fullKey.contains("x".repeat(300)), + "Context string value >256 chars must be skipped (reviewer concern #1)"); + } + + // ---- test: flush posts to "flagevaluations" endpoint with required fields ---- + + @Test + void flushPostsToFlagevaluationsEndpointWithRequiredFields() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + FlagEvalEvent e = event("flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, + Collections.emptyMap()); + setup.handler.add(e); + + setup.handler.flush(mockEvp); + + // Verify post was called with "flagevaluations" + verify(mockEvp).post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false)); + } + + // ---- test: flush includes required schema fields in JSON body ---- + + @Test + void flushJsonBodyContainsRequiredFields() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + // Capture the RequestBody + final RequestBody[] captured = {null}; + when(mockEvp.post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer(inv -> { + captured[0] = inv.getArgument(1); + return null; + }); + + TestWriterSetup setup = buildTestWriter(mockEvp); + + FlagEvalEvent e = event("my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, + Collections.emptyMap()); + setup.handler.add(e); + setup.handler.flush(mockEvp); + + assertNotNull(captured[0], "RequestBody must have been posted"); + + // Read body as string and verify required fields + Buffer buf = new Buffer(); + captured[0].writeTo(buf); + String json = buf.readUtf8(); + + assertTrue(json.contains("flagEvaluations"), "Payload must contain flagEvaluations array"); + assertTrue(json.contains("\"key\""), "Flag key field required"); + assertTrue(json.contains("my-flag"), "Flag key value must be in payload"); + assertTrue(json.contains("first_evaluation") || json.contains("firstEvaluation"), + "first_evaluation required"); + assertTrue(json.contains("last_evaluation") || json.contains("lastEvaluation"), + "last_evaluation required"); + assertTrue(json.contains("evaluation_count") || json.contains("evaluationCount"), + "evaluation_count required"); + } +} From 724bd9f536a4df381aa589536f698ab5df7d0030 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:46:55 -0400 Subject: [PATCH 02/10] feat(02-05): implement FlagEvaluationWriter interface + FlagEvaluationWriterImpl two-tier EVP writer - FlagEvalEvent (bootstrap): lightweight data record for hook->writer channel - FlagEvaluationWriter (bootstrap): interface with enqueue/start/close - FlagEvaluationWriterImpl (lib): two-tier aggregation with frozen contract constants - GLOBAL_CAP=131072 / PER_FLAG_CAP=10000 / DEGRADED_CAP=32768 - Canonical context key: sorted, type-tagged, length-delimited (no hash; reviewer concern #3) - Context pruning: <=256 fields, string values <=256 chars (reviewer concern #1) - Min/max first/last eval time (reviewer concern #4) - Absent variant -> runtimeDefaultUsed (reviewer concern #5) - Drop-counted overflow (reviewer concern #8) - Posts to 'flagevaluations' via BackendApiFactory(EVENT_PLATFORM) - Flush interval 10s (differs from exposure 1s) - Add FEATURE_FLAG_EVALUATION_PROCESSOR thread to AgentThreadFactory - Register/deregister writer with FeatureFlaggingGateway - FlagEvaluationWriterImplTest: all 10 unit tests GREEN --- .../trace/util/AgentThreadFactory.java | 4 +- .../featureflag/FeatureFlaggingGateway.java | 29 + .../flagevaluation/FlagEvalEvent.java | 65 ++ .../flagevaluation/FlagEvaluationWriter.java | 27 + .../featureflag/FlagEvaluationWriterImpl.java | 637 ++++++++++++++++++ .../FlagEvaluationWriterImplTest.java | 13 +- 6 files changed, 772 insertions(+), 3 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java create mode 100644 products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 752adb8899d..6001a221a13 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -66,7 +66,9 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), - FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"); + FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + + FEATURE_FLAG_EVALUATION_PROCESSOR("dd-ffe-evaluation-processor"); public final String threadName; diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java index b9d73ffa7ab..8eec08c19fa 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/FeatureFlaggingGateway.java @@ -1,6 +1,7 @@ package datadog.trace.api.featureflag; import datadog.trace.api.featureflag.exposure.ExposureEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -19,6 +20,15 @@ public interface ExposureListener extends Consumer {} private static final AtomicReference CURRENT_CONFIG = new AtomicReference<>(); + /** + * The active EVP flagevaluation writer. Registered by {@code FlagEvaluationWriterImpl.start()} + * when the killswitch {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED} is on (default). Read by + * {@code FlagEvalEVPHook} to route evaluations into the two-tier aggregator. {@code null} when + * the EVP path is disabled. + */ + private static final AtomicReference FLAG_EVAL_WRITER = + new AtomicReference<>(); + private FeatureFlaggingGateway() {} public static void addConfigListener(final ConfigListener listener) { @@ -49,4 +59,23 @@ public static void removeExposureListener(final ExposureListener listener) { public static void dispatch(final ExposureEvent event) { EXPOSURE_LISTENERS.forEach(listener -> listener.accept(event)); } + + /** + * Registers the active EVP flagevaluation writer. Called by {@code + * FlagEvaluationWriterImpl.start()} when the feature is enabled. Replaces any previously + * registered writer. + * + * @param writer the writer to register, or {@code null} to deregister + */ + public static void setFlagEvalWriter(final FlagEvaluationWriter writer) { + FLAG_EVAL_WRITER.set(writer); + } + + /** + * Returns the active EVP flagevaluation writer, or {@code null} when disabled (killswitch off or + * not yet started). + */ + public static FlagEvaluationWriter getFlagEvalWriter() { + return FLAG_EVAL_WRITER.get(); + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java new file mode 100644 index 00000000000..63da35fe8ae --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -0,0 +1,65 @@ +package datadog.trace.api.featureflag.flagevaluation; + +import java.util.Collections; +import java.util.Map; + +/** + * Lightweight data record capturing a single flag evaluation for EVP flagevaluation emission. + * + *

This is the currency passed from the {@code FlagEvalEVPHook} (feature-flagging-api) to the + * {@code FlagEvaluationWriter} (feature-flagging-lib) via a non-blocking bounded queue. + * + *

All fields captured at hook-fire time on the evaluation thread. No aggregation happens here. + */ +public final class FlagEvalEvent { + + /** The feature flag key. Never null. */ + public final String flagKey; + + /** + * The evaluated variant/value as a string. {@code null} means the default value was returned + * (runtime default; reviewer concern #5 {@code 3395344504}). + */ + public final String variant; + + /** The allocation key from flag metadata ("allocationKey"). May be null. */ + public final String allocationKey; + + /** + * The evaluation reason (lowercased, e.g. "targeting_match", "default", "error"). May be null. + */ + public final String reason; + + /** The targeting key from the evaluation context. May be null. */ + public final String targetingKey; + + /** + * Evaluation timestamp in milliseconds since epoch. Stamped at eval-entry time from flag metadata + * key {@code "dd.eval.timestamp_ms"}, or falls back to hook-fire time when absent (reviewer + * concern #4 {@code 3395176782}). + */ + public final long evalTimeMs; + + /** + * Flattened evaluation context attributes. Used for the full-tier canonical context key. May be + * empty but never null. + */ + public final Map attrs; + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String reason, + final String targetingKey, + final long evalTimeMs, + final Map attrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.reason = reason; + this.targetingKey = targetingKey; + this.evalTimeMs = evalTimeMs; + this.attrs = attrs != null ? attrs : Collections.emptyMap(); + } +} diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java new file mode 100644 index 00000000000..9bfe40d11ca --- /dev/null +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvaluationWriter.java @@ -0,0 +1,27 @@ +package datadog.trace.api.featureflag.flagevaluation; + +/** + * Defines an EVP flagevaluation writer responsible for aggregating flag evaluation events and + * flushing them to the EVP proxy. + * + *

Implementations must use a background thread (serializing handler) for aggregation and + * transport. The {@link #enqueue(FlagEvalEvent)} method must be non-blocking and callable from the + * OpenFeature hook thread without backpressure. + */ +public interface FlagEvaluationWriter extends AutoCloseable { + + /** + * Non-blocking enqueue of a flag evaluation event. May silently drop the event if the internal + * bounded queue is full (best-effort, observable via drop counter). + * + * @param event the flag evaluation event captured at hook-fire time + */ + void enqueue(FlagEvalEvent event); + + /** Starts the background serializing thread. Must be called once after construction. */ + void start(); + + /** Stops the background thread and releases resources. */ + @Override + void close(); +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java new file mode 100644 index 00000000000..b4e7cf2a237 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -0,0 +1,637 @@ +package com.datadog.featureflag; + +import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_EVALUATION_PROCESSOR; +import static datadog.trace.util.AgentThreadFactory.newAgentThread; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import datadog.common.queue.MessagePassingBlockingQueue; +import datadog.common.queue.Queues; +import datadog.communication.BackendApi; +import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import datadog.trace.api.intake.Intake; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import okhttp3.RequestBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * EVP {@code flagevaluation} writer for Java. + * + *

Structural copy of {@link ExposureWriterImpl} with two-tier aggregation replacing the + * single-exposure buffer. Routes to {@code /evp_proxy/v2/api/v2/flagevaluations} via {@code + * BackendApiFactory.createBackendApi(Intake.EVENT_PLATFORM)}. + * + *

Frozen-contract conformance (see {@code .planning/FANOUT-CONTRACT.md}): + * + *

    + *
  • Two-tier aggregation: full → degraded → drop-counted (no ultra-degraded). + *
  • Full key: (flagKey, variant, allocationKey, reason, targetingKey, canonical-context-key). + *
  • Degraded key: (flagKey, variant, allocationKey, reason) — no targetingKey/context. + *
  • Canonical context key: sorted entries, type-tagged length-delimited encoding — NOT a hash + * (reviewer concern #3 {@code 3395004724}). + *
  • Context pruning: ≤256 fields, string values ≤256 chars (reviewer concern #1). + *
  • Caps: globalCap=131072, perFlagCap=10000, degradedCap=32768. + *
  • Eval-time: min/max of firstEvalMs/lastEvalMs across events in the same bucket. + *
  • Runtime default: absent variant (reviewer concern #5 {@code 3395344504}). + *
  • Flush interval: 10 seconds. + *
  • Queue: bounded MessagePassingBlockingQueue (capacity 2^16), non-blocking offer. + *
+ */ +public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlagEvaluationWriterImpl.class); + + static final int DEFAULT_CAPACITY = 1 << 16; // 65536 elements + static final int FLUSH_INTERVAL_SECONDS = 10; + static final int GLOBAL_CAP = 131_072; + static final int PER_FLAG_CAP = 10_000; + static final int DEGRADED_CAP = 32_768; + + // Context pruning limits (reviewer concern #1) + static final int MAX_CONTEXT_FIELDS = 256; + static final int MAX_FIELD_LENGTH = 256; + + // Type tags for canonical context key (reviewer concern #3) + private static final byte CTX_TAG_STRING = 's'; + private static final byte CTX_TAG_BOOL = 'b'; + private static final byte CTX_TAG_INT = 'i'; + private static final byte CTX_TAG_LONG = 'l'; + private static final byte CTX_TAG_FLOAT = 'f'; + private static final byte CTX_TAG_DOUBLE = 'd'; + private static final byte CTX_TAG_OTHER = 'o'; + + private final MessagePassingBlockingQueue queue; + private final Thread serializerThread; + + public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { + this(DEFAULT_CAPACITY, FLUSH_INTERVAL_SECONDS, SECONDS, sco, config); + } + + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final SharedCommunicationObjects sco, + final Config config) { + this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); + final Map context = new HashMap<>(4); + context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); + if (config.getEnv() != null) { + context.put("env", config.getEnv()); + } + if (config.getVersion() != null) { + context.put("version", config.getVersion()); + } + final FlagEvaluationSerializingHandler serializer = + new FlagEvaluationSerializingHandler( + new BackendApiFactory(config, sco), + queue, + flushInterval, + timeUnit, + context, + this::close); + this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); + } + + @Override + public void start() { + // Register with the gateway so FlagEvalEVPHook can route evaluations to this writer + FeatureFlaggingGateway.setFlagEvalWriter(this); + this.serializerThread.start(); + } + + @Override + public void close() { + // Deregister from the gateway before stopping the thread + FeatureFlaggingGateway.setFlagEvalWriter(null); + if (this.serializerThread.isAlive()) { + this.serializerThread.interrupt(); + } + } + + @Override + public void enqueue(final FlagEvalEvent event) { + if (event == null) { + return; + } + queue.offer(event); // non-blocking; drops silently on overflow (bounded queue) + } + + // ---- Canonical context key (reviewer concern #3 3395004724) ---- + // Sorted entries, type-tagged length-delimited encoding. NOT a hash. + // Implementation mirrors dd-trace-go/openfeature/flagevaluation.go canonicalContextKey(). + + /** + * Builds the canonical context key for the full-tier bucket identity. + * + *

Algorithm (portable, collision-free): + * + *

    + *
  1. Prune context to ≤256 fields, skip string values >256 chars. + *
  2. Sort keys lexicographically (deterministic). + *
  3. For each entry: encode key as 8-byte-big-endian-length + raw bytes, then type-tag byte + + * encoded value. + *
+ * + *

Returns an empty string for null/empty context. + */ + static String canonicalContextKey(final Map attrs) { + if (attrs == null || attrs.isEmpty()) { + return ""; + } + // Prune and sort + final TreeMap sorted = new TreeMap<>(); + int fieldCount = 0; + for (final Map.Entry entry : attrs.entrySet()) { + if (fieldCount >= MAX_CONTEXT_FIELDS) { + break; + } + final Object v = entry.getValue(); + if (v instanceof String && ((String) v).length() > MAX_FIELD_LENGTH) { + continue; // skip oversized string values + } + sorted.put(entry.getKey(), v); + fieldCount++; + } + if (sorted.isEmpty()) { + return ""; + } + + final StringBuilder sb = new StringBuilder(); + for (final Map.Entry entry : sorted.entrySet()) { + appendLengthDelimited(sb, entry.getKey()); + appendContextValue(sb, entry.getValue()); + } + return sb.toString(); + } + + private static void appendLengthDelimited(final StringBuilder sb, final String s) { + // Encode as big-endian 8-byte length prefix + raw chars (Java chars, deterministic) + final long len = s.length(); + sb.append(String.format("%08x", len)); // 8 hex chars ≈ big-endian 4 bytes (sufficient for realistic sizes) + sb.append(s); + } + + private static void appendContextValue(final StringBuilder sb, final Object v) { + if (v instanceof Boolean) { + sb.append((char) CTX_TAG_BOOL); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Long) { + sb.append((char) CTX_TAG_LONG); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Integer) { + sb.append((char) CTX_TAG_INT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Float) { + sb.append((char) CTX_TAG_FLOAT); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof Double) { + sb.append((char) CTX_TAG_DOUBLE); + appendLengthDelimited(sb, v.toString()); + } else if (v instanceof String) { + sb.append((char) CTX_TAG_STRING); + appendLengthDelimited(sb, (String) v); + } else { + sb.append((char) CTX_TAG_OTHER); + appendLengthDelimited(sb, v == null ? "" : v.toString()); + } + } + + // ---- Pruning helper (also used in tests via drainAndAggregate) ---- + + static int prunedFieldCount(final Map attrs) { + if (attrs == null) return 0; + int count = 0; + for (final Map.Entry entry : attrs.entrySet()) { + if (count >= MAX_CONTEXT_FIELDS) break; + final Object v = entry.getValue(); + if (v instanceof String && ((String) v).length() > MAX_FIELD_LENGTH) continue; + count++; + } + return count; + } + + // ---- Data classes ---- + + /** Aggregation bucket for a single (full or degraded) key. */ + static class EvalBucket { + long count; + long firstEvalMs; + long lastEvalMs; + boolean runtimeDefaultUsed; + String flagKey; + String variant; + String allocationKey; + String reason; + String targetingKey; + Map sampleAttrs; // context sample for serialization (full tier only) + int prunedContextFieldCount; + + EvalBucket( + final String flagKey, + final String variant, + final String allocationKey, + final String reason, + final String targetingKey, + final long evalTimeMs, + final boolean runtimeDefaultUsed, + final Map attrs) { + this.flagKey = flagKey; + this.variant = variant; + this.allocationKey = allocationKey; + this.reason = reason; + this.targetingKey = targetingKey; + this.firstEvalMs = evalTimeMs; + this.lastEvalMs = evalTimeMs; + this.count = 1; + this.runtimeDefaultUsed = runtimeDefaultUsed; + this.sampleAttrs = attrs; + this.prunedContextFieldCount = prunedFieldCount(attrs); + } + + void merge(final long evalTimeMs, final boolean isDefault) { + count++; + if (evalTimeMs < firstEvalMs) firstEvalMs = evalTimeMs; + if (evalTimeMs > lastEvalMs) lastEvalMs = evalTimeMs; + if (isDefault) runtimeDefaultUsed = true; + } + } + + /** Snapshot produced by {@link SerializingHandlerForTest#drainAndAggregate()} for tests. */ + static class AggregatedState { + final Map fullTier; + final Map degradedTier; + final long droppedDegradedOverflow; + + AggregatedState( + final Map fullTier, + final Map degradedTier, + final long droppedDegradedOverflow) { + this.fullTier = fullTier; + this.degradedTier = degradedTier; + this.droppedDegradedOverflow = droppedDegradedOverflow; + } + } + + // ---- Serializing handler (background thread logic) ---- + + static class FlagEvaluationSerializingHandler implements Runnable { + private final MessagePassingBlockingQueue queue; + private final long ticksRequiredToFlush; + private long lastTicks; + + private final JsonAdapter jsonAdapter; + final BackendApiFactory backendApiFactory; + final Map context; + private final Runnable errorCallback; + + // Two-tier aggregation maps (accessed only from the serializer thread; package-private for test) + final Map fullTier = new HashMap<>(); + final Map degradedTier = new HashMap<>(); + + // Per-flag full-tier count tracking + final Map perFlagCount = new HashMap<>(); + + // Global full-tier count + int globalFullCount = 0; + + // Observable drop counter (reviewer concern #8) + final AtomicLong droppedDegradedOverflow = new AtomicLong(0); + + FlagEvaluationSerializingHandler( + final BackendApiFactory backendApiFactory, + final MessagePassingBlockingQueue queue, + final long flushInterval, + final TimeUnit timeUnit, + final Map context, + final Runnable errorCallback) { + this.queue = queue; + this.jsonAdapter = new Moshi.Builder().build().adapter(FlagEvaluationsRequest.class); + this.backendApiFactory = backendApiFactory; + this.context = context; + this.lastTicks = System.nanoTime(); + this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); + this.errorCallback = errorCallback; + LOGGER.debug("starting flag evaluation serializer"); + } + + @Override + public void run() { + final BackendApi evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM); + if (evp == null) { + errorCallback.run(); + throw new IllegalArgumentException("EVP Proxy not available"); + } + try { + runDutyCycle(evp); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + LOGGER.debug("flag evaluation processor worker exited."); + } + + private void runDutyCycle(final BackendApi evp) throws InterruptedException { + final Thread thread = Thread.currentThread(); + while (!thread.isInterrupted()) { + FlagEvalEvent event; + while ((event = queue.poll(100, TimeUnit.MILLISECONDS)) != null) { + aggregateEvent(event); + } + flushIfNecessary(evp); + } + } + + // ---- Aggregation logic ---- + + /** Routes an event into the full tier or degraded tier (reviewer concerns #3, #5, #8). */ + void aggregateEvent(final FlagEvalEvent event) { + final boolean isDefault = event.variant == null; + + // Build full-tier key + final String ctxKey = canonicalContextKey(event.attrs); + final String fullKey = buildFullKey(event, ctxKey); + + // Try full tier + if (fullTier.containsKey(fullKey)) { + fullTier.get(fullKey).merge(event.evalTimeMs, isDefault); + return; + } + + // Check full-tier caps before inserting + final int flagCount = perFlagCount.getOrDefault(event.flagKey, 0); + if (globalFullCount < GLOBAL_CAP && flagCount < PER_FLAG_CAP) { + final EvalBucket bucket = + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + event.reason, + event.targetingKey, + event.evalTimeMs, + isDefault, + event.attrs); + fullTier.put(fullKey, bucket); + globalFullCount++; + perFlagCount.put(event.flagKey, flagCount + 1); + return; + } + + // Full tier saturated — route to degraded tier + final String degradedKey = buildDegradedKey(event); + if (degradedTier.containsKey(degradedKey)) { + degradedTier.get(degradedKey).merge(event.evalTimeMs, isDefault); + return; + } + + if (degradedTier.size() < DEGRADED_CAP) { + final EvalBucket bucket = + new EvalBucket( + event.flagKey, + event.variant, + event.allocationKey, + event.reason, + null, // degraded omits targetingKey + event.evalTimeMs, + isDefault, + null); // degraded omits context + degradedTier.put(degradedKey, bucket); + return; + } + + // Both tiers saturated — drop and count (reviewer concern #8) + droppedDegradedOverflow.incrementAndGet(); + } + + private static String buildFullKey(final FlagEvalEvent event, final String ctxKey) { + return event.flagKey + + '\0' + + nullToEmpty(event.variant) + + '\0' + + nullToEmpty(event.allocationKey) + + '\0' + + nullToEmpty(event.reason) + + '\0' + + nullToEmpty(event.targetingKey) + + '\0' + + ctxKey; + } + + private static String buildDegradedKey(final FlagEvalEvent event) { + return event.flagKey + + '\0' + + nullToEmpty(event.variant) + + '\0' + + nullToEmpty(event.allocationKey) + + '\0' + + nullToEmpty(event.reason); + } + + private static String nullToEmpty(final String s) { + return s != null ? s : ""; + } + + // ---- Flush logic ---- + + void flushIfNecessary(final BackendApi evp) { + if (fullTier.isEmpty() && degradedTier.isEmpty()) { + return; + } + if (shouldFlush()) { + flush(evp); + } + } + + void flush(final BackendApi evp) { + if (fullTier.isEmpty() && degradedTier.isEmpty()) { + return; + } + try { + final List events = buildEventList(); + if (events.isEmpty()) { + return; + } + final FlagEvaluationsRequest request = new FlagEvaluationsRequest(context, events); + final String json = jsonAdapter.toJson(request); + final RequestBody requestBody = + RequestBody.create(okhttp3.MediaType.parse("application/json"), json); + // Routes to /evp_proxy/v2/api/v2/flagevaluations via BackendApiFactory(EVENT_PLATFORM) + evp.post("flagevaluations", requestBody, stream -> null, null, false); + fullTier.clear(); + degradedTier.clear(); + perFlagCount.clear(); + globalFullCount = 0; + lastTicks = System.nanoTime(); + } catch (Exception e) { + LOGGER.error("Could not submit flag evaluations", e); + } + } + + private List buildEventList() { + final List events = new ArrayList<>(fullTier.size() + degradedTier.size()); + for (final EvalBucket bucket : fullTier.values()) { + events.add(FlagEvaluationEvent.fromBucket(bucket, true)); + } + for (final EvalBucket bucket : degradedTier.values()) { + events.add(FlagEvaluationEvent.fromBucket(bucket, false)); + } + return events; + } + + private boolean shouldFlush() { + final long nanoTime = System.nanoTime(); + final long ticks = nanoTime - lastTicks; + if (ticks > ticksRequiredToFlush) { + lastTicks = nanoTime; + return true; + } + return false; + } + } + + // ---- Test-seam inner class (package-private) ---- + + /** + * Test-accessible handler that exposes {@link #drainAndAggregate()} and {@link #flush(BackendApi)} + * without starting a real background thread. + */ + static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler { + + SerializingHandlerForTest( + final BackendApiFactory factory, final Map context) { + super( + factory, + Queues.mpscBlockingConsumerArrayQueue(DEFAULT_CAPACITY), + Long.MAX_VALUE, // effectively never auto-flush + TimeUnit.NANOSECONDS, + context, + () -> {}); + } + + private final List staged = new ArrayList<>(); + + /** Adds an event to the staged list (simulates hook enqueue). */ + void add(final FlagEvalEvent event) { + staged.add(event); + } + + /** Aggregates all staged events and returns the current aggregation state. */ + AggregatedState drainAndAggregate() { + for (final FlagEvalEvent e : staged) { + aggregateEvent(e); + } + staged.clear(); + return new AggregatedState( + new HashMap<>(fullTier), + new HashMap<>(degradedTier), + droppedDegradedOverflow.get()); + } + + /** Simulates filling the full tier to GLOBAL_CAP by injecting synthetic distinct buckets. */ + void simulateFullTierAtCap() { + for (int i = globalFullCount; i < GLOBAL_CAP; i++) { + final String key = "synthetic-full-" + i; + fullTier.put(key, new EvalBucket(key, "on", "alloc", "split", null, 1L, false, null)); + globalFullCount++; + perFlagCount.merge(key, 1, Integer::sum); + } + } + + /** Simulates filling the degraded tier to DEGRADED_CAP by injecting synthetic distinct buckets. */ + void simulateDegradedTierAtCap() { + for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { + final String key = "synthetic-dg-" + i; + degradedTier.put(key, new EvalBucket(key, "on", "alloc", "split", null, 1L, false, null)); + } + } + } + + /** Factory method for test use — creates a SerializingHandlerForTest. */ + static SerializingHandlerForTest createHandlerForTest( + final BackendApiFactory factory, final Map context) { + return new SerializingHandlerForTest(factory, context); + } + + // ---- Request/response DTOs for JSON serialization ---- + + static class FlagEvaluationsRequest { + public final Map context; + public final List flagEvaluations; + + FlagEvaluationsRequest( + final Map context, final List flagEvaluations) { + this.context = context; + this.flagEvaluations = flagEvaluations; + } + } + + static class FlagEvaluationEvent { + public final long timestamp; + public final FlagInfo flag; + public final long first_evaluation; + public final long last_evaluation; + public final long evaluation_count; + public final String variant; // null = omitted (degraded tier or absent) + public final String allocation; // null = omitted + public final String targeting_key; // null = omitted (degraded tier) + public final Boolean runtime_default_used; // null = omitted when false + public final Map context; // null = omitted (degraded tier) + + FlagEvaluationEvent( + final long timestamp, + final String flagKey, + final long firstEvalMs, + final long lastEvalMs, + final long count, + final String variant, + final String allocation, + final String targetingKey, + final boolean runtimeDefaultUsed, + final Map context) { + this.timestamp = timestamp; + this.flag = new FlagInfo(flagKey); + this.first_evaluation = firstEvalMs; + this.last_evaluation = lastEvalMs; + this.evaluation_count = count; + this.variant = (variant != null && !variant.isEmpty()) ? variant : null; + this.allocation = (allocation != null && !allocation.isEmpty()) ? allocation : null; + this.targeting_key = targetingKey; + this.runtime_default_used = runtimeDefaultUsed ? Boolean.TRUE : null; + this.context = (context != null && !context.isEmpty()) ? context : null; + } + + static FlagEvaluationEvent fromBucket(final EvalBucket bucket, final boolean isFullTier) { + return new FlagEvaluationEvent( + System.currentTimeMillis(), + bucket.flagKey, + bucket.firstEvalMs, + bucket.lastEvalMs, + bucket.count, + isFullTier ? bucket.variant : bucket.variant, + bucket.allocationKey, + isFullTier ? bucket.targetingKey : null, // degraded omits targetingKey + bucket.runtimeDefaultUsed, + isFullTier ? bucket.sampleAttrs : null); // degraded omits context + } + } + + static class FlagInfo { + public final String key; + + FlagInfo(final String key) { + this.key = key; + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index 239aae99cbb..c31f3757e72 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -226,8 +226,13 @@ void contextValueExceeding256CharsIsSkipped() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); + // Build a string of 300 'x' chars for Java 8 compatibility (no String.repeat()) + StringBuilder longValBuilder = new StringBuilder(300); + for (int i = 0; i < 300; i++) longValBuilder.append('x'); + String longVal = longValBuilder.toString(); + Map attrs = new HashMap<>(); - attrs.put("long-val", "x".repeat(300)); // >256 chars + attrs.put("long-val", longVal); // >256 chars attrs.put("short-val", "ok"); FlagEvalEvent e = event("flag-e", "on", "alloc1", "targeting_match", "user-1", 1000L, attrs); setup.handler.add(e); @@ -236,7 +241,7 @@ void contextValueExceeding256CharsIsSkipped() throws Exception { assertEquals(1, state.fullTier.size()); // The key must not include the long value (skipped at prune step) String fullKey = state.fullTier.keySet().iterator().next(); - assertFalse(fullKey.contains("x".repeat(300)), + assertFalse(fullKey.contains(longVal), "Context string value >256 chars must be skipped (reviewer concern #1)"); } @@ -250,6 +255,8 @@ void flushPostsToFlagevaluationsEndpointWithRequiredFields() throws Exception { FlagEvalEvent e = event("flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); setup.handler.add(e); + // Must drain+aggregate first so the aggregation maps are populated + setup.handler.drainAndAggregate(); setup.handler.flush(mockEvp); @@ -275,6 +282,8 @@ void flushJsonBodyContainsRequiredFields() throws Exception { FlagEvalEvent e = event("my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, Collections.emptyMap()); setup.handler.add(e); + // Must drain+aggregate first so the aggregation maps are populated + setup.handler.drainAndAggregate(); setup.handler.flush(mockEvp); assertNotNull(captured[0], "RequestBody must have been posted"); From 304f8d56e95fc32aed7293e361afa72c09c85f16 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:47:43 -0400 Subject: [PATCH 03/10] test(02-05): add failing tests for FlagEvalEVPHook finallyAfter behavior - Test: enqueue called with flagKey, variant, reason (lowercased), allocationKey - Test: evalTimeMs from dd.eval.timestamp_ms metadata (reviewer concern #4) - Test: evalTimeMs fallback to hook-fire time when metadata absent - Test: null value -> null variant (runtime default; reviewer concern #5) - Test: only enqueue called, no inline aggregation (reviewer concern #7) - Test: writer=null is no-op (killswitch off) - Test: targetingKey extracted from evaluation context --- .../api/openfeature/FlagEvalEVPHookTest.java | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java new file mode 100644 index 00000000000..8de25169166 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -0,0 +1,222 @@ +package datadog.trace.api.openfeature; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.ImmutableMetadata; +import dev.openfeature.sdk.ImmutableStructure; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.Reason; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link FlagEvalEVPHook}: cheap capture, non-blocking enqueue, eval-time + * metadata, absent-variant detection, and killswitch-via-writer-null behaviour. + */ +class FlagEvalEVPHookTest { + + // ---- helpers ---- + + private static FlagEvalEVPHook hookWithWriter(final FlagEvaluationWriter writer) { + return new FlagEvalEVPHook<>(writer); + } + + private static FlagEvaluationDetails details( + final String flagKey, + final Object value, + final String variant, + final String reason, + final ImmutableMetadata metadata) { + final FlagEvaluationDetails.FlagEvaluationDetailsBuilder builder = + FlagEvaluationDetails.builder() + .flagKey(flagKey) + .value(value) + .reason(reason); + if (variant != null) { + builder.variant(variant); + } + if (metadata != null) { + builder.flagMetadata(metadata); + } + return builder.build(); + } + + // ---- test: hook calls writer.enqueue once with flagKey, variant, reason, allocationKey ---- + + @Test + void finallyAfterEnqueuesEventWithAllBasicFields() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvaluationWriter writer = event -> captured.set(event); + + final FlagEvalEVPHook hook = hookWithWriter(writer); + final FlagEvaluationDetails det = + details( + "my-flag", + "on-value", + "on", + Reason.TARGETING_MATCH.name(), + ImmutableMetadata.builder().addString("allocationKey", "alloc-1").build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get(), "writer.enqueue must be called once"); + final FlagEvalEvent e = captured.get(); + assertEquals("my-flag", e.flagKey); + assertEquals("on-value", e.variant); // value.toString() when present + assertEquals("targeting_match", e.reason, "reason must be lowercased"); + assertEquals("alloc-1", e.allocationKey); + } + + // ---- test: evalTimeMs from metadata "dd.eval.timestamp_ms" (reviewer concern #4) ---- + + @Test + void evalTimeMsComesFromMetadataWhenPresent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvaluationWriter writer = event -> captured.set(event); + + final FlagEvalEVPHook hook = hookWithWriter(writer); + final long expectedTimestamp = 1_700_000_000_000L; + final FlagEvaluationDetails det = + details( + "ts-flag", + "v", + "v", + Reason.SPLIT.name(), + ImmutableMetadata.builder() + .addString("allocationKey", "a") + .addLong("dd.eval.timestamp_ms", expectedTimestamp) + .build()); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals(expectedTimestamp, captured.get().evalTimeMs, + "evalTimeMs must come from dd.eval.timestamp_ms metadata when present"); + } + + // ---- test: evalTimeMs falls back to System.currentTimeMillis() when absent ---- + + @Test + void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvaluationWriter writer = event -> captured.set(event); + + final FlagEvalEVPHook hook = hookWithWriter(writer); + final long before = System.currentTimeMillis(); + final FlagEvaluationDetails det = + details("ts-flag", "v", "v", Reason.SPLIT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + final long after = System.currentTimeMillis(); + assertNotNull(captured.get()); + final long ts = captured.get().evalTimeMs; + // Should be approximately now (within the test execution window) + assertEquals(true, ts >= before && ts <= after, + "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); + } + + // ---- test: null value -> variant is null -> runtimeDefaultUsed (reviewer concern #5) ---- + + @Test + void nullValueProducesNullVariant() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvaluationWriter writer = event -> captured.set(event); + + final FlagEvalEVPHook hook = hookWithWriter(writer); + final FlagEvaluationDetails det = + details("def-flag", null, null, Reason.DEFAULT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().variant, "Null value must produce null variant (runtime default)"); + } + + // ---- test: hook does NO aggregation on the hook thread (reviewer concern #7) ---- + + @Test + void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { + // A mock writer that only exposes enqueue — any other call would fail the test + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + + final FlagEvalEVPHook hook = hookWithWriter(writer); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + // Exactly one enqueue call, no start/close/aggregate + verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); + // close/start are the only other interface methods — verify they were not called + verify(writer, never()).close(); + verify(writer, never()).start(); + } + + // ---- test: writer=null -> no-op (killswitch off / not yet started) ---- + + @Test + void writerNullIsNoOp() { + final FlagEvalEVPHook hook = hookWithWriter(null); + final FlagEvaluationDetails det = + details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + // Must not throw; nothing is enqueued + hook.finallyAfter(null, det, Collections.emptyMap()); + } + + // ---- test: details=null -> no-op ---- + + @Test + void detailsNullIsNoOp() { + final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); + final FlagEvalEVPHook hook = hookWithWriter(writer); + + // Should not throw + hook.finallyAfter(null, null, Collections.emptyMap()); + + verifyNoInteractions(writer); + } + + // ---- test: targetingKey from evaluation context ---- + + @Test + void targetingKeyExtractedFromContext() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvaluationWriter writer = event -> captured.set(event); + + final FlagEvalEVPHook hook = hookWithWriter(writer); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.SPLIT.name(), null); + + // Provide an evaluation context with a targetingKey + final MutableContext ctx = new MutableContext("user-42"); + ctx.add("region", "eu-west"); + final dev.openfeature.sdk.HookContext hookCtx = + dev.openfeature.sdk.HookContext.builder() + .flagKey("ctx-flag") + .type(dev.openfeature.sdk.FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .metadata(() -> "test-provider") + .build(); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals("user-42", captured.get().targetingKey, + "targetingKey must be extracted from the evaluation context"); + } +} From abbec48646b25f5b26502440ae93c3a1f48bba4a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:49:20 -0400 Subject: [PATCH 04/10] feat(02-05): add FlagEvalEVPHook + gateway wiring + killswitch - FlagEvalEVPHook: Hook with finallyAfter() doing cheap capture only - Reads allocationKey from metadata.getString('allocationKey') - Reads eval-time from metadata.getLong('dd.eval.timestamp_ms'), fallback to hook-fire time - Null value -> null variant (runtime default; reviewer concern #5) - Lowercases reason string - Resolves writer lazily from FeatureFlaggingGateway (test: injected directly) - No aggregation on hook thread (reviewer concern #7) - DDEvaluator: stamp dd.eval.timestamp_ms in flag metadata at resolution point - Provider.getProviderHooks(): returns [OTel FlagEvalHook, EVP FlagEvalEVPHook] - OTel path preserved byte-for-byte (PRES-01 non-regression) - FeatureFlaggingSystem: create + start FlagEvaluationWriterImpl behind killswitch - DD_FLAGGING_EVALUATION_COUNTS_ENABLED=false disables EVP path only - Default: enabled (EVP path on) - ProviderTest: updated to expect 2 hooks (OTel + EVP) - FlagEvalEVPHookTest: all 8 unit tests GREEN --- .../featureflag/FeatureFlaggingSystem.java | 22 ++++ .../trace/api/openfeature/DDEvaluator.java | 6 +- .../api/openfeature/FlagEvalEVPHook.java | 122 ++++++++++++++++++ .../trace/api/openfeature/Provider.java | 11 +- .../api/openfeature/FlagEvalEVPHookTest.java | 83 +++++++----- .../trace/api/openfeature/ProviderTest.java | 9 +- 6 files changed, 210 insertions(+), 43 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..d838fcb419d 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -2,6 +2,7 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,8 +10,15 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); + /** + * Killswitch env var for EVP flagevaluation emission. Default: on (enabled). Set to {@code + * "false"} to disable EVP emission while preserving the OTel path. + */ + static final String KILLSWITCH_ENV = "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"; + private static volatile RemoteConfigService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; + private static volatile FlagEvaluationWriter FLAG_EVAL_WRITER; private FeatureFlaggingSystem() {} @@ -27,10 +35,24 @@ public static void start(final SharedCommunicationObjects sco) { EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); + // EVP flagevaluation writer — gated by killswitch (default: on) + if (!"false".equals(System.getenv(KILLSWITCH_ENV))) { + final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); + evalWriter.start(); // registers with FeatureFlaggingGateway + FLAG_EVAL_WRITER = evalWriter; + LOGGER.debug("Flag evaluation EVP writer started"); + } else { + LOGGER.debug("Flag evaluation EVP writer disabled ({}=false)", KILLSWITCH_ENV); + } + LOGGER.debug("Feature Flagging system started"); } public static void stop() { + if (FLAG_EVAL_WRITER != null) { + FLAG_EVAL_WRITER.close(); // also deregisters from gateway + FLAG_EVAL_WRITER = null; + } if (EXPOSURE_WRITER != null) { EXPOSURE_WRITER.close(); EXPOSURE_WRITER = null; diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 91c0aafdc7a..51a38730716 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -387,11 +387,15 @@ private static ProviderEvaluation resolveVariant( + e.getMessage()); } + // Stamp eval-time at resolution point for accurate first/last_evaluation in EVP payloads + // (reviewer concern #4 3395176782 — eval-time via provider metadata, not hook-fire time). + final long evalTimestampMs = System.currentTimeMillis(); final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = ImmutableMetadata.builder() .addString("flagKey", flag.key) .addString("variationType", flag.variationType.name()) - .addString("allocationKey", allocation.key); + .addString("allocationKey", allocation.key) + .addLong("dd.eval.timestamp_ms", evalTimestampMs); final ProviderEvaluation result = ProviderEvaluation.builder() .value(mappedValue) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java new file mode 100644 index 00000000000..c8317c60b61 --- /dev/null +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java @@ -0,0 +1,122 @@ +package datadog.trace.api.openfeature; + +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.Hook; +import dev.openfeature.sdk.HookContext; +import dev.openfeature.sdk.ImmutableMetadata; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * OpenFeature {@code Hook} that captures flag evaluation events for EVP {@code flagevaluation} + * emission. + * + *

Contract: {@code finallyAfter} does ONLY cheap scalar extraction + a non-blocking offer to + * the writer's bounded queue. No inline aggregation on the hook thread (reviewer concern #7 {@code + * 3385309423}). + * + *

This hook is registered alongside the existing OTel {@link FlagEvalHook} — it does NOT + * replace it (PRES-01 non-regression). + * + *

The writer is resolved lazily from {@link FeatureFlaggingGateway#getFlagEvalWriter()} on each + * call, so the hook is always safe to register — if the writer is absent (killswitch off or not yet + * started) it is a no-op. + */ +class FlagEvalEVPHook implements Hook { + + /** + * Singleton instance: always registered when the provider is created; harmless when + * writer=null (killswitch off or not yet started). + */ + static final FlagEvalEVPHook INSTANCE = new FlagEvalEVPHook<>(null); + + /** + * Optional injected writer (test-only). When non-null, bypasses the gateway lookup. + * Production instances use {@code null} (resolved via gateway). + */ + private final FlagEvaluationWriter injectedWriter; + + /** Production constructor — resolves writer from gateway. */ + FlagEvalEVPHook() { + this.injectedWriter = null; + } + + /** Test-only constructor — injects a writer directly, bypassing the gateway. */ + FlagEvalEVPHook(final FlagEvaluationWriter writer) { + this.injectedWriter = writer; + } + + /** + * Cheap capture + non-blocking enqueue only. Runs at the {@code finally} stage so it covers + * success, error, and default-value paths (reviewer concern #7 {@code 3385309423}). + */ + @Override + public void finallyAfter( + final HookContext ctx, + final FlagEvaluationDetails details, + final Map hints) { + // Resolve writer: prefer injected (test), then gateway + final FlagEvaluationWriter w = + injectedWriter != null ? injectedWriter : FeatureFlaggingGateway.getFlagEvalWriter(); + if (w == null || details == null) { + return; + } + try { + // Cheap scalar extraction — no JSON, no map lookups beyond metadata.asMap() + final String flagKey = details.getFlagKey(); + final ImmutableMetadata metadata = details.getFlagMetadata(); + + // allocationKey: "allocationKey" (camelCase) — consistent with FlagEvalHook.java + final String allocationKey = metadata != null ? metadata.getString("allocationKey") : null; + + // eval-time: from flag metadata "dd.eval.timestamp_ms" (Long), fallback to hook-fire time + // (reviewer concern #4 3395176782). ImmutableMetadata.getLong available since sdk 1.4+. + final Long evalTimeObj = metadata != null ? metadata.getLong("dd.eval.timestamp_ms") : null; + final long evalTimeMs = evalTimeObj != null ? evalTimeObj : System.currentTimeMillis(); + + // variant: null means absent (runtime default; reviewer concern #5 3395344504) + final Object rawValue = details.getValue(); + final String variant = rawValue != null ? rawValue.toString() : null; + + // reason: lowercased + final String reason = + details.getReason() != null ? details.getReason().toLowerCase() : "unknown"; + + // targetingKey from evaluation context + final String targetingKey = + ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; + + // attrs: flatten EvaluationContext attributes for the full-tier canonical key + final Map attrs = extractAttrs(ctx); + + w.enqueue( + new FlagEvalEvent( + flagKey, variant, allocationKey, reason, targetingKey, evalTimeMs, attrs)); + } catch (Exception e) { + // Never let EVP recording break flag evaluation + } + } + + /** Extracts a flat {@code Map} from the evaluation context attributes. */ + private Map extractAttrs(final HookContext ctx) { + if (ctx == null || ctx.getCtx() == null) { + return Collections.emptyMap(); + } + final EvaluationContext evalCtx = ctx.getCtx(); + final Set keys = evalCtx.keySet(); + if (keys.isEmpty()) { + return Collections.emptyMap(); + } + final Map result = new HashMap<>(keys.size() * 2); + for (final String key : keys) { + result.put(key, evalCtx.getValue(key)); + } + return result; + } +} diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java index c492ef49c69..e43cb7e7de3 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/Provider.java @@ -16,6 +16,7 @@ import dev.openfeature.sdk.exceptions.OpenFeatureError; import dev.openfeature.sdk.exceptions.ProviderNotReadyError; import java.lang.reflect.Constructor; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -168,10 +169,14 @@ private Evaluator buildEvaluator() throws Exception { @Override public List getProviderHooks() { - if (flagEvalHook == null) { - return Collections.emptyList(); + final List hooks = new ArrayList<>(2); + if (flagEvalHook != null) { + hooks.add(flagEvalHook); } - return Collections.singletonList(flagEvalHook); + // EVP flagevaluation hook: always registered; no-op when writer is absent (killswitch off). + // Writer is resolved lazily from FeatureFlaggingGateway.getFlagEvalWriter() on each call. + hooks.add(FlagEvalEVPHook.INSTANCE); + return Collections.unmodifiableList(hooks); } @Override diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java index 8de25169166..58fb9f445e7 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -13,8 +14,9 @@ import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.FlagValueType; +import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; -import dev.openfeature.sdk.ImmutableStructure; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.Reason; import java.util.Collections; @@ -29,6 +31,25 @@ class FlagEvalEVPHookTest { // ---- helpers ---- + /** + * Creates a writer that captures the enqueued event for assertion. + * Uses an anonymous class since FlagEvaluationWriter has multiple abstract methods. + */ + private FlagEvaluationWriter capturingWriter(final AtomicReference ref) { + return new FlagEvaluationWriter() { + @Override + public void enqueue(final FlagEvalEvent event) { + ref.set(event); + } + + @Override + public void start() {} + + @Override + public void close() {} + }; + } + private static FlagEvalEVPHook hookWithWriter(final FlagEvaluationWriter writer) { return new FlagEvalEVPHook<>(writer); } @@ -40,10 +61,7 @@ private static FlagEvaluationDetails details( final String reason, final ImmutableMetadata metadata) { final FlagEvaluationDetails.FlagEvaluationDetailsBuilder builder = - FlagEvaluationDetails.builder() - .flagKey(flagKey) - .value(value) - .reason(reason); + FlagEvaluationDetails.builder().flagKey(flagKey).value(value).reason(reason); if (variant != null) { builder.variant(variant); } @@ -53,14 +71,24 @@ private static FlagEvaluationDetails details( return builder.build(); } + private static HookContext hookCtxWithTargetingKey( + final String flagKey, final String targetingKey) { + final MutableContext ctx = new MutableContext(targetingKey); + return HookContext.builder() + .flagKey(flagKey) + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(ctx) + .build(); + } + // ---- test: hook calls writer.enqueue once with flagKey, variant, reason, allocationKey ---- @Test void finallyAfterEnqueuesEventWithAllBasicFields() { final AtomicReference captured = new AtomicReference<>(); - final FlagEvaluationWriter writer = event -> captured.set(event); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); - final FlagEvalEVPHook hook = hookWithWriter(writer); final FlagEvaluationDetails det = details( "my-flag", @@ -74,7 +102,7 @@ void finallyAfterEnqueuesEventWithAllBasicFields() { assertNotNull(captured.get(), "writer.enqueue must be called once"); final FlagEvalEvent e = captured.get(); assertEquals("my-flag", e.flagKey); - assertEquals("on-value", e.variant); // value.toString() when present + assertEquals("on-value", e.variant, "variant must be value.toString() when value is present"); assertEquals("targeting_match", e.reason, "reason must be lowercased"); assertEquals("alloc-1", e.allocationKey); } @@ -84,9 +112,8 @@ void finallyAfterEnqueuesEventWithAllBasicFields() { @Test void evalTimeMsComesFromMetadataWhenPresent() { final AtomicReference captured = new AtomicReference<>(); - final FlagEvaluationWriter writer = event -> captured.set(event); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); - final FlagEvalEVPHook hook = hookWithWriter(writer); final long expectedTimestamp = 1_700_000_000_000L; final FlagEvaluationDetails det = details( @@ -102,7 +129,9 @@ void evalTimeMsComesFromMetadataWhenPresent() { hook.finallyAfter(null, det, Collections.emptyMap()); assertNotNull(captured.get()); - assertEquals(expectedTimestamp, captured.get().evalTimeMs, + assertEquals( + expectedTimestamp, + captured.get().evalTimeMs, "evalTimeMs must come from dd.eval.timestamp_ms metadata when present"); } @@ -111,9 +140,8 @@ void evalTimeMsComesFromMetadataWhenPresent() { @Test void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { final AtomicReference captured = new AtomicReference<>(); - final FlagEvaluationWriter writer = event -> captured.set(event); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); - final FlagEvalEVPHook hook = hookWithWriter(writer); final long before = System.currentTimeMillis(); final FlagEvaluationDetails det = details("ts-flag", "v", "v", Reason.SPLIT.name(), null); @@ -123,19 +151,17 @@ void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { final long after = System.currentTimeMillis(); assertNotNull(captured.get()); final long ts = captured.get().evalTimeMs; - // Should be approximately now (within the test execution window) - assertEquals(true, ts >= before && ts <= after, + assertTrue(ts >= before && ts <= after, "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); } - // ---- test: null value -> variant is null -> runtimeDefaultUsed (reviewer concern #5) ---- + // ---- test: null value -> variant is null -> runtime default (reviewer concern #5) ---- @Test void nullValueProducesNullVariant() { final AtomicReference captured = new AtomicReference<>(); - final FlagEvaluationWriter writer = event -> captured.set(event); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); - final FlagEvalEVPHook hook = hookWithWriter(writer); final FlagEvaluationDetails det = details("def-flag", null, null, Reason.DEFAULT.name(), null); @@ -149,10 +175,9 @@ void nullValueProducesNullVariant() { @Test void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { - // A mock writer that only exposes enqueue — any other call would fail the test final FlagEvaluationWriter writer = mock(FlagEvaluationWriter.class); - final FlagEvalEVPHook hook = hookWithWriter(writer); + final FlagEvaluationDetails det = details("flag", "v", "v", Reason.TARGETING_MATCH.name(), null); @@ -160,7 +185,6 @@ void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { // Exactly one enqueue call, no start/close/aggregate verify(writer, times(1)).enqueue(any(FlagEvalEvent.class)); - // close/start are the only other interface methods — verify they were not called verify(writer, never()).close(); verify(writer, never()).start(); } @@ -190,28 +214,17 @@ void detailsNullIsNoOp() { verifyNoInteractions(writer); } - // ---- test: targetingKey from evaluation context ---- + // ---- test: targetingKey extracted from evaluation context ---- @Test void targetingKeyExtractedFromContext() { final AtomicReference captured = new AtomicReference<>(); - final FlagEvaluationWriter writer = event -> captured.set(event); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); - final FlagEvalEVPHook hook = hookWithWriter(writer); final FlagEvaluationDetails det = details("ctx-flag", "v", "v", Reason.SPLIT.name(), null); - // Provide an evaluation context with a targetingKey - final MutableContext ctx = new MutableContext("user-42"); - ctx.add("region", "eu-west"); - final dev.openfeature.sdk.HookContext hookCtx = - dev.openfeature.sdk.HookContext.builder() - .flagKey("ctx-flag") - .type(dev.openfeature.sdk.FlagValueType.STRING) - .defaultValue("default") - .ctx(ctx) - .metadata(() -> "test-provider") - .build(); + final HookContext hookCtx = hookCtxWithTargetingKey("ctx-flag", "user-42"); hook.finallyAfter(hookCtx, det, Collections.emptyMap()); diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java index 27d4dd5d2b5..e9394d8934c 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/ProviderTest.java @@ -330,8 +330,10 @@ public void testGetProviderHooksReturnsFlagEvalHook() { Provider provider = new Provider(new Options().initTimeout(10, MILLISECONDS), mock(Evaluator.class)); List hooks = provider.getProviderHooks(); - assertThat(hooks.size(), equalTo(1)); + // Two hooks: OTel FlagEvalHook (index 0) + EVP FlagEvalEVPHook (index 1) + assertThat(hooks.size(), equalTo(2)); assertThat(hooks.get(0) instanceof FlagEvalHook, equalTo(true)); + assertThat(hooks.get(1) instanceof FlagEvalEVPHook, equalTo(true)); } @Test @@ -343,9 +345,8 @@ public void testShutdownCleansUpMetrics() throws Exception { provider.initialize(null); provider.shutdown(); verify(evaluator).shutdown(); - // After shutdown, getProviderHooks still returns a list (hook is still present but metrics is - // shut down) - assertThat(provider.getProviderHooks().size(), equalTo(1)); + // After shutdown, getProviderHooks still returns a list with both OTel + EVP hooks + assertThat(provider.getProviderHooks().size(), equalTo(2)); } public interface EvaluateMethod { From fcc884e4d0ed5f5551d74f83a28d2f1e7cabdbb3 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 15:50:33 -0400 Subject: [PATCH 05/10] style(02-05): apply google-java-format via spotlessApply - Apply project-required code style (google-java-format) to all new files in this plan before submitting --- .../api/openfeature/FlagEvalEVPHook.java | 16 ++-- .../api/openfeature/FlagEvalEVPHookTest.java | 15 ++-- .../featureflag/FlagEvaluationWriterImpl.java | 35 ++++---- .../FlagEvaluationWriterImplTest.java | 87 +++++++++++-------- 4 files changed, 86 insertions(+), 67 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java index c8317c60b61..067146cb438 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java @@ -17,12 +17,12 @@ * OpenFeature {@code Hook} that captures flag evaluation events for EVP {@code flagevaluation} * emission. * - *

Contract: {@code finallyAfter} does ONLY cheap scalar extraction + a non-blocking offer to - * the writer's bounded queue. No inline aggregation on the hook thread (reviewer concern #7 {@code + *

Contract: {@code finallyAfter} does ONLY cheap scalar extraction + a non-blocking offer to the + * writer's bounded queue. No inline aggregation on the hook thread (reviewer concern #7 {@code * 3385309423}). * - *

This hook is registered alongside the existing OTel {@link FlagEvalHook} — it does NOT - * replace it (PRES-01 non-regression). + *

This hook is registered alongside the existing OTel {@link FlagEvalHook} — it does NOT replace + * it (PRES-01 non-regression). * *

The writer is resolved lazily from {@link FeatureFlaggingGateway#getFlagEvalWriter()} on each * call, so the hook is always safe to register — if the writer is absent (killswitch off or not yet @@ -31,14 +31,14 @@ class FlagEvalEVPHook implements Hook { /** - * Singleton instance: always registered when the provider is created; harmless when - * writer=null (killswitch off or not yet started). + * Singleton instance: always registered when the provider is created; harmless when writer=null + * (killswitch off or not yet started). */ static final FlagEvalEVPHook INSTANCE = new FlagEvalEVPHook<>(null); /** - * Optional injected writer (test-only). When non-null, bypasses the gateway lookup. - * Production instances use {@code null} (resolved via gateway). + * Optional injected writer (test-only). When non-null, bypasses the gateway lookup. Production + * instances use {@code null} (resolved via gateway). */ private final FlagEvaluationWriter injectedWriter; diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java index 58fb9f445e7..28971e84d22 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -24,16 +24,16 @@ import org.junit.jupiter.api.Test; /** - * Unit tests for {@link FlagEvalEVPHook}: cheap capture, non-blocking enqueue, eval-time - * metadata, absent-variant detection, and killswitch-via-writer-null behaviour. + * Unit tests for {@link FlagEvalEVPHook}: cheap capture, non-blocking enqueue, eval-time metadata, + * absent-variant detection, and killswitch-via-writer-null behaviour. */ class FlagEvalEVPHookTest { // ---- helpers ---- /** - * Creates a writer that captures the enqueued event for assertion. - * Uses an anonymous class since FlagEvaluationWriter has multiple abstract methods. + * Creates a writer that captures the enqueued event for assertion. Uses an anonymous class since + * FlagEvaluationWriter has multiple abstract methods. */ private FlagEvaluationWriter capturingWriter(final AtomicReference ref) { return new FlagEvaluationWriter() { @@ -151,7 +151,8 @@ void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { final long after = System.currentTimeMillis(); assertNotNull(captured.get()); final long ts = captured.get().evalTimeMs; - assertTrue(ts >= before && ts <= after, + assertTrue( + ts >= before && ts <= after, "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); } @@ -229,7 +230,9 @@ void targetingKeyExtractedFromContext() { hook.finallyAfter(hookCtx, det, Collections.emptyMap()); assertNotNull(captured.get()); - assertEquals("user-42", captured.get().targetingKey, + assertEquals( + "user-42", + captured.get().targetingKey, "targetingKey must be extracted from the evaluation context"); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index b4e7cf2a237..3a5bce3a14d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -181,7 +181,9 @@ static String canonicalContextKey(final Map attrs) { private static void appendLengthDelimited(final StringBuilder sb, final String s) { // Encode as big-endian 8-byte length prefix + raw chars (Java chars, deterministic) final long len = s.length(); - sb.append(String.format("%08x", len)); // 8 hex chars ≈ big-endian 4 bytes (sufficient for realistic sizes) + sb.append( + String.format( + "%08x", len)); // 8 hex chars ≈ big-endian 4 bytes (sufficient for realistic sizes) sb.append(s); } @@ -298,7 +300,8 @@ static class FlagEvaluationSerializingHandler implements Runnable { final Map context; private final Runnable errorCallback; - // Two-tier aggregation maps (accessed only from the serializer thread; package-private for test) + // Two-tier aggregation maps (accessed only from the serializer thread; package-private for + // test) final Map fullTier = new HashMap<>(); final Map degradedTier = new HashMap<>(); @@ -480,7 +483,8 @@ void flush(final BackendApi evp) { } private List buildEventList() { - final List events = new ArrayList<>(fullTier.size() + degradedTier.size()); + final List events = + new ArrayList<>(fullTier.size() + degradedTier.size()); for (final EvalBucket bucket : fullTier.values()) { events.add(FlagEvaluationEvent.fromBucket(bucket, true)); } @@ -504,13 +508,12 @@ private boolean shouldFlush() { // ---- Test-seam inner class (package-private) ---- /** - * Test-accessible handler that exposes {@link #drainAndAggregate()} and {@link #flush(BackendApi)} - * without starting a real background thread. + * Test-accessible handler that exposes {@link #drainAndAggregate()} and {@link + * #flush(BackendApi)} without starting a real background thread. */ static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler { - SerializingHandlerForTest( - final BackendApiFactory factory, final Map context) { + SerializingHandlerForTest(final BackendApiFactory factory, final Map context) { super( factory, Queues.mpscBlockingConsumerArrayQueue(DEFAULT_CAPACITY), @@ -534,9 +537,7 @@ AggregatedState drainAndAggregate() { } staged.clear(); return new AggregatedState( - new HashMap<>(fullTier), - new HashMap<>(degradedTier), - droppedDegradedOverflow.get()); + new HashMap<>(fullTier), new HashMap<>(degradedTier), droppedDegradedOverflow.get()); } /** Simulates filling the full tier to GLOBAL_CAP by injecting synthetic distinct buckets. */ @@ -549,7 +550,9 @@ void simulateFullTierAtCap() { } } - /** Simulates filling the degraded tier to DEGRADED_CAP by injecting synthetic distinct buckets. */ + /** + * Simulates filling the degraded tier to DEGRADED_CAP by injecting synthetic distinct buckets. + */ void simulateDegradedTierAtCap() { for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { final String key = "synthetic-dg-" + i; @@ -583,9 +586,9 @@ static class FlagEvaluationEvent { public final long first_evaluation; public final long last_evaluation; public final long evaluation_count; - public final String variant; // null = omitted (degraded tier or absent) - public final String allocation; // null = omitted - public final String targeting_key; // null = omitted (degraded tier) + public final String variant; // null = omitted (degraded tier or absent) + public final String allocation; // null = omitted + public final String targeting_key; // null = omitted (degraded tier) public final Boolean runtime_default_used; // null = omitted when false public final Map context; // null = omitted (degraded tier) @@ -621,9 +624,9 @@ static FlagEvaluationEvent fromBucket(final EvalBucket bucket, final boolean isF bucket.count, isFullTier ? bucket.variant : bucket.variant, bucket.allocationKey, - isFullTier ? bucket.targetingKey : null, // degraded omits targetingKey + isFullTier ? bucket.targetingKey : null, // degraded omits targetingKey bucket.runtimeDefaultUsed, - isFullTier ? bucket.sampleAttrs : null); // degraded omits context + isFullTier ? bucket.sampleAttrs : null); // degraded omits context } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index c31f3757e72..cf4c5953773 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -7,22 +7,15 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.Moshi; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.SharedCommunicationObjects; -import datadog.trace.api.Config; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeUnit; import okhttp3.RequestBody; import okio.Buffer; import org.junit.jupiter.api.Test; @@ -36,15 +29,20 @@ class FlagEvaluationWriterImplTest { // ---- helpers ---- private static FlagEvalEvent event( - String flagKey, String variant, String allocationKey, String reason, String targetingKey, - long evalTimeMs, Map attrs) { + String flagKey, + String variant, + String allocationKey, + String reason, + String targetingKey, + long evalTimeMs, + Map attrs) { return new FlagEvalEvent( flagKey, variant, allocationKey, reason, targetingKey, evalTimeMs, attrs); } private static FlagEvalEvent simpleEvent(String flagKey, String variant) { - return event(flagKey, variant, "alloc1", "targeting_match", "user-1", 1000L, - Collections.emptyMap()); + return event( + flagKey, variant, "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); } // Build a writer with a mocked BackendApi, flushing synchronously via triggerFlush(). @@ -66,7 +64,8 @@ static class TestWriterSetup { final FlagEvaluationWriterImpl.SerializingHandlerForTest handler; final BackendApi mockEvp; - TestWriterSetup(FlagEvaluationWriterImpl.SerializingHandlerForTest handler, BackendApi mockEvp) { + TestWriterSetup( + FlagEvaluationWriterImpl.SerializingHandlerForTest handler, BackendApi mockEvp) { this.handler = handler; this.mockEvp = mockEvp; } @@ -79,16 +78,17 @@ void identicalEventsAggregatIntoOneBucketWithCount2() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e1 = event("flag-a", "on", "alloc1", "targeting_match", "user-1", 1000L, - Collections.emptyMap()); - FlagEvalEvent e2 = event("flag-a", "on", "alloc1", "targeting_match", "user-1", 2000L, - Collections.emptyMap()); + FlagEvalEvent e1 = + event("flag-a", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); + FlagEvalEvent e2 = + event("flag-a", "on", "alloc1", "targeting_match", "user-1", 2000L, Collections.emptyMap()); setup.handler.add(e1); setup.handler.add(e2); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); - assertEquals(1, state.fullTier.size(), "Identical events must produce exactly 1 full-tier bucket"); + assertEquals( + 1, state.fullTier.size(), "Identical events must produce exactly 1 full-tier bucket"); FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); assertEquals(2, bucket.count, "Count must be 2"); assertEquals(1000L, bucket.firstEvalMs, "first_evaluation must be min(1000, 2000)=1000"); @@ -108,14 +108,18 @@ void differentValueTypesProduceDifferentBuckets() throws Exception { Map attrsStr = new HashMap<>(); attrsStr.put("score", "1"); - FlagEvalEvent e1 = event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsInt); - FlagEvalEvent e2 = event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsStr); + FlagEvalEvent e1 = + event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsInt); + FlagEvalEvent e2 = + event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsStr); setup.handler.add(e1); setup.handler.add(e2); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); - assertEquals(2, state.fullTier.size(), + assertEquals( + 2, + state.fullTier.size(), "int 1 vs String \"1\" must produce 2 distinct buckets (type-tagged key, reviewer concern #3)"); } @@ -154,7 +158,8 @@ void degradedCapOverflowIncrementsDroppedCounter() throws Exception { setup.handler.add(e); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); - assertTrue(state.droppedDegradedOverflow > 0, + assertTrue( + state.droppedDegradedOverflow > 0, "Beyond DEGRADED_CAP must increment droppedDegradedOverflow counter (reviewer concern #8)"); } @@ -165,8 +170,8 @@ void absentVariantSetsRuntimeDefaultUsed() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e = event("flag-c", null, "alloc1", "default", "user-1", 1000L, - Collections.emptyMap()); + FlagEvalEvent e = + event("flag-c", null, "alloc1", "default", "user-1", 1000L, Collections.emptyMap()); setup.handler.add(e); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); @@ -209,13 +214,15 @@ void contextExceeding256FieldsIsPruned() throws Exception { for (int i = 0; i < 300; i++) { hugeAttrs.put("key" + i, "v" + i); } - FlagEvalEvent e = event("flag-d", "on", "alloc1", "targeting_match", "user-1", 1000L, hugeAttrs); + FlagEvalEvent e = + event("flag-d", "on", "alloc1", "targeting_match", "user-1", 1000L, hugeAttrs); setup.handler.add(e); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); - assertTrue(bucket.prunedContextFieldCount <= 256, + assertTrue( + bucket.prunedContextFieldCount <= 256, "Context must be pruned to <=256 fields (reviewer concern #1)"); } @@ -241,7 +248,8 @@ void contextValueExceeding256CharsIsSkipped() throws Exception { assertEquals(1, state.fullTier.size()); // The key must not include the long value (skipped at prune step) String fullKey = state.fullTier.keySet().iterator().next(); - assertFalse(fullKey.contains(longVal), + assertFalse( + fullKey.contains(longVal), "Context string value >256 chars must be skipped (reviewer concern #1)"); } @@ -252,8 +260,8 @@ void flushPostsToFlagevaluationsEndpointWithRequiredFields() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e = event("flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, - Collections.emptyMap()); + FlagEvalEvent e = + event("flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); setup.handler.add(e); // Must drain+aggregate first so the aggregation maps are populated setup.handler.drainAndAggregate(); @@ -272,15 +280,17 @@ void flushJsonBodyContainsRequiredFields() throws Exception { // Capture the RequestBody final RequestBody[] captured = {null}; when(mockEvp.post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false))) - .thenAnswer(inv -> { - captured[0] = inv.getArgument(1); - return null; - }); + .thenAnswer( + inv -> { + captured[0] = inv.getArgument(1); + return null; + }); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e = event("my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, - Collections.emptyMap()); + FlagEvalEvent e = + event( + "my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, Collections.emptyMap()); setup.handler.add(e); // Must drain+aggregate first so the aggregation maps are populated setup.handler.drainAndAggregate(); @@ -296,11 +306,14 @@ void flushJsonBodyContainsRequiredFields() throws Exception { assertTrue(json.contains("flagEvaluations"), "Payload must contain flagEvaluations array"); assertTrue(json.contains("\"key\""), "Flag key field required"); assertTrue(json.contains("my-flag"), "Flag key value must be in payload"); - assertTrue(json.contains("first_evaluation") || json.contains("firstEvaluation"), + assertTrue( + json.contains("first_evaluation") || json.contains("firstEvaluation"), "first_evaluation required"); - assertTrue(json.contains("last_evaluation") || json.contains("lastEvaluation"), + assertTrue( + json.contains("last_evaluation") || json.contains("lastEvaluation"), "last_evaluation required"); - assertTrue(json.contains("evaluation_count") || json.contains("evaluationCount"), + assertTrue( + json.contains("evaluation_count") || json.contains("evaluationCount"), "evaluation_count required"); } } From 78bbabae6396a5c992b5ae3181d2893b22fdb4af Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 12 Jun 2026 18:02:48 -0400 Subject: [PATCH 06/10] chore(openfeature): remove internal review annotations from EVP flagevaluation code --- .../trace/api/openfeature/DDEvaluator.java | 4 ++-- .../api/openfeature/FlagEvalEVPHook.java | 13 +++++------ .../api/openfeature/FlagEvalEVPHookTest.java | 6 ++--- .../flagevaluation/FlagEvalEvent.java | 6 ++--- .../featureflag/FlagEvaluationWriterImpl.java | 22 +++++++++---------- .../FlagEvaluationWriterImplTest.java | 16 +++++--------- 6 files changed, 31 insertions(+), 36 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java index 51a38730716..762ffe5997c 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/DDEvaluator.java @@ -387,8 +387,8 @@ private static ProviderEvaluation resolveVariant( + e.getMessage()); } - // Stamp eval-time at resolution point for accurate first/last_evaluation in EVP payloads - // (reviewer concern #4 3395176782 — eval-time via provider metadata, not hook-fire time). + // Stamp eval-time at the resolution point so first/last_evaluation reflect evaluation time, + // not hook-fire time. Passed to the hook via provider metadata "dd.eval.timestamp_ms". final long evalTimestampMs = System.currentTimeMillis(); final ImmutableMetadata.ImmutableMetadataBuilder metadataBuilder = ImmutableMetadata.builder() diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java index 067146cb438..dbb272617d1 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java @@ -18,11 +18,10 @@ * emission. * *

Contract: {@code finallyAfter} does ONLY cheap scalar extraction + a non-blocking offer to the - * writer's bounded queue. No inline aggregation on the hook thread (reviewer concern #7 {@code - * 3385309423}). + * writer's bounded queue. No inline aggregation on the hook thread. * *

This hook is registered alongside the existing OTel {@link FlagEvalHook} — it does NOT replace - * it (PRES-01 non-regression). + * it (the existing OTel metrics hook is left unchanged). * *

The writer is resolved lazily from {@link FeatureFlaggingGateway#getFlagEvalWriter()} on each * call, so the hook is always safe to register — if the writer is absent (killswitch off or not yet @@ -54,7 +53,7 @@ class FlagEvalEVPHook implements Hook { /** * Cheap capture + non-blocking enqueue only. Runs at the {@code finally} stage so it covers - * success, error, and default-value paths (reviewer concern #7 {@code 3385309423}). + * success, error, and default-value paths. */ @Override public void finallyAfter( @@ -75,12 +74,12 @@ public void finallyAfter( // allocationKey: "allocationKey" (camelCase) — consistent with FlagEvalHook.java final String allocationKey = metadata != null ? metadata.getString("allocationKey") : null; - // eval-time: from flag metadata "dd.eval.timestamp_ms" (Long), fallback to hook-fire time - // (reviewer concern #4 3395176782). ImmutableMetadata.getLong available since sdk 1.4+. + // eval-time: from flag metadata "dd.eval.timestamp_ms" (Long), fallback to hook-fire time. + // ImmutableMetadata.getLong available since sdk 1.4+. final Long evalTimeObj = metadata != null ? metadata.getLong("dd.eval.timestamp_ms") : null; final long evalTimeMs = evalTimeObj != null ? evalTimeObj : System.currentTimeMillis(); - // variant: null means absent (runtime default; reviewer concern #5 3395344504) + // variant: null means absent (runtime default) final Object rawValue = details.getValue(); final String variant = rawValue != null ? rawValue.toString() : null; diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java index 28971e84d22..04d66292741 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -107,7 +107,7 @@ void finallyAfterEnqueuesEventWithAllBasicFields() { assertEquals("alloc-1", e.allocationKey); } - // ---- test: evalTimeMs from metadata "dd.eval.timestamp_ms" (reviewer concern #4) ---- + // ---- test: evalTimeMs from metadata "dd.eval.timestamp_ms" ---- @Test void evalTimeMsComesFromMetadataWhenPresent() { @@ -156,7 +156,7 @@ void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); } - // ---- test: null value -> variant is null -> runtime default (reviewer concern #5) ---- + // ---- test: null value -> variant is null -> runtime default ---- @Test void nullValueProducesNullVariant() { @@ -172,7 +172,7 @@ void nullValueProducesNullVariant() { assertNull(captured.get().variant, "Null value must produce null variant (runtime default)"); } - // ---- test: hook does NO aggregation on the hook thread (reviewer concern #7) ---- + // ---- test: hook does NO aggregation on the hook thread ---- @Test void finallyAfterOnlyCallsEnqueueNoOtherWriterMethods() { diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java index 63da35fe8ae..a21da851e91 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -18,7 +18,7 @@ public final class FlagEvalEvent { /** * The evaluated variant/value as a string. {@code null} means the default value was returned - * (runtime default; reviewer concern #5 {@code 3395344504}). + * (runtime default). */ public final String variant; @@ -35,8 +35,8 @@ public final class FlagEvalEvent { /** * Evaluation timestamp in milliseconds since epoch. Stamped at eval-entry time from flag metadata - * key {@code "dd.eval.timestamp_ms"}, or falls back to hook-fire time when absent (reviewer - * concern #4 {@code 3395176782}). + * key {@code "dd.eval.timestamp_ms"}, or falls back to hook-fire time when absent. This ensures + * first/last_evaluation reflect evaluation time, not hook-fire time. */ public final long evalTimeMs; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 3a5bce3a14d..89ba8e64c66 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -34,18 +34,18 @@ * single-exposure buffer. Routes to {@code /evp_proxy/v2/api/v2/flagevaluations} via {@code * BackendApiFactory.createBackendApi(Intake.EVENT_PLATFORM)}. * - *

Frozen-contract conformance (see {@code .planning/FANOUT-CONTRACT.md}): + *

Two-tier aggregation contract: * *

    *
  • Two-tier aggregation: full → degraded → drop-counted (no ultra-degraded). *
  • Full key: (flagKey, variant, allocationKey, reason, targetingKey, canonical-context-key). *
  • Degraded key: (flagKey, variant, allocationKey, reason) — no targetingKey/context. *
  • Canonical context key: sorted entries, type-tagged length-delimited encoding — NOT a hash - * (reviewer concern #3 {@code 3395004724}). - *
  • Context pruning: ≤256 fields, string values ≤256 chars (reviewer concern #1). + * (collision-safe, comparable string identity). + *
  • Context pruning: ≤256 fields, string values ≤256 chars. *
  • Caps: globalCap=131072, perFlagCap=10000, degradedCap=32768. *
  • Eval-time: min/max of firstEvalMs/lastEvalMs across events in the same bucket. - *
  • Runtime default: absent variant (reviewer concern #5 {@code 3395344504}). + *
  • Runtime default: absent variant means {@code runtimeDefaultUsed=true}. *
  • Flush interval: 10 seconds. *
  • Queue: bounded MessagePassingBlockingQueue (capacity 2^16), non-blocking offer. *
@@ -60,11 +60,11 @@ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { static final int PER_FLAG_CAP = 10_000; static final int DEGRADED_CAP = 32_768; - // Context pruning limits (reviewer concern #1) + // Context pruning limits: max fields and max string value length static final int MAX_CONTEXT_FIELDS = 256; static final int MAX_FIELD_LENGTH = 256; - // Type tags for canonical context key (reviewer concern #3) + // Type tags for canonical context key (type-tagged length-delimited encoding) private static final byte CTX_TAG_STRING = 's'; private static final byte CTX_TAG_BOOL = 'b'; private static final byte CTX_TAG_INT = 'i'; @@ -130,8 +130,8 @@ public void enqueue(final FlagEvalEvent event) { queue.offer(event); // non-blocking; drops silently on overflow (bounded queue) } - // ---- Canonical context key (reviewer concern #3 3395004724) ---- - // Sorted entries, type-tagged length-delimited encoding. NOT a hash. + // ---- Canonical context key ---- + // Sorted entries, type-tagged length-delimited encoding. NOT a hash (collision-safe string key). // Implementation mirrors dd-trace-go/openfeature/flagevaluation.go canonicalContextKey(). /** @@ -311,7 +311,7 @@ static class FlagEvaluationSerializingHandler implements Runnable { // Global full-tier count int globalFullCount = 0; - // Observable drop counter (reviewer concern #8) + // Observable counter for events dropped when both tiers are at capacity final AtomicLong droppedDegradedOverflow = new AtomicLong(0); FlagEvaluationSerializingHandler( @@ -359,7 +359,7 @@ private void runDutyCycle(final BackendApi evp) throws InterruptedException { // ---- Aggregation logic ---- - /** Routes an event into the full tier or degraded tier (reviewer concerns #3, #5, #8). */ + /** Routes an event into the full tier or degraded tier, or drops and counts on overflow. */ void aggregateEvent(final FlagEvalEvent event) { final boolean isDefault = event.variant == null; @@ -414,7 +414,7 @@ void aggregateEvent(final FlagEvalEvent event) { return; } - // Both tiers saturated — drop and count (reviewer concern #8) + // Both tiers saturated — drop and count droppedDegradedOverflow.incrementAndGet(); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index cf4c5953773..c3f4fa79e88 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -120,7 +120,7 @@ void differentValueTypesProduceDifferentBuckets() throws Exception { assertEquals( 2, state.fullTier.size(), - "int 1 vs String \"1\" must produce 2 distinct buckets (type-tagged key, reviewer concern #3)"); + "int 1 vs String \"1\" must produce 2 distinct buckets (type-tagged canonical key)"); } // ---- test: full-tier overflow past GLOBAL_CAP routes to degraded ---- @@ -153,17 +153,17 @@ void degradedCapOverflowIncrementsDroppedCounter() throws Exception { setup.handler.simulateFullTierAtCap(); setup.handler.simulateDegradedTierAtCap(); - // Next event must be dropped-and-counted (reviewer concern #8) + // Next event must be dropped-and-counted FlagEvalEvent e = simpleEvent("drop-flag", "on"); setup.handler.add(e); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertTrue( state.droppedDegradedOverflow > 0, - "Beyond DEGRADED_CAP must increment droppedDegradedOverflow counter (reviewer concern #8)"); + "Beyond DEGRADED_CAP must increment droppedDegradedOverflow counter"); } - // ---- test: absent variant -> runtimeDefaultUsed=true (reviewer concern #5) ---- + // ---- test: absent variant -> runtimeDefaultUsed=true ---- @Test void absentVariantSetsRuntimeDefaultUsed() throws Exception { @@ -221,9 +221,7 @@ void contextExceeding256FieldsIsPruned() throws Exception { FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); - assertTrue( - bucket.prunedContextFieldCount <= 256, - "Context must be pruned to <=256 fields (reviewer concern #1)"); + assertTrue(bucket.prunedContextFieldCount <= 256, "Context must be pruned to <=256 fields"); } // ---- test: string context value >256 chars is skipped ---- @@ -248,9 +246,7 @@ void contextValueExceeding256CharsIsSkipped() throws Exception { assertEquals(1, state.fullTier.size()); // The key must not include the long value (skipped at prune step) String fullKey = state.fullTier.keySet().iterator().next(); - assertFalse( - fullKey.contains(longVal), - "Context string value >256 chars must be skipped (reviewer concern #1)"); + assertFalse(fullKey.contains(longVal), "Context string value >256 chars must be skipped"); } // ---- test: flush posts to "flagevaluations" endpoint with required fields ---- From be97d8d5582cdd058e9313d1f42c96f58a861548 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 13 Jun 2026 11:27:23 -0400 Subject: [PATCH 07/10] feat(openfeature): add EVP flagevaluation error object, deterministic pruning, shutdown drain, and schema validation - variant sourced from details.getVariant() (not the evaluated value) - error object captured from evaluation details, schema {"message":...} - killswitch DD_FLAGGING_EVALUATION_COUNTS_ENABLED via the tracer config system - deterministic context pruning (sort before cut), stored pruned attrs - observable queue-overflow drop counter; close() drains and final-flushes - JMH hot-path benchmark; structural validation against vendored worker schema --- .../api/config/FeatureFlaggingConfig.java | 8 + metadata/supported-configurations.json | 8 + .../featureflag/FeatureFlaggingSystem.java | 20 +- .../api/openfeature/FlagEvalEVPHook.java | 25 +- .../api/openfeature/FlagEvalEVPHookTest.java | 102 +++- .../flagevaluation/FlagEvalEvent.java | 19 + .../feature-flagging-lib/build.gradle.kts | 6 + .../FlagEvaluationHotPathBenchmark.java | 112 ++++ .../featureflag/FlagEvaluationWriterImpl.java | 298 +++++++--- .../FlagEvaluationWriterImplTest.java | 511 ++++++++++++++---- .../batchedflagevaluations.json | 184 +++++++ 11 files changed, 1091 insertions(+), 202 deletions(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/resources/flagevaluation/batchedflagevaluations.json diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..5f567908254 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -3,4 +3,12 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + /** + * Killswitch for the EVP {@code flagevaluation} emission path. Default: enabled. Disabling it + * turns off EVP flag-evaluation counts while leaving the OTel {@code feature_flag.evaluations} + * metric path untouched. Maps to {@code DD_FLAGGING_EVALUATION_COUNTS_ENABLED}. + */ + public static final String FLAGGING_EVALUATION_COUNTS_ENABLED = + "flagging.evaluation.counts.enabled"; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index a42ba33a31b..6d504ac6546 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1505,6 +1505,14 @@ "aliases": [] } ], + "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "true", + "aliases": [] + } + ], "DD_FORCE_CLEAR_TEXT_HTTP_FOR_INTAKE_CLIENT": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index d838fcb419d..d50bdaf92ee 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -2,6 +2,7 @@ import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; +import datadog.trace.api.config.FeatureFlaggingConfig; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,12 +11,6 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); - /** - * Killswitch env var for EVP flagevaluation emission. Default: on (enabled). Set to {@code - * "false"} to disable EVP emission while preserving the OTel path. - */ - static final String KILLSWITCH_ENV = "DD_FLAGGING_EVALUATION_COUNTS_ENABLED"; - private static volatile RemoteConfigService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; private static volatile FlagEvaluationWriter FLAG_EVAL_WRITER; @@ -35,14 +30,21 @@ public static void start(final SharedCommunicationObjects sco) { EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); EXPOSURE_WRITER.init(); - // EVP flagevaluation writer — gated by killswitch (default: on) - if (!"false".equals(System.getenv(KILLSWITCH_ENV))) { + // EVP flagevaluation writer — gated by the killswitch + // DD_FLAGGING_EVALUATION_COUNTS_ENABLED (default: on), read through the tracer config system. + final boolean evalCountsEnabled = + config + .configProvider() + .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); + if (evalCountsEnabled) { final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); evalWriter.start(); // registers with FeatureFlaggingGateway FLAG_EVAL_WRITER = evalWriter; LOGGER.debug("Flag evaluation EVP writer started"); } else { - LOGGER.debug("Flag evaluation EVP writer disabled ({}=false)", KILLSWITCH_ENV); + LOGGER.debug( + "Flag evaluation EVP writer disabled ({}=false)", + FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED); } LOGGER.debug("Feature Flagging system started"); diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java index dbb272617d1..1d814a7badc 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java @@ -79,14 +79,24 @@ public void finallyAfter( final Long evalTimeObj = metadata != null ? metadata.getLong("dd.eval.timestamp_ms") : null; final long evalTimeMs = evalTimeObj != null ? evalTimeObj : System.currentTimeMillis(); - // variant: null means absent (runtime default) - final Object rawValue = details.getValue(); - final String variant = rawValue != null ? rawValue.toString() : null; + // variant: the OpenFeature variant key (same source as the OTel FlagEvalHook), NOT the + // evaluated value. A null variant means no variant was selected (runtime default). + final String variant = details.getVariant(); // reason: lowercased final String reason = details.getReason() != null ? details.getReason().toLowerCase() : "unknown"; + // error message: prefer the human-readable message; fall back to the error code name when + // the message is empty (some providers populate only the code). null on success. + String errorMessage = details.getErrorMessage(); + if ((errorMessage == null || errorMessage.isEmpty()) && details.getErrorCode() != null) { + errorMessage = details.getErrorCode().name(); + } + if (errorMessage != null && errorMessage.isEmpty()) { + errorMessage = null; + } + // targetingKey from evaluation context final String targetingKey = ctx != null && ctx.getCtx() != null ? ctx.getCtx().getTargetingKey() : null; @@ -96,7 +106,14 @@ public void finallyAfter( w.enqueue( new FlagEvalEvent( - flagKey, variant, allocationKey, reason, targetingKey, evalTimeMs, attrs)); + flagKey, + variant, + allocationKey, + reason, + targetingKey, + errorMessage, + evalTimeMs, + attrs)); } catch (Exception e) { // Never let EVP recording break flag evaluation } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java index 04d66292741..726da70cb6d 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -13,6 +13,7 @@ import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; +import dev.openfeature.sdk.ErrorCode; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.FlagValueType; import dev.openfeature.sdk.HookContext; @@ -102,11 +103,36 @@ void finallyAfterEnqueuesEventWithAllBasicFields() { assertNotNull(captured.get(), "writer.enqueue must be called once"); final FlagEvalEvent e = captured.get(); assertEquals("my-flag", e.flagKey); - assertEquals("on-value", e.variant, "variant must be value.toString() when value is present"); + assertEquals("on", e.variant, "variant must be the OpenFeature variant key"); assertEquals("targeting_match", e.reason, "reason must be lowercased"); assertEquals("alloc-1", e.allocationKey); } + // ---- G1: variant comes from details.getVariant(), NOT details.getValue() ---- + + @Test + void variantIsTheVariantKeyNotTheEvaluatedValue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); + + // value and variant DIFFER, so a value-vs-variant mistake is detectable. + final FlagEvaluationDetails det = + details( + "g1-flag", + "the-evaluated-value", // value + "the-variant-key", // variant + Reason.TARGETING_MATCH.name(), + null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "the-variant-key", + captured.get().variant, + "variant must be sourced from details.getVariant(), not details.getValue()"); + } + // ---- test: evalTimeMs from metadata "dd.eval.timestamp_ms" ---- @Test @@ -156,20 +182,86 @@ void evalTimeMsFallsBackToCurrentTimeWhenMetadataAbsent() { "evalTimeMs must fall back to hook-fire time when metadata absent. got: " + ts); } - // ---- test: null value -> variant is null -> runtime default ---- + // ---- test: absent variant -> variant is null -> runtime default ---- + + @Test + void absentVariantProducesNullVariant() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); + + // A runtime default returns the default value but no variant. + final FlagEvaluationDetails det = + details("def-flag", "default-value", null, Reason.DEFAULT.name(), null); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertNull(captured.get().variant, "Absent variant must stay null (runtime default)"); + } + + // ---- test: error message captured from details (G3 error object support) ---- + + @Test + void errorMessageCapturedFromDetails() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.TYPE_MISMATCH) + .errorMessage("value does not match declared type") + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "value does not match declared type", + captured.get().errorMessage, + "errorMessage must be captured from the evaluation details"); + } + + // ---- test: error code used as fallback message when error message is empty ---- + + @Test + void errorCodeUsedAsFallbackWhenMessageEmpty() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); + + final FlagEvaluationDetails det = + FlagEvaluationDetails.builder() + .flagKey("err-flag") + .value("default") + .reason(Reason.ERROR.name()) + .errorCode(ErrorCode.FLAG_NOT_FOUND) + .build(); + + hook.finallyAfter(null, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals( + "FLAG_NOT_FOUND", + captured.get().errorMessage, + "error code name must be used when no error message is present"); + } + + // ---- test: success path has no error message ---- @Test - void nullValueProducesNullVariant() { + void successPathHasNullErrorMessage() { final AtomicReference captured = new AtomicReference<>(); final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); final FlagEvaluationDetails det = - details("def-flag", null, null, Reason.DEFAULT.name(), null); + details("ok-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); hook.finallyAfter(null, det, Collections.emptyMap()); assertNotNull(captured.get()); - assertNull(captured.get().variant, "Null value must produce null variant (runtime default)"); + assertNull(captured.get().errorMessage, "success path must have no error message"); } // ---- test: hook does NO aggregation on the hook thread ---- diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java index a21da851e91..78d503758b2 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -33,6 +33,12 @@ public final class FlagEvalEvent { /** The targeting key from the evaluation context. May be null. */ public final String targetingKey; + /** + * The evaluation error message when the evaluation failed, else {@code null}. Sourced from the + * OpenFeature evaluation details (error message, falling back to the error code). + */ + public final String errorMessage; + /** * Evaluation timestamp in milliseconds since epoch. Stamped at eval-entry time from flag metadata * key {@code "dd.eval.timestamp_ms"}, or falls back to hook-fire time when absent. This ensures @@ -54,11 +60,24 @@ public FlagEvalEvent( final String targetingKey, final long evalTimeMs, final Map attrs) { + this(flagKey, variant, allocationKey, reason, targetingKey, null, evalTimeMs, attrs); + } + + public FlagEvalEvent( + final String flagKey, + final String variant, + final String allocationKey, + final String reason, + final String targetingKey, + final String errorMessage, + final long evalTimeMs, + final Map attrs) { this.flagKey = flagKey; this.variant = variant; this.allocationKey = allocationKey; this.reason = reason; this.targetingKey = targetingKey; + this.errorMessage = errorMessage; this.evalTimeMs = evalTimeMs; this.attrs = attrs != null ? attrs : Collections.emptyMap(); } diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 2888ba1b25c..3f0cded6616 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -1,6 +1,7 @@ plugins { `java-library` id("dd-trace-java.version-file") + id("me.champeau.jmh") } apply(from = "$rootDir/gradle/java.gradle") @@ -30,3 +31,8 @@ dependencies { testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } + +jmh { + jmhVersion = libs.versions.jmh.get() + duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java new file mode 100644 index 00000000000..46d6b4a627b --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -0,0 +1,112 @@ +package com.datadog.featureflag; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.communication.BackendApiFactory; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.util.HashMap; +import java.util.Map; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Hot-path benchmark for EVP {@code flagevaluation} recording. + * + *

The OpenFeature {@code Finally} hook runs synchronously on the caller's evaluation thread, so + * the cost it charges the user's evaluation must stay flat. This benchmark isolates the two stages: + * + *

    + *
  • {@code evalThreadCapture}: what the evaluation thread pays — building the lightweight + * {@link FlagEvalEvent} snapshot (the hook's scalar/shallow capture) plus the non-blocking + * {@code enqueue}. This is the cost that must stay flat under load. + *
  • {@code workerAggregate}: the deferred work that runs off the evaluation thread on the + * background worker — deterministic context prune + canonical-context key + two-tier map + * aggregation. Measured so the off-thread cost is characterized too. + *
+ * + *

Run: {@code ./gradlew :products:feature-flagging:feature-flagging-lib:jmh} (optionally {@code + * -PjmhIncludes=FlagEvaluationHotPathBenchmark}). Use {@code -prof gc} via JMH args for allocs. + */ +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 2, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(NANOSECONDS) +@Fork(value = 1) +public class FlagEvaluationHotPathBenchmark { + + private Map attrs; + + // Eval-thread stage: a writer whose bounded queue is large and never drained, so enqueue cost is + // measured without a competing consumer thread. + private FlagEvaluationWriterImpl writer; + + // Worker stage: the aggregating handler, exercised directly (off the eval thread in production). + private FlagEvaluationWriterImpl.SerializingHandlerForTest handler; + + @Setup(Level.Iteration) + public void setUp() { + attrs = new HashMap<>(); + attrs.put("tier", "enterprise"); + attrs.put("region", "us-east-1"); + attrs.put("seats", 42); + attrs.put("beta", Boolean.TRUE); + + final Config config = Config.get(); + // The factory is never invoked here (neither the writer worker nor the handler is started), + // so a plain factory suffices; createBackendApi() is only called from run(). + final BackendApiFactory factory = new BackendApiFactory(config, null); + final Map ddContext = new HashMap<>(); + ddContext.put("service", "bench-service"); + handler = FlagEvaluationWriterImpl.createHandlerForTest(factory, ddContext); + + // Capacity large enough that the benchmark never overflows within a measurement window. + writer = new FlagEvaluationWriterImpl(1 << 20, Long.MAX_VALUE, NANOSECONDS, factory, config); + } + + /** Eval-thread cost: build the capture snapshot + non-blocking enqueue. */ + @Benchmark + public void evalThreadCapture(final Blackhole blackhole) { + final FlagEvalEvent ev = + new FlagEvalEvent( + "checkout-flag", + "treatment", + "alloc-7", + "targeting_match", + "user-123", + null, + 1_700_000_000_000L, + attrs); + writer.enqueue(ev); + blackhole.consume(ev); + } + + /** Off-thread worker cost: deterministic prune + canonical key + two-tier aggregation. */ + @Benchmark + public void workerAggregate(final Blackhole blackhole) { + final FlagEvalEvent ev = + new FlagEvalEvent( + "checkout-flag", + "treatment", + "alloc-7", + "targeting_match", + "user-123", + null, + 1_700_000_000_000L, + attrs); + handler.aggregateEvent(ev); + blackhole.consume(handler.fullTier.size()); + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 89ba8e64c66..4cc3815b049 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -21,7 +21,9 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import okhttp3.RequestBody; import org.slf4j.Logger; @@ -42,12 +44,17 @@ *

  • Degraded key: (flagKey, variant, allocationKey, reason) — no targetingKey/context. *
  • Canonical context key: sorted entries, type-tagged length-delimited encoding — NOT a hash * (collision-safe, comparable string identity). - *
  • Context pruning: ≤256 fields, string values ≤256 chars. + *
  • Context pruning: deterministic (sort before cut), ≤256 fields, string values ≤256 chars; + * the pruned attributes are what gets aggregated and serialized. *
  • Caps: globalCap=131072, perFlagCap=10000, degradedCap=32768. *
  • Eval-time: min/max of firstEvalMs/lastEvalMs across events in the same bucket. *
  • Runtime default: absent variant means {@code runtimeDefaultUsed=true}. *
  • Flush interval: 10 seconds. - *
  • Queue: bounded MessagePassingBlockingQueue (capacity 2^16), non-blocking offer. + *
  • Queue: bounded MessagePassingBlockingQueue (capacity 2^16), non-blocking offer; on overflow + * the event is dropped and the {@code droppedQueueOverflow} counter is incremented and + * surfaced on flush. + *
  • Shutdown: {@link #close()} drains the queue and performs a final flush before the worker + * thread exits. * */ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { @@ -74,8 +81,15 @@ public class FlagEvaluationWriterImpl implements FlagEvaluationWriter { private static final byte CTX_TAG_OTHER = 'o'; private final MessagePassingBlockingQueue queue; + private final FlagEvaluationSerializingHandler serializer; private final Thread serializerThread; + /** + * Observable counter for events dropped because the bounded hand-off queue was full when the hook + * tried to enqueue (backpressure). Incremented on the hook thread, surfaced on flush. + */ + private final AtomicLong droppedQueueOverflow = new AtomicLong(0); + public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Config config) { this(DEFAULT_CAPACITY, FLUSH_INTERVAL_SECONDS, SECONDS, sco, config); } @@ -86,6 +100,16 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf final TimeUnit timeUnit, final SharedCommunicationObjects sco, final Config config) { + this(capacity, flushInterval, timeUnit, new BackendApiFactory(config, sco), config); + } + + /** Package-private constructor allowing a {@link BackendApiFactory} to be injected for tests. */ + FlagEvaluationWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final BackendApiFactory backendApiFactory, + final Config config) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); final Map context = new HashMap<>(4); context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); @@ -95,13 +119,14 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf if (config.getVersion() != null) { context.put("version", config.getVersion()); } - final FlagEvaluationSerializingHandler serializer = + this.serializer = new FlagEvaluationSerializingHandler( - new BackendApiFactory(config, sco), + backendApiFactory, queue, flushInterval, timeUnit, context, + droppedQueueOverflow, this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EVALUATION_PROCESSOR, serializer); } @@ -113,12 +138,31 @@ public void start() { this.serializerThread.start(); } + /** Test seam: starts the worker thread WITHOUT registering with the global gateway. */ + void startForTest() { + this.serializerThread.start(); + } + + /** Test seam: current full-tier bucket count in the worker's aggregator. */ + int aggregatorFullTierSizeForTest() { + return serializer.fullTier.size(); + } + @Override public void close() { - // Deregister from the gateway before stopping the thread + // Deregister from the gateway so no new events are enqueued. FeatureFlaggingGateway.setFlagEvalWriter(null); - if (this.serializerThread.isAlive()) { - this.serializerThread.interrupt(); + if (!this.serializerThread.isAlive()) { + return; + } + // Ask the worker to drain the queue and final-flush, then interrupt to wake it from poll(). + serializer.requestShutdown(); + this.serializerThread.interrupt(); + try { + // Bounded wait for the worker's final flush so queued events are not lost on shutdown. + this.serializerThread.join(TimeUnit.SECONDS.toMillis(5)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } @@ -127,49 +171,67 @@ public void enqueue(final FlagEvalEvent event) { if (event == null) { return; } - queue.offer(event); // non-blocking; drops silently on overflow (bounded queue) + // Non-blocking offer. On overflow (bounded queue full) drop the event and count it so the loss + // is observable rather than silent; the count is surfaced on the next flush. + if (!queue.offer(event)) { + droppedQueueOverflow.incrementAndGet(); + } } - // ---- Canonical context key ---- - // Sorted entries, type-tagged length-delimited encoding. NOT a hash (collision-safe string key). - // Implementation mirrors dd-trace-go/openfeature/flagevaluation.go canonicalContextKey(). + /** Returns the count of events dropped due to queue-overflow backpressure (observable). */ + long droppedQueueOverflow() { + return droppedQueueOverflow.get(); + } + + // ---- Deterministic context pruning ---- /** - * Builds the canonical context key for the full-tier bucket identity. - * - *

    Algorithm (portable, collision-free): - * - *

      - *
    1. Prune context to ≤256 fields, skip string values >256 chars. - *
    2. Sort keys lexicographically (deterministic). - *
    3. For each entry: encode key as 8-byte-big-endian-length + raw bytes, then type-tag byte + - * encoded value. - *
    - * - *

    Returns an empty string for null/empty context. + * Produces the deterministically-pruned context map used for both the canonical key and the + * serialized payload. Keys are sorted FIRST, then the first {@link #MAX_CONTEXT_FIELDS} + * non-oversized values are kept, so two logically-identical contexts always prune to the same + * subset (and therefore the same canonical key). Oversized string values (>{@link + * #MAX_FIELD_LENGTH} chars) are skipped. Returns an empty map for null/empty input. */ - static String canonicalContextKey(final Map attrs) { + static Map pruneContext(final Map attrs) { if (attrs == null || attrs.isEmpty()) { - return ""; + return java.util.Collections.emptyMap(); } - // Prune and sort - final TreeMap sorted = new TreeMap<>(); - int fieldCount = 0; - for (final Map.Entry entry : attrs.entrySet()) { - if (fieldCount >= MAX_CONTEXT_FIELDS) { + // TreeMap gives a deterministic, sorted ordering BEFORE we apply the field cap. + final TreeMap out = new TreeMap<>(); + final TreeMap sorted = new TreeMap<>(attrs); + int count = 0; + for (final Map.Entry entry : sorted.entrySet()) { + if (count >= MAX_CONTEXT_FIELDS) { break; } final Object v = entry.getValue(); if (v instanceof String && ((String) v).length() > MAX_FIELD_LENGTH) { continue; // skip oversized string values } - sorted.put(entry.getKey(), v); - fieldCount++; + out.put(entry.getKey(), v); + count++; } - if (sorted.isEmpty()) { + return out; + } + + // ---- Canonical context key ---- + // Sorted entries, type-tagged length-delimited encoding. NOT a hash (collision-safe string key). + // Implementation mirrors dd-trace-go/openfeature/flagevaluation.go canonicalContextKey(). + + /** + * Builds the canonical context key from the already-pruned context map for the full-tier bucket + * identity. Expects a pruned, sorted map (e.g. produced by {@link #pruneContext(Map)}); iterating + * a {@link TreeMap} yields keys in sorted order, so the encoding is deterministic. + * + *

    Returns an empty string for null/empty context. + */ + static String canonicalContextKey(final Map prunedAttrs) { + if (prunedAttrs == null || prunedAttrs.isEmpty()) { return ""; } - + // Ensure deterministic ordering even if a non-sorted map was passed. + final Map sorted = + (prunedAttrs instanceof TreeMap) ? prunedAttrs : new TreeMap<>(prunedAttrs); final StringBuilder sb = new StringBuilder(); for (final Map.Entry entry : sorted.entrySet()) { appendLengthDelimited(sb, entry.getKey()); @@ -179,11 +241,9 @@ static String canonicalContextKey(final Map attrs) { } private static void appendLengthDelimited(final StringBuilder sb, final String s) { - // Encode as big-endian 8-byte length prefix + raw chars (Java chars, deterministic) - final long len = s.length(); - sb.append( - String.format( - "%08x", len)); // 8 hex chars ≈ big-endian 4 bytes (sufficient for realistic sizes) + // Encode as big-endian 8-hex-char length prefix + raw chars (deterministic, unambiguous + // field boundary regardless of value content). + sb.append(String.format("%08x", (long) s.length())); sb.append(s); } @@ -212,20 +272,6 @@ private static void appendContextValue(final StringBuilder sb, final Object v) { } } - // ---- Pruning helper (also used in tests via drainAndAggregate) ---- - - static int prunedFieldCount(final Map attrs) { - if (attrs == null) return 0; - int count = 0; - for (final Map.Entry entry : attrs.entrySet()) { - if (count >= MAX_CONTEXT_FIELDS) break; - final Object v = entry.getValue(); - if (v instanceof String && ((String) v).length() > MAX_FIELD_LENGTH) continue; - count++; - } - return count; - } - // ---- Data classes ---- /** Aggregation bucket for a single (full or degraded) key. */ @@ -239,8 +285,8 @@ static class EvalBucket { String allocationKey; String reason; String targetingKey; - Map sampleAttrs; // context sample for serialization (full tier only) - int prunedContextFieldCount; + String errorMessage; + Map prunedAttrs; // pruned context for serialization (full tier only) EvalBucket( final String flagKey, @@ -248,20 +294,26 @@ static class EvalBucket { final String allocationKey, final String reason, final String targetingKey, + final String errorMessage, final long evalTimeMs, final boolean runtimeDefaultUsed, - final Map attrs) { + final Map prunedAttrs) { this.flagKey = flagKey; this.variant = variant; this.allocationKey = allocationKey; this.reason = reason; this.targetingKey = targetingKey; + this.errorMessage = errorMessage; this.firstEvalMs = evalTimeMs; this.lastEvalMs = evalTimeMs; this.count = 1; this.runtimeDefaultUsed = runtimeDefaultUsed; - this.sampleAttrs = attrs; - this.prunedContextFieldCount = prunedFieldCount(attrs); + this.prunedAttrs = prunedAttrs; + } + + /** Number of context fields stored in this bucket (after pruning). */ + int prunedContextFieldCount() { + return prunedAttrs == null ? 0 : prunedAttrs.size(); } void merge(final long evalTimeMs, final boolean isDefault) { @@ -298,6 +350,7 @@ static class FlagEvaluationSerializingHandler implements Runnable { private final JsonAdapter jsonAdapter; final BackendApiFactory backendApiFactory; final Map context; + private final AtomicLong droppedQueueOverflow; private final Runnable errorCallback; // Two-tier aggregation maps (accessed only from the serializer thread; package-private for @@ -314,27 +367,39 @@ static class FlagEvaluationSerializingHandler implements Runnable { // Observable counter for events dropped when both tiers are at capacity final AtomicLong droppedDegradedOverflow = new AtomicLong(0); + // Shutdown coordination: set by close(), drives a final drain+flush before the worker exits. + private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); + private final CountDownLatch finalFlushDone = new CountDownLatch(1); + FlagEvaluationSerializingHandler( final BackendApiFactory backendApiFactory, final MessagePassingBlockingQueue queue, final long flushInterval, final TimeUnit timeUnit, final Map context, + final AtomicLong droppedQueueOverflow, final Runnable errorCallback) { this.queue = queue; this.jsonAdapter = new Moshi.Builder().build().adapter(FlagEvaluationsRequest.class); this.backendApiFactory = backendApiFactory; this.context = context; + this.droppedQueueOverflow = droppedQueueOverflow; this.lastTicks = System.nanoTime(); this.ticksRequiredToFlush = timeUnit.toNanos(flushInterval); this.errorCallback = errorCallback; LOGGER.debug("starting flag evaluation serializer"); } + /** Signals the worker to drain the queue and perform a final flush before exiting. */ + void requestShutdown() { + shutdownRequested.set(true); + } + @Override public void run() { final BackendApi evp = backendApiFactory.createBackendApi(Intake.EVENT_PLATFORM); if (evp == null) { + finalFlushDone.countDown(); errorCallback.run(); throw new IllegalArgumentException("EVP Proxy not available"); } @@ -342,13 +407,18 @@ public void run() { runDutyCycle(evp); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + } finally { + // On exit (interrupt or shutdown request), drain everything still buffered and flush it so + // queued events are not lost on shutdown. + drainAndFlush(evp); + finalFlushDone.countDown(); } LOGGER.debug("flag evaluation processor worker exited."); } private void runDutyCycle(final BackendApi evp) throws InterruptedException { final Thread thread = Thread.currentThread(); - while (!thread.isInterrupted()) { + while (!thread.isInterrupted() && !shutdownRequested.get()) { FlagEvalEvent event; while ((event = queue.poll(100, TimeUnit.MILLISECONDS)) != null) { aggregateEvent(event); @@ -357,14 +427,25 @@ private void runDutyCycle(final BackendApi evp) throws InterruptedException { } } + /** Drains all remaining queued events and performs a final flush. Used on shutdown. */ + void drainAndFlush(final BackendApi evp) { + FlagEvalEvent event; + while ((event = queue.poll()) != null) { + aggregateEvent(event); + } + flush(evp); + } + // ---- Aggregation logic ---- /** Routes an event into the full tier or degraded tier, or drops and counts on overflow. */ void aggregateEvent(final FlagEvalEvent event) { final boolean isDefault = event.variant == null; - // Build full-tier key - final String ctxKey = canonicalContextKey(event.attrs); + // Prune the context deterministically ONCE; both the canonical key and the stored payload + // use this pruned view (never the raw attrs). + final Map prunedAttrs = pruneContext(event.attrs); + final String ctxKey = canonicalContextKey(prunedAttrs); final String fullKey = buildFullKey(event, ctxKey); // Try full tier @@ -383,9 +464,10 @@ void aggregateEvent(final FlagEvalEvent event) { event.allocationKey, event.reason, event.targetingKey, + event.errorMessage, event.evalTimeMs, isDefault, - event.attrs); + prunedAttrs); fullTier.put(fullKey, bucket); globalFullCount++; perFlagCount.put(event.flagKey, flagCount + 1); @@ -407,6 +489,7 @@ void aggregateEvent(final FlagEvalEvent event) { event.allocationKey, event.reason, null, // degraded omits targetingKey + event.errorMessage, event.evalTimeMs, isDefault, null); // degraded omits context @@ -449,7 +532,7 @@ private static String nullToEmpty(final String s) { // ---- Flush logic ---- void flushIfNecessary(final BackendApi evp) { - if (fullTier.isEmpty() && degradedTier.isEmpty()) { + if (fullTier.isEmpty() && degradedTier.isEmpty() && droppedQueueOverflow.get() == 0) { return; } if (shouldFlush()) { @@ -458,6 +541,23 @@ void flushIfNecessary(final BackendApi evp) { } void flush(final BackendApi evp) { + // Surface backpressure (queue-overflow) drops as an observable warning even when there is + // nothing else to flush. + final long qDrops = droppedQueueOverflow.getAndSet(0); + if (qDrops > 0) { + LOGGER.warn( + "flag evaluation queue full — dropped {} evaluation(s) under backpressure" + + " (best-effort telemetry)", + qDrops); + } + final long dgDrops = droppedDegradedOverflow.getAndSet(0); + if (dgDrops > 0) { + LOGGER.warn( + "degraded aggregation tier full — dropped {} evaluation(s); raise degraded cap" + + " (best-effort telemetry)", + dgDrops); + } + if (fullTier.isEmpty() && degradedTier.isEmpty()) { return; } @@ -520,6 +620,7 @@ static class SerializingHandlerForTest extends FlagEvaluationSerializingHandler Long.MAX_VALUE, // effectively never auto-flush TimeUnit.NANOSECONDS, context, + new AtomicLong(0), () -> {}); } @@ -544,7 +645,7 @@ AggregatedState drainAndAggregate() { void simulateFullTierAtCap() { for (int i = globalFullCount; i < GLOBAL_CAP; i++) { final String key = "synthetic-full-" + i; - fullTier.put(key, new EvalBucket(key, "on", "alloc", "split", null, 1L, false, null)); + fullTier.put(key, new EvalBucket(key, "on", "alloc", "split", null, null, 1L, false, null)); globalFullCount++; perFlagCount.merge(key, 1, Integer::sum); } @@ -556,7 +657,8 @@ void simulateFullTierAtCap() { void simulateDegradedTierAtCap() { for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { final String key = "synthetic-dg-" + i; - degradedTier.put(key, new EvalBucket(key, "on", "alloc", "split", null, 1L, false, null)); + degradedTier.put( + key, new EvalBucket(key, "on", "alloc", "split", null, null, 1L, false, null)); } } } @@ -568,6 +670,9 @@ static SerializingHandlerForTest createHandlerForTest( } // ---- Request/response DTOs for JSON serialization ---- + // Structure mirrors the flageval-worker batchedflagevaluations.json schema: + // variant/allocation/flag are {"key": ...} objects, error is {"message": ...}, and per-event + // context nests evaluation attributes under {"evaluation": {...}}. static class FlagEvaluationsRequest { public final Map context; @@ -582,15 +687,16 @@ static class FlagEvaluationsRequest { static class FlagEvaluationEvent { public final long timestamp; - public final FlagInfo flag; + public final FlagKeyObject flag; public final long first_evaluation; public final long last_evaluation; public final long evaluation_count; - public final String variant; // null = omitted (degraded tier or absent) - public final String allocation; // null = omitted + public final KeyObject variant; // null = omitted (degraded tier or runtime default) + public final KeyObject allocation; // null = omitted public final String targeting_key; // null = omitted (degraded tier) public final Boolean runtime_default_used; // null = omitted when false - public final Map context; // null = omitted (degraded tier) + public final EventContext context; // null = omitted (degraded tier) + public final ErrorObject error; // null = omitted (no error) FlagEvaluationEvent( final long timestamp, @@ -602,17 +708,24 @@ static class FlagEvaluationEvent { final String allocation, final String targetingKey, final boolean runtimeDefaultUsed, - final Map context) { + final String errorMessage, + final Map evaluationAttrs) { this.timestamp = timestamp; - this.flag = new FlagInfo(flagKey); + this.flag = new FlagKeyObject(flagKey); this.first_evaluation = firstEvalMs; this.last_evaluation = lastEvalMs; this.evaluation_count = count; - this.variant = (variant != null && !variant.isEmpty()) ? variant : null; - this.allocation = (allocation != null && !allocation.isEmpty()) ? allocation : null; + this.variant = (variant != null && !variant.isEmpty()) ? new KeyObject(variant) : null; + this.allocation = + (allocation != null && !allocation.isEmpty()) ? new KeyObject(allocation) : null; this.targeting_key = targetingKey; this.runtime_default_used = runtimeDefaultUsed ? Boolean.TRUE : null; - this.context = (context != null && !context.isEmpty()) ? context : null; + this.context = + (evaluationAttrs != null && !evaluationAttrs.isEmpty()) + ? new EventContext(evaluationAttrs) + : null; + this.error = + (errorMessage != null && !errorMessage.isEmpty()) ? new ErrorObject(errorMessage) : null; } static FlagEvaluationEvent fromBucket(final EvalBucket bucket, final boolean isFullTier) { @@ -622,19 +735,48 @@ static FlagEvaluationEvent fromBucket(final EvalBucket bucket, final boolean isF bucket.firstEvalMs, bucket.lastEvalMs, bucket.count, - isFullTier ? bucket.variant : bucket.variant, + bucket.variant, bucket.allocationKey, isFullTier ? bucket.targetingKey : null, // degraded omits targetingKey bucket.runtimeDefaultUsed, - isFullTier ? bucket.sampleAttrs : null); // degraded omits context + bucket.errorMessage, + isFullTier ? bucket.prunedAttrs : null); // degraded omits context + } + } + + /** {@code {"key": "..."}} object for variant / allocation. */ + static class KeyObject { + public final String key; + + KeyObject(final String key) { + this.key = key; } } - static class FlagInfo { + /** {@code {"key": "..."}} object for the flag. */ + static class FlagKeyObject { public final String key; - FlagInfo(final String key) { + FlagKeyObject(final String key) { this.key = key; } } + + /** {@code {"message": "..."}} object for an evaluation error. */ + static class ErrorObject { + public final String message; + + ErrorObject(final String message) { + this.message = message; + } + } + + /** Per-event {@code context} object: evaluation attributes nest under {@code evaluation}. */ + static class EventContext { + public final Map evaluation; + + EventContext(final Map evaluation) { + this.evaluation = evaluation; + } + } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index c3f4fa79e88..5c1c41f61ca 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -10,22 +11,39 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.lang.reflect.Type; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import okhttp3.RequestBody; import okio.Buffer; import org.junit.jupiter.api.Test; /** - * Unit tests for FlagEvaluationWriterImpl: two-tier aggregation, canonical context key, caps, and - * EVP transport. + * Unit tests for FlagEvaluationWriterImpl: two-tier aggregation, canonical context key, caps, + * deterministic pruning, schema-structural payload validation, and EVP transport. */ class FlagEvaluationWriterImplTest { + // Moshi adapter for parsing the emitted JSON back into a Map for structural assertions. + private static final JsonAdapter> JSON_MAP; + + static { + final Moshi moshi = new Moshi.Builder().build(); + final Type type = Types.newParameterizedType(Map.class, String.class, Object.class); + JSON_MAP = moshi.adapter(type); + } + // ---- helpers ---- private static FlagEvalEvent event( @@ -40,12 +58,17 @@ private static FlagEvalEvent event( flagKey, variant, allocationKey, reason, targetingKey, evalTimeMs, attrs); } + private static FlagEvalEvent errorEvent( + String flagKey, String reason, String errorMessage, long evalTimeMs) { + return new FlagEvalEvent( + flagKey, null, null, reason, null, errorMessage, evalTimeMs, Collections.emptyMap()); + } + private static FlagEvalEvent simpleEvent(String flagKey, String variant) { return event( flagKey, variant, "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); } - // Build a writer with a mocked BackendApi, flushing synchronously via triggerFlush(). private TestWriterSetup buildTestWriter(BackendApi mockEvp) { BackendApiFactory factory = mock(BackendApiFactory.class); when(factory.createBackendApi(any())).thenReturn(mockEvp); @@ -53,7 +76,6 @@ private TestWriterSetup buildTestWriter(BackendApi mockEvp) { Map context = new HashMap<>(); context.put("service", "test-service"); - // Use a flush interval of 0 so we can call flush manually in tests FlagEvaluationWriterImpl.SerializingHandlerForTest handler = FlagEvaluationWriterImpl.createHandlerForTest(factory, context); @@ -71,20 +93,37 @@ static class TestWriterSetup { } } + /** Captures the JSON body posted by a flush(), parsed into a Map. */ + @SuppressWarnings("unchecked") + private Map flushAndCaptureJson(TestWriterSetup setup) throws Exception { + final RequestBody[] captured = {null}; + when(setup.mockEvp.post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured[0] = inv.getArgument(1); + return null; + }); + setup.handler.drainAndAggregate(); + setup.handler.flush(setup.mockEvp); + assertNotNull(captured[0], "RequestBody must have been posted"); + Buffer buf = new Buffer(); + captured[0].writeTo(buf); + return JSON_MAP.fromJson(buf.readUtf8()); + } + // ---- test: two identical events -> one full-tier bucket, count 2, first <= last ---- @Test - void identicalEventsAggregatIntoOneBucketWithCount2() throws Exception { + void identicalEventsAggregateIntoOneBucketWithCount2() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e1 = - event("flag-a", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); - FlagEvalEvent e2 = - event("flag-a", "on", "alloc1", "targeting_match", "user-1", 2000L, Collections.emptyMap()); - - setup.handler.add(e1); - setup.handler.add(e2); + setup.handler.add( + event( + "flag-a", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap())); + setup.handler.add( + event( + "flag-a", "on", "alloc1", "targeting_match", "user-1", 2000L, Collections.emptyMap())); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals( @@ -96,10 +135,32 @@ void identicalEventsAggregatIntoOneBucketWithCount2() throws Exception { assertTrue(bucket.firstEvalMs <= bucket.lastEvalMs); } - // ---- test: type-tagged canonical key distinguishes int 1 vs string "1" ---- + // ---- G1: variant comes from the OpenFeature variant, distinct from the evaluated value ---- + + @Test + void variantIsTheVariantKeyNotTheEvaluatedValue() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + // The event already carries the variant key ("on"); the evaluated value ("on-value") is + // intentionally NOT part of the event — the hook is responsible for sourcing variant from + // details.getVariant(). Here we assert the emitted variant.key == the variant key, never a + // value. + setup.handler.add( + event( + "flag-v", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap())); + + Map json = flushAndCaptureJson(setup); + Map ev = firstEvent(json); + Map variantObj = (Map) ev.get("variant"); + assertNotNull(variantObj, "variant object must be present"); + assertEquals("on", variantObj.get("key"), "variant.key must be the variant key"); + } + + // ---- type-tagged canonical key distinguishes int 1 vs string "1" ---- @Test - void differentValueTypesProduceDifferentBuckets() throws Exception { + void differentValueTypesProduceDifferentBuckets() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); @@ -108,13 +169,10 @@ void differentValueTypesProduceDifferentBuckets() throws Exception { Map attrsStr = new HashMap<>(); attrsStr.put("score", "1"); - FlagEvalEvent e1 = - event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsInt); - FlagEvalEvent e2 = - event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsStr); - - setup.handler.add(e1); - setup.handler.add(e2); + setup.handler.add( + event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsInt)); + setup.handler.add( + event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsStr)); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals( @@ -123,39 +181,31 @@ void differentValueTypesProduceDifferentBuckets() throws Exception { "int 1 vs String \"1\" must produce 2 distinct buckets (type-tagged canonical key)"); } - // ---- test: full-tier overflow past GLOBAL_CAP routes to degraded ---- + // ---- full-tier overflow past GLOBAL_CAP routes to degraded ---- @Test - void globalCapOverflowRoutesToDegradedTier() throws Exception { + void globalCapOverflowRoutesToDegradedTier() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - // Fill global cap first setup.handler.simulateFullTierAtCap(); - - // Next event must land in degraded - FlagEvalEvent e = simpleEvent("extra-flag", "on"); - setup.handler.add(e); + setup.handler.add(simpleEvent("extra-flag", "on")); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertTrue(state.degradedTier.size() > 0, "Overflow past GLOBAL_CAP must route to degraded"); assertEquals(0, state.droppedDegradedOverflow, "No drops yet (degraded not full)"); } - // ---- test: degraded-tier overflow past DEGRADED_CAP increments dropped counter ---- + // ---- G4: degraded-tier overflow increments the observable dropped counter ---- @Test - void degradedCapOverflowIncrementsDroppedCounter() throws Exception { + void degradedCapOverflowIncrementsDroppedCounter() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - // Fill both tiers setup.handler.simulateFullTierAtCap(); setup.handler.simulateDegradedTierAtCap(); - - // Next event must be dropped-and-counted - FlagEvalEvent e = simpleEvent("drop-flag", "on"); - setup.handler.add(e); + setup.handler.add(simpleEvent("drop-flag", "on")); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertTrue( @@ -163,16 +213,108 @@ void degradedCapOverflowIncrementsDroppedCounter() throws Exception { "Beyond DEGRADED_CAP must increment droppedDegradedOverflow counter"); } - // ---- test: absent variant -> runtimeDefaultUsed=true ---- + // ---- G4: queue-overflow backpressure increments an observable drop counter on enqueue ---- + + @Test + void queueOverflowIncrementsObservableDropCounter() { + // Tiny capacity so we can overflow deterministically. Capacity must be a power of two. + BackendApi mockEvp = mock(BackendApi.class); + BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl(2, 10L, java.util.concurrent.TimeUnit.SECONDS, factory, cfg()); + + // Never start() the worker, so nothing drains the queue; offers past capacity must be counted. + for (int i = 0; i < 100; i++) { + writer.enqueue(simpleEvent("of-flag", "on")); + } + + assertTrue( + writer.droppedQueueOverflow() > 0, + "queue-overflow drops must be counted in the observable droppedQueueOverflow counter"); + } + + private static datadog.trace.api.Config cfg() { + datadog.trace.api.Config config = mock(datadog.trace.api.Config.class); + when(config.getServiceName()).thenReturn("test-service"); + return config; + } + + // ---- G2: enqueue does NOT aggregate (no buckets formed) — aggregation is the worker's job ---- + + @Test + void enqueueDoesNotAggregateOnTheCallingThread() { + BackendApi mockEvp = mock(BackendApi.class); + BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS, factory, cfg()); + + // Do NOT start the worker. enqueue() must only offer to the queue; the aggregation maps stay + // empty because nothing on the calling thread aggregates. + writer.enqueue(simpleEvent("g2-flag", "on")); + writer.enqueue(simpleEvent("g2-flag", "on")); + + assertEquals( + 0, + writer.aggregatorFullTierSizeForTest(), + "enqueue must not build aggregation buckets on the calling thread"); + assertEquals(0, writer.droppedQueueOverflow(), "no overflow expected for 2 events in cap 16"); + } + + // ---- G5: close() drains the queue and final-flushes before the worker exits ---- + + @Test + void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { + final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); + final RequestBody[] captured = {null}; + BackendApi mockEvp = mock(BackendApi.class); + when(mockEvp.post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + inv -> { + captured[0] = inv.getArgument(1); + posted.countDown(); + return null; + }); + BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + + // Large flush interval so the ONLY flush that can happen is the shutdown drain-flush. + FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 64, + java.util.concurrent.TimeUnit.DAYS.toSeconds(1), + java.util.concurrent.TimeUnit.SECONDS, + factory, + cfg()); + writer.startForTest(); + writer.enqueue(simpleEvent("shutdown-flag", "on")); + + // close() must drain + final-flush, not just interrupt. + writer.close(); + + assertTrue( + posted.await(5, java.util.concurrent.TimeUnit.SECONDS), + "close() must drain the queue and final-flush before exit"); + assertNotNull(captured[0], "a flag evaluation payload must have been posted on shutdown"); + Buffer buf = new Buffer(); + captured[0].writeTo(buf); + @SuppressWarnings("unchecked") + Map json = JSON_MAP.fromJson(buf.readUtf8()); + assertNotNull( + eventForFlag(json, "shutdown-flag"), "the queued event must be in the final flush"); + } + + // ---- absent variant -> runtimeDefaultUsed=true ---- @Test - void absentVariantSetsRuntimeDefaultUsed() throws Exception { + void absentVariantSetsRuntimeDefaultUsed() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e = - event("flag-c", null, "alloc1", "default", "user-1", 1000L, Collections.emptyMap()); - setup.handler.add(e); + setup.handler.add( + event("flag-c", null, "alloc1", "default", "user-1", 1000L, Collections.emptyMap())); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); @@ -180,33 +322,30 @@ void absentVariantSetsRuntimeDefaultUsed() throws Exception { assertTrue(bucket.runtimeDefaultUsed, "Absent variant must set runtimeDefaultUsed=true"); } - // ---- test: degraded event omits targeting_key + context ---- + // ---- degraded event omits targeting_key + context ---- @Test void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - // Route to degraded by filling full tier setup.handler.simulateFullTierAtCap(); Map attrs = new HashMap<>(); attrs.put("region", "us-east-1"); - FlagEvalEvent e = event("dg-flag", "on", "alloc1", "targeting_match", "user-99", 1000L, attrs); - setup.handler.add(e); + setup.handler.add(event("dg-flag", "on", "alloc1", "targeting_match", "user-99", 1000L, attrs)); - FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); - assertTrue(state.degradedTier.size() > 0); - // Degraded key includes only flagKey, variant, allocationKey, reason — no targetingKey/context - String degradedKey = state.degradedTier.keySet().iterator().next(); - assertFalse(degradedKey.contains("user-99"), "Degraded key must omit targetingKey"); - assertFalse(degradedKey.contains("us-east-1"), "Degraded key must omit context"); + Map json = flushAndCaptureJson(setup); + Map dg = eventForFlag(json, "dg-flag"); + assertNotNull(dg, "degraded event must be emitted"); + assertNull(dg.get("targeting_key"), "Degraded event must omit targeting_key"); + assertNull(dg.get("context"), "Degraded event must omit context"); } - // ---- test: context >256 fields is pruned ---- + // ---- G6: context >256 fields is pruned to <=256 and stored pruned ---- @Test - void contextExceeding256FieldsIsPruned() throws Exception { + void contextExceeding256FieldsIsPrunedToStoredPrunedAttrs() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); @@ -214,24 +353,45 @@ void contextExceeding256FieldsIsPruned() throws Exception { for (int i = 0; i < 300; i++) { hugeAttrs.put("key" + i, "v" + i); } - FlagEvalEvent e = - event("flag-d", "on", "alloc1", "targeting_match", "user-1", 1000L, hugeAttrs); - setup.handler.add(e); + setup.handler.add( + event("flag-d", "on", "alloc1", "targeting_match", "user-1", 1000L, hugeAttrs)); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); - assertTrue(bucket.prunedContextFieldCount <= 256, "Context must be pruned to <=256 fields"); + assertEquals( + 256, + bucket.prunedContextFieldCount(), + "Context must be pruned to exactly 256 fields and the PRUNED attrs stored"); + assertEquals(256, bucket.prunedAttrs.size(), "stored prunedAttrs map must be the pruned view"); + } + + // ---- G6: pruning is deterministic — same input -> same kept subset across runs ---- + + @Test + void pruningIsDeterministicSortBeforeCut() { + Map attrs = new HashMap<>(); + for (int i = 0; i < 300; i++) { + attrs.put(String.format("k%03d", i), "v" + i); + } + Map p1 = FlagEvaluationWriterImpl.pruneContext(attrs); + Map p2 = FlagEvaluationWriterImpl.pruneContext(new HashMap<>(attrs)); + assertEquals(256, p1.size()); + assertEquals(p1.keySet(), p2.keySet(), "Pruned key set must be deterministic across calls"); + // Sorted-before-cut means the first 256 sorted keys survive: k000..k255, never k256+. + assertTrue(p1.containsKey("k000"), "lowest sorted key must survive"); + assertTrue(p1.containsKey("k255"), "256th sorted key must survive"); + assertFalse(p1.containsKey("k256"), "257th sorted key must be cut"); + assertFalse(p1.containsKey("k299"), "highest sorted key must be cut"); } - // ---- test: string context value >256 chars is skipped ---- + // ---- G6: string context value >256 chars is skipped from the pruned attrs ---- @Test - void contextValueExceeding256CharsIsSkipped() throws Exception { + void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - // Build a string of 300 'x' chars for Java 8 compatibility (no String.repeat()) StringBuilder longValBuilder = new StringBuilder(300); for (int i = 0; i < 300; i++) longValBuilder.append('x'); String longVal = longValBuilder.toString(); @@ -239,77 +399,216 @@ void contextValueExceeding256CharsIsSkipped() throws Exception { Map attrs = new HashMap<>(); attrs.put("long-val", longVal); // >256 chars attrs.put("short-val", "ok"); - FlagEvalEvent e = event("flag-e", "on", "alloc1", "targeting_match", "user-1", 1000L, attrs); - setup.handler.add(e); + setup.handler.add(event("flag-e", "on", "alloc1", "targeting_match", "user-1", 1000L, attrs)); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); - // The key must not include the long value (skipped at prune step) - String fullKey = state.fullTier.keySet().iterator().next(); - assertFalse(fullKey.contains(longVal), "Context string value >256 chars must be skipped"); + FlagEvaluationWriterImpl.EvalBucket bucket = state.fullTier.values().iterator().next(); + assertFalse( + bucket.prunedAttrs.containsKey("long-val"), + "Oversized string value must be skipped from the pruned attrs"); + assertTrue(bucket.prunedAttrs.containsKey("short-val"), "Normal value must survive pruning"); } - // ---- test: flush posts to "flagevaluations" endpoint with required fields ---- + // ---- flush posts to the "flagevaluations" endpoint ---- @Test - void flushPostsToFlagevaluationsEndpointWithRequiredFields() throws Exception { + void flushPostsToFlagevaluationsEndpoint() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e = - event("flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); - setup.handler.add(e); - // Must drain+aggregate first so the aggregation maps are populated + setup.handler.add( + event( + "flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap())); setup.handler.drainAndAggregate(); - setup.handler.flush(mockEvp); - // Verify post was called with "flagevaluations" verify(mockEvp).post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false)); } - // ---- test: flush includes required schema fields in JSON body ---- + // ---- G3: emitted full-tier payload validates against the flageval-worker schema shape ---- @Test - void flushJsonBodyContainsRequiredFields() throws Exception { + void fullTierPayloadValidatesAgainstWorkerSchema() throws Exception { BackendApi mockEvp = mock(BackendApi.class); - // Capture the RequestBody - final RequestBody[] captured = {null}; - when(mockEvp.post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false))) - .thenAnswer( - inv -> { - captured[0] = inv.getArgument(1); - return null; - }); + TestWriterSetup setup = buildTestWriter(mockEvp); + Map attrs = new HashMap<>(); + attrs.put("region", "us-east-1"); + setup.handler.add(event("my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, attrs)); + + Map json = flushAndCaptureJson(setup); + FlagEvalSchema.assertValidBatch(json); + + // And spot-check the object-vs-string shape the prior code got wrong (G3): + Map ev = firstEvent(json); + assertObjectWithKey(ev.get("variant"), "on", "variant must serialize as {\"key\":...}"); + assertObjectWithKey( + ev.get("allocation"), "alloc-x", "allocation must serialize as {\"key\":...}"); + assertObjectWithKey(ev.get("flag"), "my-flag", "flag must serialize as {\"key\":...}"); + // context must nest under "evaluation", not raw at top level (additionalProperties:false). + Map ctx = (Map) ev.get("context"); + assertNotNull(ctx, "full-tier context must be present"); + Map evalAttrs = (Map) ctx.get("evaluation"); + assertNotNull(evalAttrs, "context.evaluation must hold the attributes"); + assertEquals("us-east-1", evalAttrs.get("region")); + } + + // ---- G3: error path serializes error as {"message":...} and validates against the schema ---- + + @Test + void errorPayloadSerializesErrorObjectAndValidatesSchema() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - FlagEvalEvent e = - event( - "my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, Collections.emptyMap()); - setup.handler.add(e); - // Must drain+aggregate first so the aggregation maps are populated - setup.handler.drainAndAggregate(); - setup.handler.flush(mockEvp); + setup.handler.add(errorEvent("err-flag", "error", "type mismatch", 5000L)); - assertNotNull(captured[0], "RequestBody must have been posted"); + Map json = flushAndCaptureJson(setup); + FlagEvalSchema.assertValidBatch(json); - // Read body as string and verify required fields - Buffer buf = new Buffer(); - captured[0].writeTo(buf); - String json = buf.readUtf8(); + Map ev = firstEvent(json); + Map error = (Map) ev.get("error"); + assertNotNull(error, "error must be present on the error path"); + assertEquals( + "type mismatch", error.get("message"), "error must serialize as {\"message\":...}"); + // runtime default: absent variant -> runtime_default_used true. + assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); + } - assertTrue(json.contains("flagEvaluations"), "Payload must contain flagEvaluations array"); - assertTrue(json.contains("\"key\""), "Flag key field required"); - assertTrue(json.contains("my-flag"), "Flag key value must be in payload"); - assertTrue( - json.contains("first_evaluation") || json.contains("firstEvaluation"), - "first_evaluation required"); - assertTrue( - json.contains("last_evaluation") || json.contains("lastEvaluation"), - "last_evaluation required"); - assertTrue( - json.contains("evaluation_count") || json.contains("evaluationCount"), - "evaluation_count required"); + private static void assertObjectWithKey(Object o, String expectedKey, String msg) { + assertTrue(o instanceof Map, msg + " (must be a JSON object, not a bare string)"); + assertEquals(expectedKey, ((Map) o).get("key"), msg); + } + + @SuppressWarnings("unchecked") + private static Map firstEvent(Map batch) { + List events = (List) batch.get("flagEvaluations"); + assertNotNull(events, "flagEvaluations array must be present"); + assertFalse(events.isEmpty(), "flagEvaluations must not be empty"); + return (Map) events.get(0); + } + + @SuppressWarnings("unchecked") + private static Map eventForFlag(Map batch, String flagKey) { + List events = (List) batch.get("flagEvaluations"); + for (Object o : events) { + Map ev = (Map) o; + Map flag = (Map) ev.get("flag"); + if (flag != null && flagKey.equals(flag.get("key"))) { + return ev; + } + } + return null; + } + + /** + * Minimal structural validator that enforces the hard constraints of the real flageval-worker + * {@code batchedflagevaluations.json} schema (vendored at {@code + * src/test/resources/flagevaluation/batchedflagevaluations.json}): required fields, integer + * types, {@code {"key":...}} / {@code {"message":...}} object shapes, and {@code + * additionalProperties:false} at the event and key-object levels. + */ + static final class FlagEvalSchema { + private static final Set ALLOWED_EVENT_KEYS = + new HashSet<>( + Arrays.asList( + "timestamp", + "flag", + "first_evaluation", + "last_evaluation", + "evaluation_count", + "runtime_default_used", + "targeting_key", + "context", + "variant", + "allocation", + "targeting_rule", + "error")); + private static final Set REQUIRED_EVENT_KEYS = + new HashSet<>( + Arrays.asList( + "timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count")); + + @SuppressWarnings("unchecked") + static void assertValidBatch(Map batch) { + assertTrue( + batch.containsKey("flagEvaluations"), + "batch must contain required flagEvaluations array"); + Object arr = batch.get("flagEvaluations"); + assertTrue(arr instanceof List, "flagEvaluations must be an array"); + for (Object o : (List) arr) { + assertTrue(o instanceof Map, "each flag evaluation must be an object"); + assertValidEvent((Map) o); + } + } + + static void assertValidEvent(Map ev) { + // additionalProperties:false at the event level + for (String k : ev.keySet()) { + assertTrue(ALLOWED_EVENT_KEYS.contains(k), "unexpected event property not in schema: " + k); + } + // required fields present + for (String r : REQUIRED_EVENT_KEYS) { + assertTrue(ev.containsKey(r), "required field missing: " + r); + } + // integer-typed required fields (Moshi parses JSON numbers as Double; assert integral) + assertIntegral(ev.get("timestamp"), "timestamp"); + assertIntegral(ev.get("first_evaluation"), "first_evaluation"); + assertIntegral(ev.get("last_evaluation"), "last_evaluation"); + assertIntegral(ev.get("evaluation_count"), "evaluation_count"); + // key-objects + assertKeyObject(ev.get("flag"), "flag", true); + assertKeyObject(ev.get("variant"), "variant", false); + assertKeyObject(ev.get("allocation"), "allocation", false); + assertKeyObject(ev.get("targeting_rule"), "targeting_rule", false); + // error object {"message": string} + Object error = ev.get("error"); + if (error != null) { + assertTrue(error instanceof Map, "error must be an object"); + Map e = (Map) error; + assertTrue(e.containsKey("message"), "error.message is required"); + assertEquals( + new HashSet<>(Collections.singletonList("message")), + e.keySet(), + "error object allows only {message}"); + } + // runtime_default_used must be boolean if present + Object rdu = ev.get("runtime_default_used"); + if (rdu != null) { + assertTrue(rdu instanceof Boolean, "runtime_default_used must be a boolean"); + } + // context: only {evaluation, dd} allowed + Object context = ev.get("context"); + if (context != null) { + assertTrue(context instanceof Map, "context must be an object"); + for (Object k : ((Map) context).keySet()) { + assertTrue( + "evaluation".equals(k) || "dd".equals(k), + "context allows only {evaluation, dd}, got: " + k); + } + } + } + + private static void assertIntegral(Object v, String field) { + assertNotNull(v, field + " must be present"); + assertTrue(v instanceof Number, field + " must be a number"); + double d = ((Number) v).doubleValue(); + assertEquals(Math.floor(d), d, 0.0, field + " must be an integer"); + } + + private static void assertKeyObject(Object o, String field, boolean required) { + if (o == null) { + assertFalse(required, field + " is required"); + return; + } + assertTrue(o instanceof Map, field + " must be a {\"key\":...} object, not a bare value"); + Map m = (Map) o; + assertTrue(m.containsKey("key"), field + ".key is required"); + assertEquals( + new HashSet<>(Collections.singletonList("key")), + m.keySet(), + field + " object allows only {key}"); + assertTrue(m.get("key") instanceof String, field + ".key must be a string"); + } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/resources/flagevaluation/batchedflagevaluations.json b/products/feature-flagging/feature-flagging-lib/src/test/resources/flagevaluation/batchedflagevaluations.json new file mode 100644 index 00000000000..b3a4b1fda67 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/resources/flagevaluation/batchedflagevaluations.json @@ -0,0 +1,184 @@ +{ + "title": "BatchedFlagEvaluations", + "type": "object", + "properties": { + "context": { + "title": "InputContextDatadog", + "type": "object", + "properties": { + "geo": { + "type": "object", + "properties": { + "country_iso_code": { "type": "string" }, + "country": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "rum": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "id": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "service": { "type": "string" }, + "version": { "type": "string" }, + "env": { "type": "string" }, + "device": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "brand": { "type": "string" }, + "model": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "os": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "version": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + }, + "flagEvaluations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "timestamp": { + "type": "integer", + "description": "The timestamp (milliseconds since epoch) at which the evaluation occurred." + }, + "flag": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "first_evaluation": { + "type": "integer", + "minimum": 1759276800000 + }, + "last_evaluation": { + "type": "integer", + "minimum": 1759276800000 + }, + "evaluation_count": { + "type": "integer", + "minimum": 1 + }, + "runtime_default_used": { "type": "boolean" }, + "targeting_key": { "type": "string" }, + "context": { + "type": "object", + "properties": { + "evaluation": { "type": "object" }, + "dd": { + "type": "object", + "properties": { + "service": { "type": "string" }, + "rum": { + "type": "object", + "properties": { + "application": { + "type": "object", + "properties": { + "id": { "type": "string" } + }, + "required": [], + "additionalProperties": false + }, + "view": { + "type": "object", + "properties": { + "url": { "type": "string" } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [], + "additionalProperties": true + } + }, + "required": [], + "additionalProperties": false + }, + "variant": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "allocation": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "targeting_rule": { + "type": "object", + "properties": { + "key": { "type": "string" } + }, + "required": ["key"], + "additionalProperties": false + }, + "error": { + "type": "object", + "properties": { + "message": { "type": "string" } + }, + "required": ["message"], + "additionalProperties": false + } + }, + "required": [ + "timestamp", + "flag", + "first_evaluation", + "last_evaluation", + "evaluation_count" + ], + "additionalProperties": false + } + } + }, + "required": ["flagEvaluations"], + "additionalProperties": false +} From c8c95a5616bd19a8696765f8417c7b14f2d2c4c6 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 13 Jun 2026 11:28:24 -0400 Subject: [PATCH 08/10] chore(openfeature): remove internal planning annotations --- .../api/openfeature/FlagEvalEVPHookTest.java | 4 ++-- .../FlagEvaluationWriterImplTest.java | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java index 726da70cb6d..69630a8ccee 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -108,7 +108,7 @@ void finallyAfterEnqueuesEventWithAllBasicFields() { assertEquals("alloc-1", e.allocationKey); } - // ---- G1: variant comes from details.getVariant(), NOT details.getValue() ---- + // ---- variant comes from details.getVariant(), NOT details.getValue() ---- @Test void variantIsTheVariantKeyNotTheEvaluatedValue() { @@ -199,7 +199,7 @@ void absentVariantProducesNullVariant() { assertNull(captured.get().variant, "Absent variant must stay null (runtime default)"); } - // ---- test: error message captured from details (G3 error object support) ---- + // ---- test: error message captured from details (error object support) ---- @Test void errorMessageCapturedFromDetails() { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index 5c1c41f61ca..fe22502bebf 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -135,7 +135,7 @@ void identicalEventsAggregateIntoOneBucketWithCount2() { assertTrue(bucket.firstEvalMs <= bucket.lastEvalMs); } - // ---- G1: variant comes from the OpenFeature variant, distinct from the evaluated value ---- + // ---- variant comes from the OpenFeature variant, distinct from the evaluated value ---- @Test void variantIsTheVariantKeyNotTheEvaluatedValue() throws Exception { @@ -196,7 +196,7 @@ void globalCapOverflowRoutesToDegradedTier() { assertEquals(0, state.droppedDegradedOverflow, "No drops yet (degraded not full)"); } - // ---- G4: degraded-tier overflow increments the observable dropped counter ---- + // ---- degraded-tier overflow increments the observable dropped counter ---- @Test void degradedCapOverflowIncrementsDroppedCounter() { @@ -213,7 +213,7 @@ void degradedCapOverflowIncrementsDroppedCounter() { "Beyond DEGRADED_CAP must increment droppedDegradedOverflow counter"); } - // ---- G4: queue-overflow backpressure increments an observable drop counter on enqueue ---- + // ---- queue-overflow backpressure increments an observable drop counter on enqueue ---- @Test void queueOverflowIncrementsObservableDropCounter() { @@ -240,7 +240,7 @@ private static datadog.trace.api.Config cfg() { return config; } - // ---- G2: enqueue does NOT aggregate (no buckets formed) — aggregation is the worker's job ---- + // ---- enqueue does NOT aggregate (no buckets formed) — aggregation is the worker's job ---- @Test void enqueueDoesNotAggregateOnTheCallingThread() { @@ -263,7 +263,7 @@ void enqueueDoesNotAggregateOnTheCallingThread() { assertEquals(0, writer.droppedQueueOverflow(), "no overflow expected for 2 events in cap 16"); } - // ---- G5: close() drains the queue and final-flushes before the worker exits ---- + // ---- close() drains the queue and final-flushes before the worker exits ---- @Test void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { @@ -342,7 +342,7 @@ void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { assertNull(dg.get("context"), "Degraded event must omit context"); } - // ---- G6: context >256 fields is pruned to <=256 and stored pruned ---- + // ---- context >256 fields is pruned to <=256 and stored pruned ---- @Test void contextExceeding256FieldsIsPrunedToStoredPrunedAttrs() { @@ -366,7 +366,7 @@ void contextExceeding256FieldsIsPrunedToStoredPrunedAttrs() { assertEquals(256, bucket.prunedAttrs.size(), "stored prunedAttrs map must be the pruned view"); } - // ---- G6: pruning is deterministic — same input -> same kept subset across runs ---- + // ---- pruning is deterministic — same input -> same kept subset across runs ---- @Test void pruningIsDeterministicSortBeforeCut() { @@ -385,7 +385,7 @@ void pruningIsDeterministicSortBeforeCut() { assertFalse(p1.containsKey("k299"), "highest sorted key must be cut"); } - // ---- G6: string context value >256 chars is skipped from the pruned attrs ---- + // ---- string context value >256 chars is skipped from the pruned attrs ---- @Test void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { @@ -426,7 +426,7 @@ void flushPostsToFlagevaluationsEndpoint() throws Exception { verify(mockEvp).post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false)); } - // ---- G3: emitted full-tier payload validates against the flageval-worker schema shape ---- + // ---- emitted full-tier payload validates against the flageval-worker schema shape ---- @Test void fullTierPayloadValidatesAgainstWorkerSchema() throws Exception { @@ -440,7 +440,7 @@ void fullTierPayloadValidatesAgainstWorkerSchema() throws Exception { Map json = flushAndCaptureJson(setup); FlagEvalSchema.assertValidBatch(json); - // And spot-check the object-vs-string shape the prior code got wrong (G3): + // And spot-check the object-vs-string shape the prior code got wrong: Map ev = firstEvent(json); assertObjectWithKey(ev.get("variant"), "on", "variant must serialize as {\"key\":...}"); assertObjectWithKey( @@ -454,7 +454,7 @@ void fullTierPayloadValidatesAgainstWorkerSchema() throws Exception { assertEquals("us-east-1", evalAttrs.get("region")); } - // ---- G3: error path serializes error as {"message":...} and validates against the schema ---- + // ---- error path serializes error as {"message":...} and validates against the schema ---- @Test void errorPayloadSerializesErrorObjectAndValidatesSchema() throws Exception { From c459981b15a45e9aec7d4028a298638e0af47d6a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 15 Jun 2026 11:29:03 -0400 Subject: [PATCH 09/10] fix(openfeature): suppress single-thread SpotBugs MT findings on flag evaluation writer The flag evaluation serializer's lastTicks and globalFullCount fields are confined to the single serializer thread but are written from more than one method, which SpotBugs flags as AT_NONATOMIC_64BIT_PRIMITIVE and AT_STALE_THREAD_WRITE_OF_PRIMITIVE. Annotate both fields with @SuppressFBWarnings and a thread-confinement justification, matching the existing convention used across the tracer. --- .../com/datadog/featureflag/FlagEvaluationWriterImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 4cc3815b049..b942661c675 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -16,6 +16,7 @@ import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; import datadog.trace.api.intake.Intake; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -345,6 +346,10 @@ static class AggregatedState { static class FlagEvaluationSerializingHandler implements Runnable { private final MessagePassingBlockingQueue queue; private final long ticksRequiredToFlush; + + @SuppressFBWarnings( + value = "AT_NONATOMIC_64BIT_PRIMITIVE", + justification = "the field is confined to the single serializer thread") private long lastTicks; private final JsonAdapter jsonAdapter; @@ -362,6 +367,9 @@ static class FlagEvaluationSerializingHandler implements Runnable { final Map perFlagCount = new HashMap<>(); // Global full-tier count + @SuppressFBWarnings( + value = "AT_STALE_THREAD_WRITE_OF_PRIMITIVE", + justification = "the field is confined to the single serializer thread") int globalFullCount = 0; // Observable counter for events dropped when both tiers are at capacity From 8ef8ea42fc87eaa604a71a835023c1f896ad4cca Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 16 Jun 2026 03:16:24 -0400 Subject: [PATCH 10/10] fix(openfeature): align EVP flagevaluation aggregation with worker schema --- .../api/openfeature/FlagEvalEVPHook.java | 29 +- .../api/openfeature/FlagEvalEVPHookTest.java | 40 ++- .../flagevaluation/FlagEvalEvent.java | 10 +- .../feature-flagging-lib/build.gradle.kts | 3 + .../feature-flagging-lib/gradle.lockfile | 33 +- .../FlagEvaluationHotPathBenchmark.java | 18 +- .../featureflag/FlagEvaluationWriterImpl.java | 44 ++- .../FlagEvaluationWriterImplTest.java | 304 ++++++++++-------- 8 files changed, 267 insertions(+), 214 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java index 1d814a7badc..089d77044ac 100644 --- a/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java +++ b/products/feature-flagging/feature-flagging-api/src/main/java/datadog/trace/api/openfeature/FlagEvalEVPHook.java @@ -3,15 +3,12 @@ import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; import datadog.trace.api.featureflag.flagevaluation.FlagEvaluationWriter; -import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.Hook; import dev.openfeature.sdk.HookContext; import dev.openfeature.sdk.ImmutableMetadata; import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.Set; /** * OpenFeature {@code Hook} that captures flag evaluation events for EVP {@code flagevaluation} @@ -83,10 +80,6 @@ public void finallyAfter( // evaluated value. A null variant means no variant was selected (runtime default). final String variant = details.getVariant(); - // reason: lowercased - final String reason = - details.getReason() != null ? details.getReason().toLowerCase() : "unknown"; - // error message: prefer the human-readable message; fall back to the error code name when // the message is empty (some providers populate only the code). null on success. String errorMessage = details.getErrorMessage(); @@ -106,33 +99,17 @@ public void finallyAfter( w.enqueue( new FlagEvalEvent( - flagKey, - variant, - allocationKey, - reason, - targetingKey, - errorMessage, - evalTimeMs, - attrs)); + flagKey, variant, allocationKey, targetingKey, errorMessage, evalTimeMs, attrs)); } catch (Exception e) { // Never let EVP recording break flag evaluation } } - /** Extracts a flat {@code Map} from the evaluation context attributes. */ + /** Extracts converted, flattened attributes from the evaluation context. */ private Map extractAttrs(final HookContext ctx) { if (ctx == null || ctx.getCtx() == null) { return Collections.emptyMap(); } - final EvaluationContext evalCtx = ctx.getCtx(); - final Set keys = evalCtx.keySet(); - if (keys.isEmpty()) { - return Collections.emptyMap(); - } - final Map result = new HashMap<>(keys.size() * 2); - for (final String key : keys) { - result.put(key, evalCtx.getValue(key)); - } - return result; + return DDEvaluator.flattenContext(ctx.getCtx()); } } diff --git a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java index 69630a8ccee..599d47f8da0 100644 --- a/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java +++ b/products/feature-flagging/feature-flagging-api/src/test/java/datadog/trace/api/openfeature/FlagEvalEVPHookTest.java @@ -20,7 +20,10 @@ import dev.openfeature.sdk.ImmutableMetadata; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.Reason; +import dev.openfeature.sdk.Value; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -83,7 +86,7 @@ private static HookContext hookCtxWithTargetingKey( .build(); } - // ---- test: hook calls writer.enqueue once with flagKey, variant, reason, allocationKey ---- + // ---- test: hook calls writer.enqueue once with flagKey, variant, allocationKey ---- @Test void finallyAfterEnqueuesEventWithAllBasicFields() { @@ -104,7 +107,6 @@ void finallyAfterEnqueuesEventWithAllBasicFields() { final FlagEvalEvent e = captured.get(); assertEquals("my-flag", e.flagKey); assertEquals("on", e.variant, "variant must be the OpenFeature variant key"); - assertEquals("targeting_match", e.reason, "reason must be lowercased"); assertEquals("alloc-1", e.allocationKey); } @@ -327,4 +329,38 @@ void targetingKeyExtractedFromContext() { captured.get().targetingKey, "targetingKey must be extracted from the evaluation context"); } + + @Test + void contextAttributesAreFlattenedAndConvertedBeforeEnqueue() { + final AtomicReference captured = new AtomicReference<>(); + final FlagEvalEVPHook hook = hookWithWriter(capturingWriter(captured)); + + final Map profile = new HashMap<>(); + profile.put("tier", "gold"); + final Map attributes = new HashMap<>(); + attributes.put("score", 42); + attributes.put("profile", profile); + final MutableContext context = + new MutableContext(Value.objectToValue(attributes).asStructure().asMap()); + context.setTargetingKey("user-42"); + + final HookContext hookCtx = + HookContext.builder() + .flagKey("ctx-flag") + .type(FlagValueType.STRING) + .defaultValue("default") + .ctx(context) + .build(); + final FlagEvaluationDetails det = + details("ctx-flag", "v", "v", Reason.TARGETING_MATCH.name(), null); + + hook.finallyAfter(hookCtx, det, Collections.emptyMap()); + + assertNotNull(captured.get()); + assertEquals(42, captured.get().attrs.get("score")); + assertEquals("gold", captured.get().attrs.get("profile.tier")); + assertTrue( + captured.get().attrs.values().stream().noneMatch(Value.class::isInstance), + "context attrs must contain converted scalar values, not OpenFeature Value wrappers"); + } } diff --git a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java index 78d503758b2..83a7c14148b 100644 --- a/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java +++ b/products/feature-flagging/feature-flagging-bootstrap/src/main/java/datadog/trace/api/featureflag/flagevaluation/FlagEvalEvent.java @@ -25,11 +25,6 @@ public final class FlagEvalEvent { /** The allocation key from flag metadata ("allocationKey"). May be null. */ public final String allocationKey; - /** - * The evaluation reason (lowercased, e.g. "targeting_match", "default", "error"). May be null. - */ - public final String reason; - /** The targeting key from the evaluation context. May be null. */ public final String targetingKey; @@ -56,18 +51,16 @@ public FlagEvalEvent( final String flagKey, final String variant, final String allocationKey, - final String reason, final String targetingKey, final long evalTimeMs, final Map attrs) { - this(flagKey, variant, allocationKey, reason, targetingKey, null, evalTimeMs, attrs); + this(flagKey, variant, allocationKey, targetingKey, null, evalTimeMs, attrs); } public FlagEvalEvent( final String flagKey, final String variant, final String allocationKey, - final String reason, final String targetingKey, final String errorMessage, final long evalTimeMs, @@ -75,7 +68,6 @@ public FlagEvalEvent( this.flagKey = flagKey; this.variant = variant; this.allocationKey = allocationKey; - this.reason = reason; this.targetingKey = targetingKey; this.errorMessage = errorMessage; this.evalTimeMs = evalTimeMs; diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 3f0cded6616..68a1ec654c5 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -28,6 +28,9 @@ dependencies { testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) + testImplementation("com.github.java-json-tools:json-schema-validator:2.2.10") { + exclude(group = "com.google.guava", module = "guava") + } testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } diff --git a/products/feature-flagging/feature-flagging-lib/gradle.lockfile b/products/feature-flagging/feature-flagging-lib/gradle.lockfile index 1e211e3823a..c01255519b4 100644 --- a/products/feature-flagging/feature-flagging-lib/gradle.lockfile +++ b/products/feature-flagging/feature-flagging-lib/gradle.lockfile @@ -12,14 +12,28 @@ com.datadoghq:dd-instrument-java:0.0.4=testCompileClasspath,testRuntimeClasspath com.datadoghq:dd-javac-plugin-client:0.2.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.datadoghq:java-dogstatsd-client:4.4.5=runtimeClasspath,testRuntimeClasspath com.datadoghq:sketches-java:0.8.3=runtimeClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.2.3=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.2.3=testCompileClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.2.3=testCompileClasspath,testRuntimeClasspath +com.github.fge:btf:1.2=testCompileClasspath,testRuntimeClasspath +com.github.fge:msg-simple:1.1=testCompileClasspath,testRuntimeClasspath +com.github.fge:uri-template:0.9=testCompileClasspath,testRuntimeClasspath +com.github.java-json-tools:jackson-coreutils:1.9=testCompileClasspath,testRuntimeClasspath +com.github.java-json-tools:json-schema-core:1.2.10=testCompileClasspath,testRuntimeClasspath +com.github.java-json-tools:json-schema-validator:2.2.10=testCompileClasspath,testRuntimeClasspath com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.14=runtimeClasspath,testRuntimeClasspath +com.github.jnr:jffi:1.3.14=runtimeClasspath +com.github.jnr:jffi:1.3.15=testRuntimeClasspath com.github.jnr:jnr-a64asm:1.0.0=runtimeClasspath,testRuntimeClasspath com.github.jnr:jnr-constants:0.10.4=runtimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.19=runtimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.18=runtimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.21=runtimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.24=runtimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.19=runtimeClasspath +com.github.jnr:jnr-enxio:0.32.20=testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.18=runtimeClasspath +com.github.jnr:jnr-ffi:2.2.19=testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.21=runtimeClasspath +com.github.jnr:jnr-posix:3.1.22=testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.24=runtimeClasspath +com.github.jnr:jnr-unixsocket:0.38.25=testRuntimeClasspath com.github.jnr:jnr-x86asm:1.0.2=runtimeClasspath,testRuntimeClasspath com.github.spotbugs:spotbugs-annotations:4.9.8=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath com.github.spotbugs:spotbugs:4.9.8=spotbugs @@ -27,8 +41,10 @@ com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs com.google.code.findbugs:jsr305:3.0.2=compileClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath com.google.code.gson:gson:2.13.2=spotbugs com.google.errorprone:error_prone_annotations:2.41.0=spotbugs -com.google.guava:guava:20.0=testCompileClasspath,testRuntimeClasspath +com.google.guava:guava:16.0.1=testCompileClasspath +com.google.guava:guava:20.0=testRuntimeClasspath com.google.re2j:re2j:1.7=testRuntimeClasspath +com.googlecode.libphonenumber:libphonenumber:8.0.0=testCompileClasspath,testRuntimeClasspath com.squareup.moshi:moshi:1.11.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath @@ -40,13 +56,17 @@ commons-io:commons-io:2.20.0=spotbugs de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath io.sqreen:libsqreen:17.3.0=testRuntimeClasspath +javax.activation:activation:1.1=testCompileClasspath,testRuntimeClasspath +javax.mail:mailapi:1.4.3=testCompileClasspath,testRuntimeClasspath javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath jaxen:jaxen:2.0.0=spotbugs +joda-time:joda-time:2.9.7=testCompileClasspath,testRuntimeClasspath junit:junit:4.13.2=testRuntimeClasspath net.bytebuddy:byte-buddy-agent:1.18.10=testCompileClasspath,testRuntimeClasspath net.bytebuddy:byte-buddy:1.18.10=testCompileClasspath,testRuntimeClasspath net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath net.java.dev.jna:jna:5.8.0=testRuntimeClasspath +net.sf.jopt-simple:jopt-simple:5.0.3=testCompileClasspath,testRuntimeClasspath net.sf.saxon:Saxon-HE:12.9=spotbugs org.apache.ant:ant-antlr:1.10.14=codenarc org.apache.ant:ant-junit:1.10.14=codenarc @@ -90,6 +110,7 @@ org.junit:junit-bom:5.14.0=spotbugs org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath org.mockito:mockito-core:4.4.0=testCompileClasspath,testRuntimeClasspath org.mockito:mockito-junit-jupiter:4.4.0=testCompileClasspath,testRuntimeClasspath +org.mozilla:rhino:1.7.7.1=testCompileClasspath,testRuntimeClasspath org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath org.ow2.asm:asm-analysis:9.7.1=runtimeClasspath,testRuntimeClasspath diff --git a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java index 46d6b4a627b..996e38bcd39 100644 --- a/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java +++ b/products/feature-flagging/feature-flagging-lib/src/jmh/java/com/datadog/featureflag/FlagEvaluationHotPathBenchmark.java @@ -81,14 +81,7 @@ public void setUp() { public void evalThreadCapture(final Blackhole blackhole) { final FlagEvalEvent ev = new FlagEvalEvent( - "checkout-flag", - "treatment", - "alloc-7", - "targeting_match", - "user-123", - null, - 1_700_000_000_000L, - attrs); + "checkout-flag", "treatment", "alloc-7", "user-123", null, 1_700_000_000_000L, attrs); writer.enqueue(ev); blackhole.consume(ev); } @@ -98,14 +91,7 @@ public void evalThreadCapture(final Blackhole blackhole) { public void workerAggregate(final Blackhole blackhole) { final FlagEvalEvent ev = new FlagEvalEvent( - "checkout-flag", - "treatment", - "alloc-7", - "targeting_match", - "user-123", - null, - 1_700_000_000_000L, - attrs); + "checkout-flag", "treatment", "alloc-7", "user-123", null, 1_700_000_000_000L, attrs); handler.aggregateEvent(ev); blackhole.consume(handler.fullTier.size()); } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index b942661c675..a847394ba5f 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -41,8 +41,10 @@ * *
      *
    • Two-tier aggregation: full → degraded → drop-counted (no ultra-degraded). - *
    • Full key: (flagKey, variant, allocationKey, reason, targetingKey, canonical-context-key). - *
    • Degraded key: (flagKey, variant, allocationKey, reason) — no targetingKey/context. + *
    • Full key: (flagKey, variant, allocationKey, runtimeDefault, errorMessage, targetingKey, + * canonical-context-key). + *
    • Degraded key: (flagKey, variant, allocationKey, runtimeDefault, errorMessage) — no + * targetingKey/context. *
    • Canonical context key: sorted entries, type-tagged length-delimited encoding — NOT a hash * (collision-safe, comparable string identity). *
    • Context pruning: deterministic (sort before cut), ≤256 fields, string values ≤256 chars; @@ -172,9 +174,18 @@ public void enqueue(final FlagEvalEvent event) { if (event == null) { return; } - // Non-blocking offer. On overflow (bounded queue full) drop the event and count it so the loss + // Non-blocking offer. Bound the context snapshot before the queue, then count overflow so loss // is observable rather than silent; the count is surfaced on the next flush. - if (!queue.offer(event)) { + final FlagEvalEvent boundedEvent = + new FlagEvalEvent( + event.flagKey, + event.variant, + event.allocationKey, + event.targetingKey, + event.errorMessage, + event.evalTimeMs, + pruneContext(event.attrs)); + if (!queue.offer(boundedEvent)) { droppedQueueOverflow.incrementAndGet(); } } @@ -184,6 +195,11 @@ long droppedQueueOverflow() { return droppedQueueOverflow.get(); } + /** Test seam: returns one queued event without starting the worker. */ + FlagEvalEvent pollQueuedEventForTest() { + return queue.poll(); + } + // ---- Deterministic context pruning ---- /** @@ -284,7 +300,6 @@ static class EvalBucket { String flagKey; String variant; String allocationKey; - String reason; String targetingKey; String errorMessage; Map prunedAttrs; // pruned context for serialization (full tier only) @@ -293,7 +308,6 @@ static class EvalBucket { final String flagKey, final String variant, final String allocationKey, - final String reason, final String targetingKey, final String errorMessage, final long evalTimeMs, @@ -302,7 +316,6 @@ static class EvalBucket { this.flagKey = flagKey; this.variant = variant; this.allocationKey = allocationKey; - this.reason = reason; this.targetingKey = targetingKey; this.errorMessage = errorMessage; this.firstEvalMs = evalTimeMs; @@ -470,7 +483,6 @@ void aggregateEvent(final FlagEvalEvent event) { event.flagKey, event.variant, event.allocationKey, - event.reason, event.targetingKey, event.errorMessage, event.evalTimeMs, @@ -495,7 +507,6 @@ void aggregateEvent(final FlagEvalEvent event) { event.flagKey, event.variant, event.allocationKey, - event.reason, null, // degraded omits targetingKey event.errorMessage, event.evalTimeMs, @@ -516,7 +527,9 @@ private static String buildFullKey(final FlagEvalEvent event, final String ctxKe + '\0' + nullToEmpty(event.allocationKey) + '\0' - + nullToEmpty(event.reason) + + (event.variant == null ? "1" : "0") + + '\0' + + nullToEmpty(event.errorMessage) + '\0' + nullToEmpty(event.targetingKey) + '\0' @@ -530,7 +543,9 @@ private static String buildDegradedKey(final FlagEvalEvent event) { + '\0' + nullToEmpty(event.allocationKey) + '\0' - + nullToEmpty(event.reason); + + (event.variant == null ? "1" : "0") + + '\0' + + nullToEmpty(event.errorMessage); } private static String nullToEmpty(final String s) { @@ -653,7 +668,7 @@ AggregatedState drainAndAggregate() { void simulateFullTierAtCap() { for (int i = globalFullCount; i < GLOBAL_CAP; i++) { final String key = "synthetic-full-" + i; - fullTier.put(key, new EvalBucket(key, "on", "alloc", "split", null, null, 1L, false, null)); + fullTier.put(key, new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); globalFullCount++; perFlagCount.merge(key, 1, Integer::sum); } @@ -665,8 +680,7 @@ void simulateFullTierAtCap() { void simulateDegradedTierAtCap() { for (int i = degradedTier.size(); i < DEGRADED_CAP; i++) { final String key = "synthetic-dg-" + i; - degradedTier.put( - key, new EvalBucket(key, "on", "alloc", "split", null, null, 1L, false, null)); + degradedTier.put(key, new EvalBucket(key, "on", "alloc", null, null, 1L, false, null)); } } } @@ -738,7 +752,7 @@ static class FlagEvaluationEvent { static FlagEvaluationEvent fromBucket(final EvalBucket bucket, final boolean isFullTier) { return new FlagEvaluationEvent( - System.currentTimeMillis(), + bucket.lastEvalMs, bucket.flagKey, bucket.firstEvalMs, bucket.lastEvalMs, diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index fe22502bebf..95a8ac85c58 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -11,20 +11,27 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.fge.jackson.JsonLoader; +import com.github.fge.jsonschema.core.exceptions.ProcessingException; +import com.github.fge.jsonschema.core.report.ProcessingReport; +import com.github.fge.jsonschema.main.JsonSchema; +import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.trace.api.featureflag.flagevaluation.FlagEvalEvent; +import java.io.IOException; import java.lang.reflect.Type; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import okhttp3.RequestBody; import okio.Buffer; import org.junit.jupiter.api.Test; @@ -37,6 +44,7 @@ class FlagEvaluationWriterImplTest { // Moshi adapter for parsing the emitted JSON back into a Map for structural assertions. private static final JsonAdapter> JSON_MAP; + private static final long REALISTIC_EVAL_MS = 1_760_000_000_000L; static { final Moshi moshi = new Moshi.Builder().build(); @@ -50,23 +58,25 @@ private static FlagEvalEvent event( String flagKey, String variant, String allocationKey, - String reason, String targetingKey, long evalTimeMs, Map attrs) { - return new FlagEvalEvent( - flagKey, variant, allocationKey, reason, targetingKey, evalTimeMs, attrs); + return new FlagEvalEvent(flagKey, variant, allocationKey, targetingKey, evalTimeMs, attrs); } - private static FlagEvalEvent errorEvent( - String flagKey, String reason, String errorMessage, long evalTimeMs) { + private static FlagEvalEvent errorEvent(String flagKey, String errorMessage, long evalTimeMs) { return new FlagEvalEvent( - flagKey, null, null, reason, null, errorMessage, evalTimeMs, Collections.emptyMap()); + flagKey, null, null, null, errorMessage, evalTimeMs, Collections.emptyMap()); } private static FlagEvalEvent simpleEvent(String flagKey, String variant) { - return event( - flagKey, variant, "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap()); + return event(flagKey, variant, "alloc1", "user-1", 1000L, Collections.emptyMap()); + } + + private static String repeat(char c, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); } private TestWriterSetup buildTestWriter(BackendApi mockEvp) { @@ -93,9 +103,18 @@ static class TestWriterSetup { } } - /** Captures the JSON body posted by a flush(), parsed into a Map. */ - @SuppressWarnings("unchecked") - private Map flushAndCaptureJson(TestWriterSetup setup) throws Exception { + static class CapturedJson { + final String raw; + final Map parsed; + + CapturedJson(final String raw, final Map parsed) { + this.raw = raw; + this.parsed = parsed; + } + } + + /** Captures the JSON body posted by a flush(), preserving raw JSON for schema validation. */ + private CapturedJson flushAndCapture(TestWriterSetup setup) throws Exception { final RequestBody[] captured = {null}; when(setup.mockEvp.post(eq("flagevaluations"), any(RequestBody.class), any(), any(), eq(false))) .thenAnswer( @@ -108,7 +127,13 @@ private Map flushAndCaptureJson(TestWriterSetup setup) throws Ex assertNotNull(captured[0], "RequestBody must have been posted"); Buffer buf = new Buffer(); captured[0].writeTo(buf); - return JSON_MAP.fromJson(buf.readUtf8()); + String raw = buf.readUtf8(); + return new CapturedJson(raw, JSON_MAP.fromJson(raw)); + } + + /** Captures the JSON body posted by a flush(), parsed into a Map. */ + private Map flushAndCaptureJson(TestWriterSetup setup) throws Exception { + return flushAndCapture(setup).parsed; } // ---- test: two identical events -> one full-tier bucket, count 2, first <= last ---- @@ -118,12 +143,8 @@ void identicalEventsAggregateIntoOneBucketWithCount2() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - setup.handler.add( - event( - "flag-a", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap())); - setup.handler.add( - event( - "flag-a", "on", "alloc1", "targeting_match", "user-1", 2000L, Collections.emptyMap())); + setup.handler.add(event("flag-a", "on", "alloc1", "user-1", 1000L, Collections.emptyMap())); + setup.handler.add(event("flag-a", "on", "alloc1", "user-1", 2000L, Collections.emptyMap())); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals( @@ -146,9 +167,7 @@ void variantIsTheVariantKeyNotTheEvaluatedValue() throws Exception { // intentionally NOT part of the event — the hook is responsible for sourcing variant from // details.getVariant(). Here we assert the emitted variant.key == the variant key, never a // value. - setup.handler.add( - event( - "flag-v", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap())); + setup.handler.add(event("flag-v", "on", "alloc1", "user-1", 1000L, Collections.emptyMap())); Map json = flushAndCaptureJson(setup); Map ev = firstEvent(json); @@ -169,10 +188,8 @@ void differentValueTypesProduceDifferentBuckets() { Map attrsStr = new HashMap<>(); attrsStr.put("score", "1"); - setup.handler.add( - event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsInt)); - setup.handler.add( - event("flag-b", "on", "alloc1", "targeting_match", "user-1", 1000L, attrsStr)); + setup.handler.add(event("flag-b", "on", "alloc1", "user-1", 1000L, attrsInt)); + setup.handler.add(event("flag-b", "on", "alloc1", "user-1", 1000L, attrsStr)); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals( @@ -263,6 +280,31 @@ void enqueueDoesNotAggregateOnTheCallingThread() { assertEquals(0, writer.droppedQueueOverflow(), "no overflow expected for 2 events in cap 16"); } + @Test + void enqueueQueuesPrunedContextSnapshotBeforeBuffering() { + BackendApi mockEvp = mock(BackendApi.class); + BackendApiFactory factory = mock(BackendApiFactory.class); + when(factory.createBackendApi(any())).thenReturn(mockEvp); + FlagEvaluationWriterImpl writer = + new FlagEvaluationWriterImpl( + 16, Long.MAX_VALUE, java.util.concurrent.TimeUnit.NANOSECONDS, factory, cfg()); + + Map rawAttrs = new HashMap<>(); + rawAttrs.put("oversized", repeat('x', 300)); + for (int i = 0; i < 300; i++) { + rawAttrs.put(String.format("k%03d", i), "v" + i); + } + + writer.enqueue(event("bounded-flag", "on", "alloc1", "user-1", 1000L, rawAttrs)); + + FlagEvalEvent queued = writer.pollQueuedEventForTest(); + assertNotNull(queued, "enqueue must offer one event to the bounded queue"); + assertEquals(256, queued.attrs.size(), "queued attrs must be pruned before buffering"); + assertFalse(queued.attrs.containsKey("oversized"), "oversized string must not be queued"); + assertTrue(queued.attrs.containsKey("k000"), "deterministic first sorted key survives"); + assertFalse(queued.attrs.containsKey("k299"), "fields beyond the 256-field cap are not queued"); + } + // ---- close() drains the queue and final-flushes before the worker exits ---- @Test @@ -306,6 +348,25 @@ void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { eventForFlag(json, "shutdown-flag"), "the queued event must be in the final flush"); } + @Test + void emittedTimestampUsesLastEvaluationTimeNotFlushTime() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.add( + event("ts-flag", "on", "alloc1", "user-1", REALISTIC_EVAL_MS, Collections.emptyMap())); + setup.handler.add( + event("ts-flag", "on", "alloc1", "user-1", REALISTIC_EVAL_MS + 10, Collections.emptyMap())); + + Map json = flushAndCaptureJson(setup); + Map ev = firstEvent(json); + assertEquals( + (double) (REALISTIC_EVAL_MS + 10), + ((Number) ev.get("timestamp")).doubleValue(), + 0.0, + "event timestamp must use the aggregate evaluation time, not flush time"); + } + // ---- absent variant -> runtimeDefaultUsed=true ---- @Test @@ -313,8 +374,7 @@ void absentVariantSetsRuntimeDefaultUsed() { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - setup.handler.add( - event("flag-c", null, "alloc1", "default", "user-1", 1000L, Collections.emptyMap())); + setup.handler.add(event("flag-c", null, "alloc1", "user-1", 1000L, Collections.emptyMap())); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); @@ -333,7 +393,7 @@ void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { Map attrs = new HashMap<>(); attrs.put("region", "us-east-1"); - setup.handler.add(event("dg-flag", "on", "alloc1", "targeting_match", "user-99", 1000L, attrs)); + setup.handler.add(event("dg-flag", "on", "alloc1", "user-99", 1000L, attrs)); Map json = flushAndCaptureJson(setup); Map dg = eventForFlag(json, "dg-flag"); @@ -353,8 +413,7 @@ void contextExceeding256FieldsIsPrunedToStoredPrunedAttrs() { for (int i = 0; i < 300; i++) { hugeAttrs.put("key" + i, "v" + i); } - setup.handler.add( - event("flag-d", "on", "alloc1", "targeting_match", "user-1", 1000L, hugeAttrs)); + setup.handler.add(event("flag-d", "on", "alloc1", "user-1", 1000L, hugeAttrs)); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); @@ -399,7 +458,7 @@ void contextValueExceeding256CharsIsSkippedFromPrunedAttrs() { Map attrs = new HashMap<>(); attrs.put("long-val", longVal); // >256 chars attrs.put("short-val", "ok"); - setup.handler.add(event("flag-e", "on", "alloc1", "targeting_match", "user-1", 1000L, attrs)); + setup.handler.add(event("flag-e", "on", "alloc1", "user-1", 1000L, attrs)); FlagEvaluationWriterImpl.AggregatedState state = setup.handler.drainAndAggregate(); assertEquals(1, state.fullTier.size()); @@ -417,9 +476,7 @@ void flushPostsToFlagevaluationsEndpoint() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - setup.handler.add( - event( - "flag-f", "on", "alloc1", "targeting_match", "user-1", 1000L, Collections.emptyMap())); + setup.handler.add(event("flag-f", "on", "alloc1", "user-1", 1000L, Collections.emptyMap())); setup.handler.drainAndAggregate(); setup.handler.flush(mockEvp); @@ -435,12 +492,13 @@ void fullTierPayloadValidatesAgainstWorkerSchema() throws Exception { Map attrs = new HashMap<>(); attrs.put("region", "us-east-1"); - setup.handler.add(event("my-flag", "on", "alloc-x", "targeting_match", "user-1", 5000L, attrs)); + setup.handler.add(event("my-flag", "on", "alloc-x", "user-1", REALISTIC_EVAL_MS, attrs)); - Map json = flushAndCaptureJson(setup); - FlagEvalSchema.assertValidBatch(json); + CapturedJson captured = flushAndCapture(setup); + FlagEvalSchema.assertValidJson(captured.raw); // And spot-check the object-vs-string shape the prior code got wrong: + Map json = captured.parsed; Map ev = firstEvent(json); assertObjectWithKey(ev.get("variant"), "on", "variant must serialize as {\"key\":...}"); assertObjectWithKey( @@ -461,11 +519,12 @@ void errorPayloadSerializesErrorObjectAndValidatesSchema() throws Exception { BackendApi mockEvp = mock(BackendApi.class); TestWriterSetup setup = buildTestWriter(mockEvp); - setup.handler.add(errorEvent("err-flag", "error", "type mismatch", 5000L)); + setup.handler.add(errorEvent("err-flag", "type mismatch", REALISTIC_EVAL_MS)); - Map json = flushAndCaptureJson(setup); - FlagEvalSchema.assertValidBatch(json); + CapturedJson captured = flushAndCapture(setup); + FlagEvalSchema.assertValidJson(captured.raw); + Map json = captured.parsed; Map ev = firstEvent(json); Map error = (Map) ev.get("error"); assertNotNull(error, "error must be present on the error path"); @@ -475,6 +534,30 @@ void errorPayloadSerializesErrorObjectAndValidatesSchema() throws Exception { assertEquals(Boolean.TRUE, ev.get("runtime_default_used")); } + @Test + void workerSchemaRejectsTopLevelReason() throws Exception { + BackendApi mockEvp = mock(BackendApi.class); + TestWriterSetup setup = buildTestWriter(mockEvp); + + setup.handler.add( + event("reason-schema-flag", "on", "alloc-x", "user-1", REALISTIC_EVAL_MS, new HashMap<>())); + + CapturedJson captured = flushAndCapture(setup); + JsonNode json = JsonLoader.fromString(captured.raw).deepCopy(); + ((ObjectNode) json.get("flagEvaluations").get(0)).put("reason", "targeting_match"); + + FlagEvalSchema.assertInvalidJson(json.toString()); + } + + @Test + void flagEvalEventDoesNotCarryReason() { + boolean hasReasonField = + Arrays.stream(FlagEvalEvent.class.getDeclaredFields()) + .anyMatch(field -> field.getName().equals("reason")); + + assertFalse(hasReasonField, "EVP event snapshots must not carry hidden reason cardinality"); + } + private static void assertObjectWithKey(Object o, String expectedKey, String msg) { assertTrue(o instanceof Map, msg + " (must be a JSON object, not a bare string)"); assertEquals(expectedKey, ((Map) o).get("key"), msg); @@ -501,114 +584,55 @@ private static Map eventForFlag(Map batch, Strin return null; } - /** - * Minimal structural validator that enforces the hard constraints of the real flageval-worker - * {@code batchedflagevaluations.json} schema (vendored at {@code - * src/test/resources/flagevaluation/batchedflagevaluations.json}): required fields, integer - * types, {@code {"key":...}} / {@code {"message":...}} object shapes, and {@code - * additionalProperties:false} at the event and key-object levels. - */ static final class FlagEvalSchema { - private static final Set ALLOWED_EVENT_KEYS = - new HashSet<>( - Arrays.asList( - "timestamp", - "flag", - "first_evaluation", - "last_evaluation", - "evaluation_count", - "runtime_default_used", - "targeting_key", - "context", - "variant", - "allocation", - "targeting_rule", - "error")); - private static final Set REQUIRED_EVENT_KEYS = - new HashSet<>( - Arrays.asList( - "timestamp", "flag", "first_evaluation", "last_evaluation", "evaluation_count")); + private static final JsonSchema WORKER_SCHEMA = loadSchema(); - @SuppressWarnings("unchecked") - static void assertValidBatch(Map batch) { - assertTrue( - batch.containsKey("flagEvaluations"), - "batch must contain required flagEvaluations array"); - Object arr = batch.get("flagEvaluations"); - assertTrue(arr instanceof List, "flagEvaluations must be an array"); - for (Object o : (List) arr) { - assertTrue(o instanceof Map, "each flag evaluation must be an object"); - assertValidEvent((Map) o); - } + static void assertValidJson(String json) { + ProcessingReport report = validate(json); + assertTrue(report.isSuccess(), "worker schema errors:\n" + report); } - static void assertValidEvent(Map ev) { - // additionalProperties:false at the event level - for (String k : ev.keySet()) { - assertTrue(ALLOWED_EVENT_KEYS.contains(k), "unexpected event property not in schema: " + k); - } - // required fields present - for (String r : REQUIRED_EVENT_KEYS) { - assertTrue(ev.containsKey(r), "required field missing: " + r); - } - // integer-typed required fields (Moshi parses JSON numbers as Double; assert integral) - assertIntegral(ev.get("timestamp"), "timestamp"); - assertIntegral(ev.get("first_evaluation"), "first_evaluation"); - assertIntegral(ev.get("last_evaluation"), "last_evaluation"); - assertIntegral(ev.get("evaluation_count"), "evaluation_count"); - // key-objects - assertKeyObject(ev.get("flag"), "flag", true); - assertKeyObject(ev.get("variant"), "variant", false); - assertKeyObject(ev.get("allocation"), "allocation", false); - assertKeyObject(ev.get("targeting_rule"), "targeting_rule", false); - // error object {"message": string} - Object error = ev.get("error"); - if (error != null) { - assertTrue(error instanceof Map, "error must be an object"); - Map e = (Map) error; - assertTrue(e.containsKey("message"), "error.message is required"); - assertEquals( - new HashSet<>(Collections.singletonList("message")), - e.keySet(), - "error object allows only {message}"); - } - // runtime_default_used must be boolean if present - Object rdu = ev.get("runtime_default_used"); - if (rdu != null) { - assertTrue(rdu instanceof Boolean, "runtime_default_used must be a boolean"); - } - // context: only {evaluation, dd} allowed - Object context = ev.get("context"); - if (context != null) { - assertTrue(context instanceof Map, "context must be an object"); - for (Object k : ((Map) context).keySet()) { - assertTrue( - "evaluation".equals(k) || "dd".equals(k), - "context allows only {evaluation, dd}, got: " + k); - } + static void assertInvalidJson(String json) { + ProcessingReport report = validate(json); + assertFalse(report.isSuccess(), "worker schema unexpectedly accepted payload"); + } + + private static ProcessingReport validate(String json) { + try { + JsonNode payload = JsonLoader.fromString(json); + return WORKER_SCHEMA.validate(payload); + } catch (IOException | ProcessingException e) { + throw new AssertionError("worker schema validation failed", e); } } - private static void assertIntegral(Object v, String field) { - assertNotNull(v, field + " must be present"); - assertTrue(v instanceof Number, field + " must be a number"); - double d = ((Number) v).doubleValue(); - assertEquals(Math.floor(d), d, 0.0, field + " must be an integer"); + private static JsonSchema loadSchema() { + try { + JsonNode schema = + JsonLoader.fromResource("/flagevaluation/batchedflagevaluations.json").deepCopy(); + stripEmptyRequired(schema); + return JsonSchemaFactory.byDefault().getJsonSchema(schema); + } catch (IOException | ProcessingException e) { + throw new ExceptionInInitializerError(e); + } } - private static void assertKeyObject(Object o, String field, boolean required) { - if (o == null) { - assertFalse(required, field + " is required"); - return; + private static void stripEmptyRequired(JsonNode node) { + if (node instanceof ObjectNode) { + ObjectNode object = (ObjectNode) node; + JsonNode required = object.get("required"); + if (required != null && required.isArray() && required.size() == 0) { + object.remove("required"); + } + Iterator> fields = object.fields(); + while (fields.hasNext()) { + stripEmptyRequired(fields.next().getValue()); + } + } else if (node != null && node.isArray()) { + for (JsonNode child : node) { + stripEmptyRequired(child); + } } - assertTrue(o instanceof Map, field + " must be a {\"key\":...} object, not a bare value"); - Map m = (Map) o; - assertTrue(m.containsKey("key"), field + ".key is required"); - assertEquals( - new HashSet<>(Collections.singletonList("key")), - m.keySet(), - field + " object allows only {key}"); - assertTrue(m.get("key") instanceof String, field + ".key must be a string"); } } }