Add buffer configuration to ClickHouse destination

.generator/schemas/v2/openapi.yaml

@@ -64398,6 +64398,8 @@ components:
           $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatch"
         batch_encoding:
           $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatchEncoding"
+        buffer:
+          $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         compression:
           $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompression"
         database:

cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/frozen.json

@@ -0,0 +1 @@
+"2026-06-24T16:45:05.037Z"

cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/recording.har

@@ -0,0 +1,67 @@
+{
+  "log": {
+    "_recordingName": "Observability Pipelines/Validate an observability pipeline with ClickHouse destination with all fields set returns \"OK\" response",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.5"
+    },
+    "entries": [
+      {
+        "_id": "b5389bbcbd4c392a8c9da4f2d324e4d8",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 1141,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "_fromType": "array",
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 582,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"auth\":{\"password_key\":\"CLICKHOUSE_PASSWORD\",\"strategy\":\"basic\",\"username_key\":\"CLICKHOUSE_USERNAME\"},\"batch\":{\"max_events\":1000,\"timeout_secs\":1},\"batch_encoding\":{\"allow_nullable_fields\":true,\"codec\":\"arrow_stream\"},\"buffer\":{\"max_events\":500,\"type\":\"memory\",\"when_full\":\"block\"},\"compression\":{\"algorithm\":\"gzip\",\"level\":6},\"database\":\"my_database\",\"date_time_best_effort\":true,\"endpoint_url_key\":\"CLICKHOUSE_ENDPOINT_URL\",\"format\":\"arrow_stream\",\"id\":\"clickhouse-destination\",\"inputs\":[\"my-processor-group\"],\"skip_unknown_fields\":true,\"table\":\"application_logs\",\"tls\":{\"ca_file\":\"/path/to/ca.crt\",\"crt_file\":\"/path/to/cert.crt\",\"key_file\":\"/path/to/key.key\",\"key_pass_key\":\"TLS_KEY_PASSPHRASE\"},\"type\":\"clickhouse\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"status:error\",\"type\":\"filter\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with ClickHouse Destination All Fields\"},\"type\":\"pipelines\"}}"
+          },
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
+        },
+        "response": {
+          "bodySize": 14,
+          "content": {
+            "mimeType": "application/vnd.api+json",
+            "size": 14,
+            "text": "{\"errors\":[]}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/vnd.api+json"
+            }
+          ],
+          "headersSize": 670,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2026-06-24T16:45:06.751Z",
+        "time": 142
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}

examples/v2/observability-pipelines/ValidatePipeline_253109967.ts

@@ -0,0 +1,93 @@
+/**
+ * Validate an observability pipeline with ClickHouse destination with all fields set returns "OK" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+const apiInstance = new v2.ObservabilityPipelinesApi(configuration);
+
+const params: v2.ObservabilityPipelinesApiValidatePipelineRequest = {
+  body: {
+    data: {
+      attributes: {
+        config: {
+          destinations: [
+            {
+              id: "clickhouse-destination",
+              inputs: ["my-processor-group"],
+              type: "clickhouse",
+              endpointUrlKey: "CLICKHOUSE_ENDPOINT_URL",
+              database: "my_database",
+              table: "application_logs",
+              format: "arrow_stream",
+              skipUnknownFields: true,
+              dateTimeBestEffort: true,
+              compression: {
+                algorithm: "gzip",
+                level: 6,
+              },
+              auth: {
+                strategy: "basic",
+                usernameKey: "CLICKHOUSE_USERNAME",
+                passwordKey: "CLICKHOUSE_PASSWORD",
+              },
+              batch: {
+                maxEvents: 1000,
+                timeoutSecs: 1,
+              },
+              batchEncoding: {
+                codec: "arrow_stream",
+                allowNullableFields: true,
+              },
+              tls: {
+                crtFile: "/path/to/cert.crt",
+                caFile: "/path/to/ca.crt",
+                keyFile: "/path/to/key.key",
+                keyPassKey: "TLS_KEY_PASSPHRASE",
+              },
+              buffer: {
+                type: "memory",
+                maxEvents: 500,
+                whenFull: "block",
+              },
+            },
+          ],
+          processorGroups: [
+            {
+              enabled: true,
+              id: "my-processor-group",
+              include: "service:my-service",
+              inputs: ["datadog-agent-source"],
+              processors: [
+                {
+                  enabled: true,
+                  id: "filter-processor",
+                  include: "status:error",
+                  type: "filter",
+                },
+              ],
+            },
+          ],
+          sources: [
+            {
+              id: "datadog-agent-source",
+              type: "datadog_agent",
+            },
+          ],
+        },
+        name: "Pipeline with ClickHouse Destination All Fields",
+      },
+      type: "pipelines",
+    },
+  },
+};
+
+apiInstance
+  .validatePipeline(params)
+  .then((data: v2.ValidationResponse) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

features/v2/observability_pipelines.feature

@@ -192,6 +192,14 @@ Feature: Observability Pipelines
     Then the response status is 200 OK
     And the response "errors" has length 0
 
+  @team:DataDog/observability-pipelines
+  Scenario: Validate an observability pipeline with ClickHouse destination with all fields set returns "OK" response
+    Given new "ValidatePipeline" request
+    And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "clickhouse-destination", "inputs": ["my-processor-group"], "type": "clickhouse", "endpoint_url_key": "CLICKHOUSE_ENDPOINT_URL", "database": "my_database", "table": "application_logs", "format": "arrow_stream", "skip_unknown_fields": true, "date_time_best_effort": true, "compression": {"algorithm": "gzip", "level": 6}, "auth": {"strategy": "basic", "username_key": "CLICKHOUSE_USERNAME", "password_key": "CLICKHOUSE_PASSWORD"}, "batch": {"max_events": 1000, "timeout_secs": 1}, "batch_encoding": {"codec": "arrow_stream", "allow_nullable_fields": true}, "tls": {"crt_file": "/path/to/cert.crt", "ca_file": "/path/to/ca.crt", "key_file": "/path/to/key.key", "key_pass_key": "TLS_KEY_PASSPHRASE"}, "buffer": {"type": "memory", "max_events": 500, "when_full": "block"}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with ClickHouse Destination All Fields"}, "type": "pipelines"}}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "errors" has length 0
+
   @team:DataDog/observability-pipelines
   Scenario: Validate an observability pipeline with HTTP server source valid_tokens returns "OK" response
     Given new "ValidatePipeline" request

packages/datadog-api-client-v2/models/ObservabilityPipelineClickhouseDestination.ts

@@ -3,6 +3,7 @@
  * This product includes software developed at Datadog (https://www.datadoghq.com/).
  * Copyright 2020-Present Datadog, Inc.
  */
+import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
 import { ObservabilityPipelineClickhouseDestinationAuth } from "./ObservabilityPipelineClickhouseDestinationAuth";
 import { ObservabilityPipelineClickhouseDestinationBatch } from "./ObservabilityPipelineClickhouseDestinationBatch";
 import { ObservabilityPipelineClickhouseDestinationBatchEncoding } from "./ObservabilityPipelineClickhouseDestinationBatchEncoding";
@@ -33,6 +34,10 @@ export class ObservabilityPipelineClickhouseDestination {
    * Required when `format` is `arrow_stream`. The `codec` field must be set to `arrow_stream`.
    */
   "batchEncoding"?: ObservabilityPipelineClickhouseDestinationBatchEncoding;
+  /**
+   * Configuration for buffer settings on destination components.
+   */
+  "buffer"?: ObservabilityPipelineBufferOptions;
   /**
    * Compression setting for outbound HTTP requests to ClickHouse.
    * Can be specified as a shorthand string (`"gzip"` or `"none"`) or as an object
@@ -114,6 +119,10 @@ export class ObservabilityPipelineClickhouseDestination {
       baseName: "batch_encoding",
       type: "ObservabilityPipelineClickhouseDestinationBatchEncoding",
     },
+    buffer: {
+      baseName: "buffer",
+      type: "ObservabilityPipelineBufferOptions",
+    },
     compression: {
       baseName: "compression",
       type: "ObservabilityPipelineClickhouseDestinationCompression",