diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 4dd4eb9c3140..46dc7f61d800 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -64398,6 +64398,8 @@ components: $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatch" batch_encoding: $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatchEncoding" + buffer: + $ref: "#/components/schemas/ObservabilityPipelineBufferOptions" compression: $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompression" database: diff --git a/cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/frozen.json b/cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/frozen.json new file mode 100644 index 000000000000..1b001bcdc7b4 --- /dev/null +++ b/cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/frozen.json @@ -0,0 +1 @@ +"2026-06-24T16:45:05.037Z" diff --git a/cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/recording.har b/cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/recording.har new file mode 100644 index 000000000000..8b7918952eb9 --- /dev/null +++ b/cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-ClickHouse-destination-with-all-fields-set-return_1413101805/recording.har @@ -0,0 +1,67 @@ +{ + "log": { + "_recordingName": "Observability Pipelines/Validate an observability pipeline with ClickHouse destination with all fields set returns \"OK\" response", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.5" + }, + "entries": [ + { + "_id": "b5389bbcbd4c392a8c9da4f2d324e4d8", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 1141, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + } + ], + "headersSize": 582, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"auth\":{\"password_key\":\"CLICKHOUSE_PASSWORD\",\"strategy\":\"basic\",\"username_key\":\"CLICKHOUSE_USERNAME\"},\"batch\":{\"max_events\":1000,\"timeout_secs\":1},\"batch_encoding\":{\"allow_nullable_fields\":true,\"codec\":\"arrow_stream\"},\"buffer\":{\"max_events\":500,\"type\":\"memory\",\"when_full\":\"block\"},\"compression\":{\"algorithm\":\"gzip\",\"level\":6},\"database\":\"my_database\",\"date_time_best_effort\":true,\"endpoint_url_key\":\"CLICKHOUSE_ENDPOINT_URL\",\"format\":\"arrow_stream\",\"id\":\"clickhouse-destination\",\"inputs\":[\"my-processor-group\"],\"skip_unknown_fields\":true,\"table\":\"application_logs\",\"tls\":{\"ca_file\":\"/path/to/ca.crt\",\"crt_file\":\"/path/to/cert.crt\",\"key_file\":\"/path/to/key.key\",\"key_pass_key\":\"TLS_KEY_PASSPHRASE\"},\"type\":\"clickhouse\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"status:error\",\"type\":\"filter\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with ClickHouse Destination All Fields\"},\"type\":\"pipelines\"}}" + }, + "queryString": [], + "url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate" + }, + "response": { + "bodySize": 14, + "content": { + "mimeType": "application/vnd.api+json", + "size": 14, + "text": "{\"errors\":[]}\n" + }, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/vnd.api+json" + } + ], + "headersSize": 670, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2026-06-24T16:45:06.751Z", + "time": 142 + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/examples/v2/observability-pipelines/ValidatePipeline_253109967.ts b/examples/v2/observability-pipelines/ValidatePipeline_253109967.ts new file mode 100644 index 000000000000..21052799eac4 --- /dev/null +++ b/examples/v2/observability-pipelines/ValidatePipeline_253109967.ts @@ -0,0 +1,93 @@ +/** + * Validate an observability pipeline with ClickHouse destination with all fields set returns "OK" response + */ + +import { client, v2 } from "@datadog/datadog-api-client"; + +const configuration = client.createConfiguration(); +const apiInstance = new v2.ObservabilityPipelinesApi(configuration); + +const params: v2.ObservabilityPipelinesApiValidatePipelineRequest = { + body: { + data: { + attributes: { + config: { + destinations: [ + { + id: "clickhouse-destination", + inputs: ["my-processor-group"], + type: "clickhouse", + endpointUrlKey: "CLICKHOUSE_ENDPOINT_URL", + database: "my_database", + table: "application_logs", + format: "arrow_stream", + skipUnknownFields: true, + dateTimeBestEffort: true, + compression: { + algorithm: "gzip", + level: 6, + }, + auth: { + strategy: "basic", + usernameKey: "CLICKHOUSE_USERNAME", + passwordKey: "CLICKHOUSE_PASSWORD", + }, + batch: { + maxEvents: 1000, + timeoutSecs: 1, + }, + batchEncoding: { + codec: "arrow_stream", + allowNullableFields: true, + }, + tls: { + crtFile: "/path/to/cert.crt", + caFile: "/path/to/ca.crt", + keyFile: "/path/to/key.key", + keyPassKey: "TLS_KEY_PASSPHRASE", + }, + buffer: { + type: "memory", + maxEvents: 500, + whenFull: "block", + }, + }, + ], + processorGroups: [ + { + enabled: true, + id: "my-processor-group", + include: "service:my-service", + inputs: ["datadog-agent-source"], + processors: [ + { + enabled: true, + id: "filter-processor", + include: "status:error", + type: "filter", + }, + ], + }, + ], + sources: [ + { + id: "datadog-agent-source", + type: "datadog_agent", + }, + ], + }, + name: "Pipeline with ClickHouse Destination All Fields", + }, + type: "pipelines", + }, + }, +}; + +apiInstance + .validatePipeline(params) + .then((data: v2.ValidationResponse) => { + console.log( + "API called successfully. Returned data: " + JSON.stringify(data) + ); + }) + .catch((error: any) => console.error(error)); diff --git a/features/v2/observability_pipelines.feature b/features/v2/observability_pipelines.feature index e5ee4d0545d8..e42bbd7b0335 100644 --- a/features/v2/observability_pipelines.feature +++ b/features/v2/observability_pipelines.feature @@ -192,6 +192,14 @@ Feature: Observability Pipelines Then the response status is 200 OK And the response "errors" has length 0 + @team:DataDog/observability-pipelines + Scenario: Validate an observability pipeline with ClickHouse destination with all fields set returns "OK" response + Given new "ValidatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "clickhouse-destination", "inputs": ["my-processor-group"], "type": "clickhouse", "endpoint_url_key": "CLICKHOUSE_ENDPOINT_URL", "database": "my_database", "table": "application_logs", "format": "arrow_stream", "skip_unknown_fields": true, "date_time_best_effort": true, "compression": {"algorithm": "gzip", "level": 6}, "auth": {"strategy": "basic", "username_key": "CLICKHOUSE_USERNAME", "password_key": "CLICKHOUSE_PASSWORD"}, "batch": {"max_events": 1000, "timeout_secs": 1}, "batch_encoding": {"codec": "arrow_stream", "allow_nullable_fields": true}, "tls": {"crt_file": "/path/to/cert.crt", "ca_file": "/path/to/ca.crt", "key_file": "/path/to/key.key", "key_pass_key": "TLS_KEY_PASSPHRASE"}, "buffer": {"type": "memory", "max_events": 500, "when_full": "block"}}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with ClickHouse Destination All Fields"}, "type": "pipelines"}} + When the request is sent + Then the response status is 200 OK + And the response "errors" has length 0 + @team:DataDog/observability-pipelines Scenario: Validate an observability pipeline with HTTP server source valid_tokens returns "OK" response Given new "ValidatePipeline" request diff --git a/packages/datadog-api-client-v2/models/ObservabilityPipelineClickhouseDestination.ts b/packages/datadog-api-client-v2/models/ObservabilityPipelineClickhouseDestination.ts index b84e0810e8c0..64d3637bd33d 100644 --- a/packages/datadog-api-client-v2/models/ObservabilityPipelineClickhouseDestination.ts +++ b/packages/datadog-api-client-v2/models/ObservabilityPipelineClickhouseDestination.ts @@ -3,6 +3,7 @@ * This product includes software developed at Datadog (https://www.datadoghq.com/). * Copyright 2020-Present Datadog, Inc. */ +import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions"; import { ObservabilityPipelineClickhouseDestinationAuth } from "./ObservabilityPipelineClickhouseDestinationAuth"; import { ObservabilityPipelineClickhouseDestinationBatch } from "./ObservabilityPipelineClickhouseDestinationBatch"; import { ObservabilityPipelineClickhouseDestinationBatchEncoding } from "./ObservabilityPipelineClickhouseDestinationBatchEncoding"; @@ -33,6 +34,10 @@ export class ObservabilityPipelineClickhouseDestination { * Required when `format` is `arrow_stream`. The `codec` field must be set to `arrow_stream`. */ "batchEncoding"?: ObservabilityPipelineClickhouseDestinationBatchEncoding; + /** + * Configuration for buffer settings on destination components. + */ + "buffer"?: ObservabilityPipelineBufferOptions; /** * Compression setting for outbound HTTP requests to ClickHouse. * Can be specified as a shorthand string (`"gzip"` or `"none"`) or as an object @@ -114,6 +119,10 @@ export class ObservabilityPipelineClickhouseDestination { baseName: "batch_encoding", type: "ObservabilityPipelineClickhouseDestinationBatchEncoding", }, + buffer: { + baseName: "buffer", + type: "ObservabilityPipelineBufferOptions", + }, compression: { baseName: "compression", type: "ObservabilityPipelineClickhouseDestinationCompression",