diff --git a/content/docs/api/index.mdx b/content/docs/api/index.mdx index 5bac885..6b33f58 100644 --- a/content/docs/api/index.mdx +++ b/content/docs/api/index.mdx @@ -6,7 +6,9 @@ redirect_from: import { Card, Cards } from 'fumadocs-ui/components/card'; -This section provides detailed information about the available API endpoints for interacting with Parseable. +This section is the quickest way to understand how data reaches Parseable and which HTTP endpoint fits your integration. + +Most teams do not need every route listed here. In practice, you usually start with one ingestion path, send data into a dataset, and then use the query and dataset APIs only when you need direct control from your own tooling. ## Getting Started @@ -15,47 +17,88 @@ Before you begin, make sure you have: - The base URL of your Parseable server - Required authentication credentials (if authentication is enabled) -## API Endpoints +Most integrations fall into one of these paths: + +- Use `/api/v1/ingest` when your application already has JSON events and you want to route them with headers. +- Use `/v1/logs`, `/v1/traces`, or `/v1/metrics` when your collector or SDK is already sending native OTLP data. +- Use `/v1/prometheus/write` when Prometheus or another compatible system is forwarding metrics through Remote Write. +- Use `/api/v1/query` and `/api/v1/logstream/{stream_name}` when you need SQL queries or direct dataset management from your own code. + +If you are deciding where to start, the simplest rule is this: + +- JSON events from an app or script: `/api/v1/ingest` +- OpenTelemetry logs: `/v1/logs` +- OpenTelemetry traces: `/v1/traces` +- OpenTelemetry metrics: `/v1/metrics` +- Prometheus Remote Write metrics: `/v1/prometheus/write` + +## Ingestion APIs - Create a new log dataset with the specified name and configuration. + Send JSON events and route them with the `X-P-Stream` header. - Send logs to an existing log dataset. + Send OpenTelemetry logs to Parseable using the native `/v1/logs` endpoint. - - Delete an existing log dataset and all its associated data. + Send OpenTelemetry traces to Parseable using the native `/v1/traces` endpoint. - - Send log data to a specific log dataset using HTTP headers. + Send OpenTelemetry metrics to Parseable using the native `/v1/metrics` endpoint. + + Send snappy-compressed Prometheus Remote Write payloads to `/v1/prometheus/write`. + + +## Query and Dataset APIs + + + + Run SQL queries across datasets. The dataset is selected inside the `FROM` clause. + - Query data from log streams using SQL. + Send JSON events directly to an existing dataset by path. + + + Create a dataset before writing events directly to `/api/v1/logstream/{stream_name}`. + + + Delete a dataset and its associated data. ## Authentication -API reference endpoints require authentication. The OpenAPI reference uses HTTP Basic authentication by default: +These endpoints require authentication. The reference uses HTTP Basic authentication in the examples: ```bash Authorization: Basic @@ -63,22 +106,7 @@ Authorization: Basic If you use Parseable API keys, send the key in the `X-API-Key` header as described in the [API Keys](/docs/user-guide/api-keys) guide. -## Rate Limiting - -API requests are subject to rate limiting. The default rate limit is 100 requests per minute per IP address. - -## Error Handling - -All error responses follow a standard format: - -```json -{ - "error": { - "code": "error_code", - "message": "Human-readable error message" - } -} -``` +For ingestion endpoints, the other header to pay attention to is usually `X-P-Stream`. That header tells Parseable which dataset should receive the payload. OTLP routes also use `X-P-Log-Source` to identify whether the payload is logs, traces, or metrics. ## Need Help? diff --git a/content/docs/api/meta.json b/content/docs/api/meta.json index 1e7cc38..121a592 100644 --- a/content/docs/api/meta.json +++ b/content/docs/api/meta.json @@ -3,9 +3,13 @@ "pages": [ "---v1---", "v1/ingestLogsWithHeaders", + "v1/ingestOtlpLogs", + "v1/ingestOtlpTraces", + "v1/ingestOtlpMetrics", + "v1/prometheusRemoteWrite", "v1/queryLogStream", - "v1/deleteLogStream", "v1/sendLogsToStream", - "v1/createLogStream" + "v1/createLogStream", + "v1/deleteLogStream" ] } diff --git a/content/docs/api/v1/createLogStream.mdx b/content/docs/api/v1/createLogStream.mdx index 968ef98..ac37fee 100644 --- a/content/docs/api/v1/createLogStream.mdx +++ b/content/docs/api/v1/createLogStream.mdx @@ -1,5 +1,5 @@ --- -title: Create a log dataset +title: Create a dataset full: true _openapi: method: PUT @@ -8,20 +8,16 @@ _openapi: headings: [] contents: - content: > - This endpoint is used to create a new log dataset within Parseable. - - A **log dataset** is a group of similar logs. For example, you can - create a log dataset for a specific application's logs, another log - dataset for your database logs, and so on. You can create as many log - streams as needed to organize and manage logs efficiently. + Create a dataset in Parseable using the dataset name in the path. --- {/* Define variables for path parameters */} export const stream_name = "stream_name"; {/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */} -This endpoint is used to create a new log dataset within Parseable. -A **log dataset** is a group of similar logs. For example, you can create a log dataset for a specific application's logs, another log dataset for your database logs, and so on. You can create as many log streams as needed to organize and manage logs efficiently. +Create a dataset before you start sending data directly to a named stream endpoint. + +Use this route when you want the dataset name to be part of the URL itself, for example before writing JSON events to `/api/v1/logstream/{stream_name}`. Once the dataset exists, other ingestion calls can target it normally. - \ No newline at end of file + diff --git a/content/docs/api/v1/deleteLogStream.mdx b/content/docs/api/v1/deleteLogStream.mdx index f96bcaf..f742652 100644 --- a/content/docs/api/v1/deleteLogStream.mdx +++ b/content/docs/api/v1/deleteLogStream.mdx @@ -1,5 +1,5 @@ --- -title: Delete a log dataset +title: Delete a dataset full: true _openapi: method: DELETE @@ -8,9 +8,9 @@ _openapi: headings: [] contents: - content: > - This endpoint is used to delete an existing log dataset. + Delete an existing dataset. - When a log dataset is deleted, all associated data is permanently + When a dataset is deleted, all associated data is permanently removed. --- {/* Define variables for path parameters */} @@ -18,8 +18,9 @@ export const stream_name = "stream_name"; {/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */} -This endpoint is used to delete an existing log dataset. -When a log dataset is deleted, all associated data is permanently removed. +Delete an existing dataset when you no longer want to keep its data or accept new writes to it. +This is a destructive operation. Once a dataset is deleted, the associated data is removed as well, so this route is usually used from admin tooling or controlled cleanup workflows. - \ No newline at end of file + + diff --git a/content/docs/api/v1/ingestLogsWithHeaders.mdx b/content/docs/api/v1/ingestLogsWithHeaders.mdx index 8e345e7..654513b 100644 --- a/content/docs/api/v1/ingestLogsWithHeaders.mdx +++ b/content/docs/api/v1/ingestLogsWithHeaders.mdx @@ -1,5 +1,5 @@ --- -title: Ingest logs with headers +title: Ingest JSON events with headers full: true _openapi: method: POST @@ -8,94 +8,16 @@ _openapi: headings: [] contents: - content: > - **Log Ingestion API via Custom Headers** + Send JSON events and route them with request headers. - This approach allows you to send logs to Parseable using custom - headers to specify the target dataset. - - - Required headers: - - - **X-P-Stream**: The name of the dataset to ingest logs into - - - The API accepts logs in JSON format. You can send single log entries - or arrays of log entries. - - - Example: - - ```json - - { - "level": "info", - "message": "User logged in", - "timestamp": "2023-01-01T12:00:00Z", - "user_id": "user123" - } - - ``` - - - Or as an array: - - ```json - - [ - { - "level": "info", - "message": "User logged in", - "timestamp": "2023-01-01T12:00:00Z", - "user_id": "user123" - }, - { - "level": "error", - "message": "Failed to process request", - "timestamp": "2023-01-01T12:01:00Z", - "error_code": "ERR-1001" - } - ] - - ``` + Use `X-P-Stream` to specify the destination dataset. --- {/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */} -**Log Ingestion API via Custom Headers** -This approach allows you to send logs to Parseable using custom headers to specify the target dataset. - -Required headers: -- **X-P-Stream**: The name of the dataset to ingest logs into - -The API accepts logs in JSON format. You can send single log entries or arrays of log entries. - -Example: -```json -{ - "level": "info", - "message": "User logged in", - "timestamp": "2023-01-01T12:00:00Z", - "user_id": "user123" -} -``` +Use this route when your application already produces plain JSON and you want Parseable to store it without going through an OTLP pipeline. -Or as an array: -```json -[ - { - "level": "info", - "message": "User logged in", - "timestamp": "2023-01-01T12:00:00Z", - "user_id": "user123" - }, - { - "level": "error", - "message": "Failed to process request", - "timestamp": "2023-01-01T12:01:00Z", - "error_code": "ERR-1001" - } -] -``` +The important header here is `X-P-Stream`. It tells Parseable which dataset should receive the payload. This route works well for custom app logs, quick integrations, scripts, and services that already emit JSON records. - \ No newline at end of file + diff --git a/content/docs/api/v1/ingestOtlpLogs.mdx b/content/docs/api/v1/ingestOtlpLogs.mdx new file mode 100644 index 0000000..867d35c --- /dev/null +++ b/content/docs/api/v1/ingestOtlpLogs.mdx @@ -0,0 +1,20 @@ +--- +title: Ingest OTLP logs +full: true +_openapi: + method: POST + toc: [] + structuredData: + headings: [] + contents: + - content: > + Send OpenTelemetry logs to Parseable using the native `/v1/logs` + endpoint. Use `X-P-Stream` to choose the destination dataset and + `X-P-Log-Source: otel-logs` to identify the payload type. +--- + +Use this route when your collector or SDK is already exporting OpenTelemetry logs in OTLP format. + +`X-P-Stream` chooses the destination dataset, and `X-P-Log-Source: otel-logs` tells Parseable how to interpret the payload. This is the route to use when you want native OTLP logs rather than custom JSON ingestion. + + diff --git a/content/docs/api/v1/ingestOtlpMetrics.mdx b/content/docs/api/v1/ingestOtlpMetrics.mdx new file mode 100644 index 0000000..25e35e5 --- /dev/null +++ b/content/docs/api/v1/ingestOtlpMetrics.mdx @@ -0,0 +1,20 @@ +--- +title: Ingest OTLP metrics +full: true +_openapi: + method: POST + toc: [] + structuredData: + headings: [] + contents: + - content: > + Send OpenTelemetry metrics to Parseable using the native `/v1/metrics` + endpoint. Use `X-P-Stream` to choose the destination dataset and + `X-P-Log-Source: otel-metrics` to identify the payload type. +--- + +Use this route when OpenTelemetry metrics are being forwarded directly to Parseable. + +`X-P-Stream` chooses the destination dataset, and `X-P-Log-Source: otel-metrics` marks the payload as OTLP metrics. This route is useful when metrics are already in the OpenTelemetry pipeline and do not need Prometheus Remote Write. + + diff --git a/content/docs/api/v1/ingestOtlpTraces.mdx b/content/docs/api/v1/ingestOtlpTraces.mdx new file mode 100644 index 0000000..47a5223 --- /dev/null +++ b/content/docs/api/v1/ingestOtlpTraces.mdx @@ -0,0 +1,20 @@ +--- +title: Ingest OTLP traces +full: true +_openapi: + method: POST + toc: [] + structuredData: + headings: [] + contents: + - content: > + Send OpenTelemetry traces to Parseable using the native `/v1/traces` + endpoint. Use `X-P-Stream` to choose the destination dataset and + `X-P-Log-Source: otel-traces` to identify the payload type. +--- + +Use this route when your collector or SDK is exporting OpenTelemetry traces and you want Parseable to receive them directly. + +`X-P-Stream` chooses the destination dataset, and `X-P-Log-Source: otel-traces` identifies the payload type. This is the endpoint to use for native OTLP traces instead of custom event ingestion. + + diff --git a/content/docs/api/v1/prometheusRemoteWrite.mdx b/content/docs/api/v1/prometheusRemoteWrite.mdx new file mode 100644 index 0000000..65ba1b7 --- /dev/null +++ b/content/docs/api/v1/prometheusRemoteWrite.mdx @@ -0,0 +1,21 @@ +--- +title: Prometheus Remote Write +full: true +_openapi: + method: POST + toc: [] + structuredData: + headings: [] + contents: + - content: > + Send snappy-compressed Prometheus Remote Write payloads to + `/v1/prometheus/write`. Use `X-P-Stream` to choose the destination + dataset and `X-P-Log-Source: otel-metrics` to identify the payload + type. +--- + +Use this route when metrics are being forwarded with the Prometheus Remote Write protocol. + +This endpoint accepts the standard snappy-compressed Remote Write payload. `X-P-Stream` chooses the destination dataset, and `X-P-Log-Source: otel-metrics` tells Parseable to store the converted payload as metrics data. + + diff --git a/content/docs/api/v1/queryLogStream.mdx b/content/docs/api/v1/queryLogStream.mdx index e438f9f..37b0116 100644 --- a/content/docs/api/v1/queryLogStream.mdx +++ b/content/docs/api/v1/queryLogStream.mdx @@ -1,5 +1,5 @@ --- -title: Query a log dataset +title: Query a dataset with SQL full: true _openapi: method: POST @@ -8,46 +8,45 @@ _openapi: headings: [] contents: - content: | - **Parseable Log Query API** - This endpoint allows you to query logs using PostgreSQL syntax. + Run a SQL query in Parseable. - The query API accepts the following parameters: + The request body accepts: - **query**: The SQL query to execute (required) - **startTime**: The start time for the query range (required) - **endTime**: The end time for the query range (required) - - **streamName**: The name of the log dataset to query (required) + - **sendNull**: Return null fields in the response (optional) + - **streaming**: Stream large query results (optional) + + The dataset is selected in the SQL itself, usually in the `FROM` clause. Example request body: ```json { - "query": "SELECT * FROM log WHERE log ILIKE '%error%';", - "startTime": "2023-01-01 00:00:00.000000", - "endTime": "2023-01-01 23:59:59.999999", - "streamName": "example-dataset" + "query": "SELECT * FROM \"example-dataset\" WHERE level = 'error'", + "startTime": "2023-01-01T00:00:00Z", + "endTime": "2023-01-01T23:59:59Z", + "sendNull": false, + "streaming": false } ``` --- {/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */} -**Parseable Log Query API** -This endpoint allows you to query logs using PostgreSQL syntax. +Use this route when you want to query data in Parseable from your own code instead of the UI. -The query API accepts the following parameters: -- **query**: The SQL query to execute (required) -- **startTime**: The start time for the query range (required) -- **endTime**: The end time for the query range (required) -- **streamName**: The name of the log dataset to query (required) +The dataset is chosen inside the SQL itself, usually in the `FROM` clause. The time range is passed alongside the query so Parseable can scope the result to the period you want to inspect. Example request body: ```json { - "query": "SELECT * FROM log WHERE log ILIKE '%error%';", - "startTime": "2023-01-01 00:00:00.000000", - "endTime": "2023-01-01 23:59:59.999999", - "streamName": "example-dataset" + "query": "SELECT * FROM \"example-dataset\" WHERE level = 'error'", + "startTime": "2023-01-01T00:00:00Z", + "endTime": "2023-01-01T23:59:59Z", + "sendNull": false, + "streaming": false } ``` - \ No newline at end of file + diff --git a/content/docs/api/v1/sendLogsToStream.mdx b/content/docs/api/v1/sendLogsToStream.mdx index 22e2201..4670d4a 100644 --- a/content/docs/api/v1/sendLogsToStream.mdx +++ b/content/docs/api/v1/sendLogsToStream.mdx @@ -1,5 +1,5 @@ --- -title: Send logs to a dataset +title: Send JSON events to a dataset full: true _openapi: method: POST @@ -8,28 +8,19 @@ _openapi: headings: [] contents: - content: > - **Log Ingestion API via Stream Endpoint** + Send JSON events directly to an existing dataset. - In this approach, you need to create a dataset first using the - **Create Stream API**. - - Once the dataset is created, you can send logs to the dataset - ingestion API endpoint: - - **http://INGESTION_ENDPOINT/api/v1/logstream/`stream_name`** - - This API does not require any headers for specifying the dataset name. + The destination dataset is taken from the path, so no `X-P-Stream` + header is required. --- {/* Define variables for path parameters */} export const stream_name = "stream_name"; {/* This file was generated by Fumadocs. Do not edit this file directly. Any changes should be made by running the generation command again. */} -**Log Ingestion API via Stream Endpoint** -In this approach, you need to create a dataset first using the **Create Stream API**. -Once the dataset is created, you can send logs to the dataset ingestion API endpoint: -**http://INGESTION_ENDPOINT/api/v1/logstream/`stream_name`** -This API does not require any headers for specifying the dataset name. +Use this route when you already know the dataset name and want to send JSON events straight to it. + +Unlike `/api/v1/ingest`, the destination dataset is part of the URL, so you do not need to pass `X-P-Stream`. This makes the route useful for direct integrations where the dataset name is fixed in advance. - \ No newline at end of file + diff --git a/public/parseable-api-schema-cleaned.yaml b/public/parseable-api-schema-cleaned.yaml index 7b6938c..2588f44 100644 --- a/public/parseable-api-schema-cleaned.yaml +++ b/public/parseable-api-schema-cleaned.yaml @@ -28,11 +28,11 @@ paths: put: tags: - Dataset Management - summary: Create a log dataset + summary: Create a dataset operationId: createLogStream description: | - This endpoint is used to create a new log dataset within Parseable. - A **log dataset** is a group of similar logs. For example, you can create a log dataset for a specific application's logs, another log dataset for your database logs, and so on. You can create as many log streams as needed to organize and manage logs efficiently. + Create a dataset in Parseable using the dataset name in the path. + This endpoint is used for stream creation before writing JSON events directly to `/api/v1/logstream/{stream_name}`. security: - basicAuth: [] parameters: @@ -49,11 +49,11 @@ paths: delete: tags: - Dataset Management - summary: Delete a log dataset + summary: Delete a dataset operationId: deleteLogStream description: | - This endpoint is used to delete an existing log dataset. - When a log dataset is deleted, all associated data is permanently removed. + Delete an existing dataset. + When a dataset is deleted, all associated data is permanently removed. security: - basicAuth: [] parameters: @@ -70,14 +70,11 @@ paths: post: tags: - Dataset Management - summary: Send logs to a dataset + summary: Send JSON events to a dataset operationId: sendLogsToStream description: | - **Log Ingestion API via Stream Endpoint** - In this approach, you need to create a dataset first using the **Create Stream API**. - Once the dataset is created, you can send logs to the dataset ingestion API endpoint: - **http://INGESTION_ENDPOINT/api/v1/logstream/{stream_name}** - This API does not require any headers for specifying the dataset name. + Send JSON events directly to an existing dataset. + The destination dataset is taken from the `{stream_name}` path parameter, so no `X-P-Stream` header is needed. requestBody: content: application/json: @@ -109,28 +106,12 @@ paths: /api/v1/query: post: tags: - - Query API - summary: Query a log dataset + - Query + summary: Query a dataset with SQL operationId: queryLogStream description: | - **Parseable Log Query API** - This endpoint allows you to query logs using PostgreSQL syntax. - - The query API accepts the following parameters: - - **query**: The SQL query to execute (required) - - **startTime**: The start time for the query range (required) - - **endTime**: The end time for the query range (required) - - **streamName**: The name of the log dataset to query (required) - - Example request body: - ```json - { - "query": "SELECT * FROM log WHERE log ILIKE '%error%';", - "startTime": "2023-01-01 00:00:00.000000", - "endTime": "2023-01-01 23:59:59.999999", - "streamName": "example-dataset" - } - ``` + Run a SQL query in Parseable. + The dataset is selected inside the SQL statement itself, typically in the `FROM` clause. requestBody: content: application/json: @@ -138,7 +119,6 @@ paths: type: object required: - query - - streamName - startTime - endTime properties: @@ -148,8 +128,16 @@ paths: type: string endTime: type: string - streamName: - type: string + sendNull: + type: boolean + streaming: + type: boolean + example: + query: SELECT * FROM "example-dataset" WHERE level = 'error' + startTime: "2023-01-01T00:00:00Z" + endTime: "2023-01-01T23:59:59Z" + sendNull: false + streaming: false security: - basicAuth: [] responses: @@ -160,45 +148,112 @@ paths: /api/v1/ingest: post: tags: - - Log Ingestion - summary: Ingest logs with headers + - JSON Ingestion + summary: Ingest JSON events with headers operationId: ingestLogsWithHeaders description: | - **Log Ingestion API via Custom Headers** - This approach allows you to send logs to Parseable using custom headers to specify the target dataset. - - Required headers: - - **X-P-Stream**: The name of the dataset to ingest logs into - - The API accepts logs in JSON format. You can send single log entries or arrays of log entries. - - Example: - ```json - { - "level": "info", - "message": "User logged in", - "timestamp": "2023-01-01T12:00:00Z", - "user_id": "user123" - } - ``` - - Or as an array: - ```json - [ - { - "level": "info", - "message": "User logged in", - "timestamp": "2023-01-01T12:00:00Z", - "user_id": "user123" - }, - { - "level": "error", - "message": "Failed to process request", - "timestamp": "2023-01-01T12:01:00Z", - "error_code": "ERR-1001" - } - ] - ``` + Ingest JSON events and route them with request headers. + Use `X-P-Stream` to specify the destination dataset. + requestBody: + content: + application/json: + schema: + type: object + example: + level: info + message: User logged in + timestamp: "2023-01-01T12:00:00Z" + user_id: user123 + security: + - basicAuth: [] + parameters: + - name: X-P-Stream + in: header + schema: + type: string + required: true + responses: + '200': + description: Successful response + content: + application/json: {} + /v1/logs: + post: + tags: + - OTLP Ingestion + summary: Ingest OTLP logs + operationId: ingestOtlpLogs + description: | + Ingest OpenTelemetry logs through Parseable's native OTLP logs endpoint. + Use `X-P-Stream` to choose the destination dataset and `X-P-Log-Source: otel-logs` to identify the payload type. + requestBody: + content: + application/json: + schema: + type: object + security: + - basicAuth: [] + parameters: + - name: X-P-Stream + in: header + schema: + type: string + required: true + - name: X-P-Log-Source + in: header + schema: + type: string + enum: + - otel-logs + required: true + responses: + '200': + description: Successful response + content: + application/json: {} + /v1/traces: + post: + tags: + - OTLP Ingestion + summary: Ingest OTLP traces + operationId: ingestOtlpTraces + description: | + Ingest OpenTelemetry traces through Parseable's native OTLP traces endpoint. + Use `X-P-Stream` to choose the destination dataset and `X-P-Log-Source: otel-traces` to identify the payload type. + requestBody: + content: + application/json: + schema: + type: object + security: + - basicAuth: [] + parameters: + - name: X-P-Stream + in: header + schema: + type: string + required: true + - name: X-P-Log-Source + in: header + schema: + type: string + enum: + - otel-traces + required: true + responses: + '200': + description: Successful response + content: + application/json: {} + /v1/metrics: + post: + tags: + - OTLP Ingestion + summary: Ingest OTLP metrics + operationId: ingestOtlpMetrics + description: | + Ingest OpenTelemetry metrics through Parseable's native OTLP metrics endpoint. + Use `X-P-Stream` to choose the destination dataset and `X-P-Log-Source: otel-metrics` to identify the payload type. requestBody: content: application/json: @@ -212,6 +267,49 @@ paths: schema: type: string required: true + - name: X-P-Log-Source + in: header + schema: + type: string + enum: + - otel-metrics + required: true + responses: + '200': + description: Successful response + content: + application/json: {} + /v1/prometheus/write: + post: + tags: + - Prometheus Remote Write + summary: Ingest Prometheus remote write metrics + operationId: prometheusRemoteWrite + description: | + Ingest metrics using the Prometheus Remote Write protocol. + This endpoint accepts snappy-compressed protobuf payloads and routes the converted metrics into the target dataset. + security: + - basicAuth: [] + parameters: + - name: Content-Encoding + in: header + schema: + type: string + enum: + - snappy + required: true + - name: X-P-Stream + in: header + schema: + type: string + required: true + - name: X-P-Log-Source + in: header + schema: + type: string + enum: + - otel-metrics + required: true responses: '200': description: Successful response