diff --git a/btx/build.gradle b/btx/build.gradle new file mode 100644 index 00000000..4f85f58c --- /dev/null +++ b/btx/build.gradle @@ -0,0 +1,64 @@ +plugins { + id 'java' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } +} + +repositories { + mavenCentral() + mavenLocal() +} + +dependencies { + // Braintrust SDK (local project dependencies) + testImplementation project(':braintrust-sdk') + testImplementation project(':braintrust-sdk:instrumentation:openai_2_8_0') + testImplementation project(':braintrust-sdk:instrumentation:anthropic_2_2_0') + testImplementation project(':braintrust-sdk:instrumentation:genai_1_18_0') + testImplementation project(':braintrust-sdk:instrumentation:langchain_1_8_0') + + // Jackson for JSON processing + testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1' + + // OpenAI SDK + testImplementation 'com.openai:openai-java:2.8.1' + + // Anthropic SDK + testImplementation 'com.anthropic:anthropic-java:2.10.0' + + // Gemini SDK + testImplementation 'org.springframework.ai:spring-ai-google-genai:1.1.0' + + // LangChain4j + testImplementation 'dev.langchain4j:langchain4j:1.9.1' + testImplementation 'dev.langchain4j:langchain4j-http-client:1.9.1' + testImplementation 'dev.langchain4j:langchain4j-open-ai:1.9.1' + + // OpenTelemetry + testImplementation 'io.opentelemetry:opentelemetry-api:1.54.1' + + // YAML parsing for spec files + testImplementation 'org.yaml:snakeyaml:2.3' + + // Test framework + testImplementation(testFixtures(project(":test-harness"))) + testImplementation "org.junit.jupiter:junit-jupiter:${rootProject.ext.junitVersion}" + testImplementation "org.junit.jupiter:junit-jupiter-params:${rootProject.ext.junitVersion}" + testImplementation "io.opentelemetry:opentelemetry-sdk:${rootProject.ext.otelVersion}" + testRuntimeOnly 'org.slf4j:slf4j-simple:2.0.17' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +test { + useJUnitPlatform() + workingDir = rootProject.projectDir + testLogging { + events "passed", "skipped", "failed" + showStandardStreams = true + exceptionFormat "full" + } +} diff --git a/btx/spec/README.md b/btx/spec/README.md new file mode 100644 index 00000000..ff1b4176 --- /dev/null +++ b/btx/spec/README.md @@ -0,0 +1,9 @@ +# Braintrust Spec + +Cross language specs for implementing a Braintrust SDK. + +Contains: + +- markdown files describing complex features +- yaml describing end-to-end tests and assertions +- yaml describing cross-language constants (envars, string attributes) diff --git a/btx/spec/llm_span/README.md b/btx/spec/llm_span/README.md new file mode 100644 index 00000000..c3746915 --- /dev/null +++ b/btx/spec/llm_span/README.md @@ -0,0 +1,3 @@ +# llm span end-to-end tests + +TODO: document this diff --git a/btx/spec/llm_span/anthropic/messages.yaml b/btx/spec/llm_span/anthropic/messages.yaml new file mode 100644 index 00000000..2bd888c5 --- /dev/null +++ b/btx/spec/llm_span/anthropic/messages.yaml @@ -0,0 +1,34 @@ +name: messages +type: llm_span_test +provider: anthropic +endpoint: /v1/messages +enabled_runners: ["python", "typescript", "java", "csharp"] +requests: + - model: claude-haiku-4-5-20251001 + temperature: 0.0 + max_tokens: 128 + system: "You are a helpful assistant." + messages: + - role: user + content: What is the capital of France? +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: claude-haiku-4-5-20251001 + provider: anthropic + span_attributes: + name: anthropic.messages.create + type: llm + input: + - content: What is the capital of France? + role: user + - content: "You are a helpful assistant." + role: system + output: + content: + - text: The capital of France is Paris. + type: text + role: assistant diff --git a/btx/spec/llm_span/google/attachments.yaml b/btx/spec/llm_span/google/attachments.yaml new file mode 100644 index 00000000..857e3492 --- /dev/null +++ b/btx/spec/llm_span/google/attachments.yaml @@ -0,0 +1,44 @@ +name: attachments +type: llm_span_test +provider: google +endpoint: /v1/models/gemini-2.0-flash:generateContent +enabled_runners: ["python", "typescript", "java", "go"] +requests: + - contents: + - role: user + parts: + - text: What color is this image? + - inline_data: + mime_type: image/png + # 1x1 red pixel + data: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg== + generationConfig: + temperature: 0.0 +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: gemini-2.0-flash + span_attributes: + name: generate_content + type: llm + input: + model: gemini-2.0-flash + contents: + - role: user + parts: + - text: What color is this image? + - image_url: + url: + content_type: image/png + filename: !fn is_non_empty_string + key: !fn is_non_empty_string + type: braintrust_attachment + output: + candidates: + - content: + parts: + - text: !fn is_non_empty_string + role: model diff --git a/btx/spec/llm_span/google/generate_content.yaml b/btx/spec/llm_span/google/generate_content.yaml new file mode 100644 index 00000000..a2b7d19c --- /dev/null +++ b/btx/spec/llm_span/google/generate_content.yaml @@ -0,0 +1,34 @@ +name: generate_content +type: llm_span_test +provider: google +endpoint: /v1/models/gemini-2.5-flash:generateContent +enabled_runners: ["python", "typescript", "java", "go"] +requests: + - contents: + - role: user + parts: + - text: What is the capital of France? + generationConfig: + temperature: 0.0 +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: gemini-2.5-flash + span_attributes: + name: generate_content + type: llm + input: + model: gemini-2.5-flash + contents: + - role: user + parts: + - text: What is the capital of France? + output: + candidates: + - content: + parts: + - text: !fn is_non_empty_string + role: model diff --git a/btx/spec/llm_span/openai/attachments.yaml b/btx/spec/llm_span/openai/attachments.yaml new file mode 100644 index 00000000..21b930c6 --- /dev/null +++ b/btx/spec/llm_span/openai/attachments.yaml @@ -0,0 +1,52 @@ +name: attachments +type: llm_span_test +provider: openai +endpoint: /v1/chat/completions +requests: + - model: gpt-4o-mini + temperature: 0.0 + messages: + - role: system + content: you are a helpful assistant + - role: user + content: + - type: text + text: What color is this image? + - type: image_url + image_url: + # 1x1 red pixel + url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg== +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: !starts_with "gpt-4o-mini" + provider: openai + span_attributes: + name: Chat Completion + type: llm + input: + - role: system + content: you are a helpful assistant + - role: user + content: + - text: What color is this image? + type: text + - image_url: + url: + content_type: image/png + filename: !fn is_non_empty_string + key: !fn is_non_empty_string + type: braintrust_attachment + type: image_url + output: + - !or + - finish_reason: stop + index: 0 + message: + role: assistant + content: !fn is_non_empty_string + - role: assistant + content: !fn is_non_empty_string diff --git a/btx/spec/llm_span/openai/completions.yaml b/btx/spec/llm_span/openai/completions.yaml new file mode 100644 index 00000000..b6cce0f1 --- /dev/null +++ b/btx/spec/llm_span/openai/completions.yaml @@ -0,0 +1,34 @@ +name: completions +type: llm_span_test +provider: openai +endpoint: /v1/chat/completions +requests: + - model: gpt-4o-mini + temperature: 0.0 + messages: + - role: system + content: you are a helpful assistant + - role: user + content: What is the capital of France? +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: !starts_with "gpt-4o-mini" + provider: openai + span_attributes: + name: Chat Completion + type: llm + input: + - role: system + content: you are a helpful assistant + - role: user + content: What is the capital of France? + output: + - finish_reason: stop + index: 0 + message: + role: assistant + content: The capital of France is Paris. diff --git a/btx/spec/llm_span/openai/reasoning.yaml b/btx/spec/llm_span/openai/reasoning.yaml new file mode 100644 index 00000000..e465849c --- /dev/null +++ b/btx/spec/llm_span/openai/reasoning.yaml @@ -0,0 +1,84 @@ +name: reasoning +type: llm_span_test +provider: openai +endpoint: /v1/responses +# java instrumentation doesn't support responses yet +enabled_runners: ["python", "typescript", "java"] +requests: + - model: o4-mini + reasoning: + effort: high + summary: detailed + input: + - role: user + content: > + Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what + would be the formula for the nth term? + - model: o4-mini + reasoning: + effort: high + summary: detailed + input: + # Full context from the first response is inserted by the runner + - role: user + content: Using the pattern you discovered, what would be the 10th term? And can you find the sum of the first 10 terms? +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + completion_reasoning_tokens: !fn is_non_negative_number + metadata: + model: !starts_with "o4-mini" + provider: openai + span_attributes: + type: llm + input: + - role: user + content: > + Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what + would be the formula for the nth term? + output: + - type: reasoning + summary: !fn is_reasoning_message + - type: message + role: assistant + status: completed + content: + - type: output_text + annotations: [] + logprobs: [] + text: !fn is_non_empty_string + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + completion_reasoning_tokens: !fn is_non_negative_number + metadata: + model: !starts_with "o4-mini" + provider: openai + span_attributes: + type: llm + input: + # Light assertions since most of the context is duplicated/inserted + - role: user + - type: reasoning + summary: !fn is_reasoning_message + - type: message + role: assistant + status: completed + content: + - type: output_text + - role: user + content: Using the pattern you discovered, what would be the 10th term? And can you find the sum of the first 10 terms? + output: + - type: reasoning + summary: !fn is_reasoning_message + - type: message + role: assistant + status: completed + content: + - type: output_text + annotations: [] + logprobs: [] + text: !fn "lambda value: \"440\" in value" diff --git a/btx/spec/llm_span/openai/streaming.yaml b/btx/spec/llm_span/openai/streaming.yaml new file mode 100644 index 00000000..f9a74843 --- /dev/null +++ b/btx/spec/llm_span/openai/streaming.yaml @@ -0,0 +1,39 @@ +name: streaming +type: llm_span_test +provider: openai +endpoint: /v1/chat/completions +enabled_runners: ["python", "typescript", "java", "go", "ruby"] +requests: + - model: gpt-4o-mini + max_tokens: 800 + temperature: 0.0 + messages: + - role: system + content: you are a thoughtful assistant + - role: user + content: Count from 1 to 10 slowly. + stream: true + stream_options: + include_usage: true +expected_brainstore_spans: + - metrics: + time_to_first_token: !fn is_non_negative_number + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: !starts_with "gpt-4o-mini" + provider: openai + span_attributes: + name: Chat Completion + type: llm + input: + - role: system + content: you are a thoughtful assistant + - role: user + content: Count from 1 to 10 slowly. + output: + - index: 0 + message: + role: assistant + content: !fn is_non_empty_string diff --git a/btx/spec/llm_span/openai/tools.yaml b/btx/spec/llm_span/openai/tools.yaml new file mode 100644 index 00000000..1aa39c98 --- /dev/null +++ b/btx/spec/llm_span/openai/tools.yaml @@ -0,0 +1,60 @@ +name: tools +type: llm_span_test +provider: openai +endpoint: /v1/chat/completions +enabled_runners: ["python", "typescript", "java", "ruby"] +requests: + - model: gpt-4o + max_tokens: 500 + temperature: 0.0 + tools: + - type: function + function: + name: get_weather + description: Get the current weather for a location + parameters: + type: object + properties: + location: + type: string + description: The city and state, e.g. San Francisco, CA + unit: + type: string + enum: [celsius, fahrenheit] + description: The unit of temperature + required: [location] + messages: + - role: user + content: What is the weather like in Paris, France? +expected_brainstore_spans: + - metrics: + tokens: !fn is_non_negative_number + prompt_tokens: !fn is_non_negative_number + completion_tokens: !fn is_non_negative_number + metadata: + model: !starts_with gpt-4o + provider: openai + span_attributes: + name: Chat Completion + type: llm + input: + - role: user + content: What is the weather like in Paris, France? + output: + - !or + - finish_reason: tool_calls + index: 0 + message: + role: assistant + tool_calls: + - type: function + function: + name: get_weather + arguments: !fn "lambda value: \"Paris\" in value" + # gen_ai output messages + - role: assistant + tool_calls: + - type: function + function: + name: get_weather + arguments: !fn "lambda value: \"Paris\" in value" diff --git a/btx/spec/semconv/envar.yaml b/btx/spec/semconv/envar.yaml new file mode 100644 index 00000000..f967ae90 --- /dev/null +++ b/btx/spec/semconv/envar.yaml @@ -0,0 +1,28 @@ +description: Canonical environment variable spec for Braintrust SDKs + +config_envars: + - name: BRAINTRUST_API_KEY + required: true + documentation: API key for authentication with Braintrust. + + - name: BRAINTRUST_APP_URL + required: false + default: "https://www.braintrust.dev" + documentation: Base URL for Braintrust web application. + + - name: BRAINTRUST_API_URL + required: false + default: "https://api.braintrust.dev" + documentation: Base URL for Braintrust API. + + - name: BRAINTRUST_PROJECT + required: false + documentation: Project name for Braintrust logging and tracing. The SDK will provide a default of its choosing if this is not set. + + - name: BRAINTRUST_PROJECT_ID + required: false + documentation: Project ID (UUID) for logging spans. Takes precedence over BRAINTRUST_PROJECT if both are set. + + - name: BRAINTRUST_ORG_NAME + required: false + documentation: Organization name to use when logging in. Useful when account has multiple organizations. diff --git a/btx/spec/semconv/span_attributes.yaml b/btx/spec/semconv/span_attributes.yaml new file mode 100644 index 00000000..cd715a7d --- /dev/null +++ b/btx/spec/semconv/span_attributes.yaml @@ -0,0 +1,11 @@ +description: names and docs for span attributes special to Braintrust + +span_attributes: + - name: braintrust.parent + documentation: todo + - name: braintrust.metrics + documentation: todo + +braintrust_metrics: + - name: time_to_first_token + documentation: todo diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpec.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpec.java new file mode 100644 index 00000000..6e2796fc --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpec.java @@ -0,0 +1,85 @@ +package dev.braintrust.sdkspecimpl; + +import java.util.List; +import java.util.Map; + +/** + * Parsed representation of a single YAML file under btx/spec/llm_span/. + * + *

Example YAML: + * + *

+ * name: completions
+ * type: llm_span_test
+ * provider: openai
+ * endpoint: /v1/chat/completions
+ * enabled_runners: ["python", "java"]
+ * requests:
+ *   - model: gpt-4o-mini
+ *     messages: [...]
+ * expected_brainstore_spans:
+ *   - metrics: {...}
+ *     metadata: {...}
+ *     span_attributes: {...}
+ *     input: [...]
+ *     output: [...]
+ * 
+ */ +public record LlmSpanSpec( + String name, + String type, + String provider, + String endpoint, + String client, + List enabledRunners, + List> requests, + List> expectedBrainstoreSpans, + String sourcePath) { + + @Override + public String toString() { + return displayName(); + } + + /** + * Test display name used by JUnit. + * + *

Includes the client name when a provider has more than one client, to distinguish e.g. + * {@code "openai/completions [openai]"} from {@code "openai/completions [langchain-openai]"}. + */ + public String displayName() { + String[] parts = sourcePath.split("[/\\\\]"); + String base = parts.length >= 2 ? parts[parts.length - 2] + "/" + name : name; + List allClients = SpecLoader.clientsForProvider(provider); + return allClients.size() > 1 ? base + " [" + client + "]" : base; + } + + @SuppressWarnings("unchecked") + static LlmSpanSpec fromMap(Map raw, String sourcePath, String client) { + String name = (String) raw.get("name"); + String type = (String) raw.getOrDefault("type", "llm_span_test"); + String provider = (String) raw.get("provider"); + String endpoint = (String) raw.get("endpoint"); + + List enabledRunners = null; + Object er = raw.get("enabled_runners"); + if (er instanceof List) { + enabledRunners = (List) er; + } + + List> requests = (List>) raw.get("requests"); + List> expectedSpans = + (List>) raw.get("expected_brainstore_spans"); + + return new LlmSpanSpec( + name, + type, + provider, + endpoint, + client, + enabledRunners, + requests, + expectedSpans, + sourcePath); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java new file mode 100644 index 00000000..8f3de62b --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/LlmSpanSpecTest.java @@ -0,0 +1,69 @@ +package dev.braintrust.sdkspecimpl; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +import dev.braintrust.TestHarness; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * JUnit 5 parametrized test runner for the cross-language LLM span specs. + * + *

Loads all YAML files from {@code btx/spec/llm_span/} that include {@code "java"} in their + * {@code enabled_runners} list (or have no filter at all), executes them in-process using the + * Braintrust Java SDK, and validates the exported spans against the spec's {@code + * expected_brainstore_spans} assertions. + */ +class LlmSpanSpecTest { + + // Initialized statically so they are available when specs() is called during + // test discovery (which happens before any @Before* lifecycle methods run). + private static final TestHarness HARNESS = TestHarness.setup(); + private static final SpecExecutor EXECUTOR = new SpecExecutor(HARNESS); + private static final SpanFetcher SPAN_FETCHER = new SpanFetcher(HARNESS); + + static Stream specs() throws Exception { + List all = SpecLoader.loadAll(); + assertFalse(all.isEmpty(), "No specs found under btx/spec/llm_span/"); + final AtomicInteger totalExpectedSpans = new AtomicInteger(0); + var pool = new ForkJoinPool(3); + var results = + pool.submit( + () -> + all.parallelStream() + .map( + spec -> { + try { + var rootSpanId = + EXECUTOR.execute(spec); + totalExpectedSpans.addAndGet( + spec.expectedBrainstoreSpans() + .size() + + 1); + return Arguments.of( + spec, rootSpanId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .toList()) + .get(); + HARNESS.awaitExportedSpans(totalExpectedSpans.get()); + return results.stream(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("specs") + void runSpec(LlmSpanSpec spec, String rootSpanId) throws Exception { + int expectedSpanCount = spec.expectedBrainstoreSpans().size(); + List> brainstoreSpans = + SPAN_FETCHER.fetch(rootSpanId, expectedSpanCount); + SpanValidator.validate(brainstoreSpans, spec.expectedBrainstoreSpans(), spec.displayName()); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java new file mode 100644 index 00000000..f8f2c468 --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanConverter.java @@ -0,0 +1,98 @@ +package dev.braintrust.sdkspecimpl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Converts in-memory OTel {@link SpanData} spans into brainstore span format. + * + *

Brainstore spans are the canonical representation used in Braintrust's storage layer and + * returned by the BTQL API. The {@code expected_brainstore_spans} in the YAML spec files are + * written against this format. + * + *

The Braintrust SDK stores span payload in OTel span attributes as JSON strings: + * + *

    + *
  • {@code braintrust.metrics} → brainstore {@code metrics} field + *
  • {@code braintrust.metadata} → brainstore {@code metadata} field + *
  • {@code braintrust.span_attributes} → brainstore {@code span_attributes} field (with {@code + * name} injected from the OTel span name) + *
  • {@code braintrust.input_json} → brainstore {@code input} field + *
  • {@code braintrust.output_json} → brainstore {@code output} field + *
+ * + *

Only LLM instrumentation spans (those that have a {@code braintrust.span_attributes} + * attribute) are converted. The root wrapper span created by {@link SpecExecutor} is excluded. + */ +public class SpanConverter { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final TypeReference> MAP_TYPE = new TypeReference<>() {}; + + /** + * Convert a list of exported OTel spans into brainstore format. + * + *

Only spans that carry the {@code braintrust.span_attributes} attribute are included (i.e. + * LLM instrumentation spans). The root wrapper span is dropped. Spans are returned in the order + * they appear in {@code otelSpans}. + */ + public static List> toBrainstoreSpans(List otelSpans) { + return otelSpans.stream() + .filter(SpanConverter::isLlmInstrumentationSpan) + .map(SpanConverter::toSingleBrainstoreSpan) + .toList(); + } + + private static boolean isLlmInstrumentationSpan(SpanData span) { + return span.getAttributes().get(AttributeKey.stringKey("braintrust.span_attributes")) + != null; + } + + private static Map toSingleBrainstoreSpan(SpanData span) { + Map result = new LinkedHashMap<>(); + + result.put("name", span.getName()); + result.put("metrics", parseJsonMap(span, "braintrust.metrics")); + result.put("metadata", parseJsonMap(span, "braintrust.metadata")); + result.put("input", parseJsonValue(span, "braintrust.input_json")); + result.put("output", parseJsonValue(span, "braintrust.output_json")); + + // span_attributes: merge the JSON object with the OTel span name so that + // span_attributes.name matches what Braintrust stores after ingestion. + Map spanAttrs = parseJsonMap(span, "braintrust.span_attributes"); + if (spanAttrs == null) { + spanAttrs = new LinkedHashMap<>(); + } else { + spanAttrs = new LinkedHashMap<>(spanAttrs); + } + spanAttrs.put("name", span.getName()); + result.put("span_attributes", spanAttrs); + + return result; + } + + private static Map parseJsonMap(SpanData span, String attrKey) { + String json = span.getAttributes().get(AttributeKey.stringKey(attrKey)); + if (json == null) return null; + try { + return MAPPER.readValue(json, MAP_TYPE); + } catch (Exception e) { + throw new RuntimeException("Failed to parse " + attrKey + " as JSON map: " + json, e); + } + } + + private static Object parseJsonValue(SpanData span, String attrKey) { + String json = span.getAttributes().get(AttributeKey.stringKey(attrKey)); + if (json == null) return null; + try { + return MAPPER.readValue(json, Object.class); + } catch (Exception e) { + throw new RuntimeException("Failed to parse " + attrKey + " as JSON: " + json, e); + } + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java new file mode 100644 index 00000000..5c4a858c --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanFetcher.java @@ -0,0 +1,387 @@ +package dev.braintrust.sdkspecimpl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.TestHarness; +import dev.braintrust.VCR; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +/** + * Fetches brainstore spans from the Braintrust API via a BTQL query. + * + *

Used in {@code RECORD} and {@code OFF} VCR modes where real API calls are made and spans are + * actually ingested into Braintrust. Retries with backoff until the expected number of child spans + * appears (mirrors the Python BTX framework's {@code fetch_braintrust_spans}). + */ +public class SpanFetcher { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final TypeReference> MAP_TYPE = new TypeReference<>() {}; + + private static final int BACKOFF_SECONDS = 30; + private static final int MAX_TOTAL_WAIT_SECONDS = 600; + + private final HttpClient httpClient; + private final TestHarness harness; + private final String btqlUrl; + private final String apiKey; + private final String projectId; + + public SpanFetcher(TestHarness harness) { + this.harness = harness; + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + this.btqlUrl = harness.braintrustApiBaseUrl().replaceAll("/+$", "") + "/btql"; + this.apiKey = harness.braintrustApiKey(); + this.projectId = TestHarness.defaultProjectId(); + } + + /** + * Fetch child brainstore spans for the given trace, retrying until {@code + * numExpectedChildSpans} are available. + * + * @param rootSpanId hex trace ID (from OTel span context, e.g. {@code + * "e6f892e37dac9e3ef2f8906d6600d70c"}) + * @param numExpectedChildSpans number of child spans to wait for + */ + public List> fetch(String rootSpanId, int numExpectedChildSpans) + throws Exception { + List otelSpans = + harness.awaitExportedSpans(numExpectedChildSpans + 1).stream() + .filter(spanData -> spanData.getTraceId().equals(rootSpanId)) + .toList(); + List> brainstoreSpans; + List> convertedOtelSpans = SpanConverter.toBrainstoreSpans(otelSpans); + if (isReplayMode()) { + // Fast path: convert the in-memory OTel spans to brainstore format locally. + brainstoreSpans = convertedOtelSpans; + } else { + // Live path: spans were actually sent to Braintrust — fetch them back via BTQL. + brainstoreSpans = fetchFromBrainstore(rootSpanId, numExpectedChildSpans); + // assert that our converted otel spans will match what is in brainstore + assertConverterMatchesBrainstore(convertedOtelSpans, brainstoreSpans, rootSpanId); + } + return brainstoreSpans; + } + + /** + * Fetch child brainstore spans for the given trace, retrying until {@code + * numExpectedChildSpans} are available. + * + * @param rootSpanId hex trace ID (from OTel span context, e.g. {@code + * "e6f892e37dac9e3ef2f8906d6600d70c"}) + * @param numExpectedChildSpans number of child spans to wait for + */ + private List> fetchFromBrainstore( + String rootSpanId, int numExpectedChildSpans) throws Exception { + int totalWait = 0; + LookupException lastError = null; + + while (true) { + try { + return fetchOnce(rootSpanId, numExpectedChildSpans); + } catch (LookupException e) { + lastError = e; + if (totalWait >= MAX_TOTAL_WAIT_SECONDS) { + break; + } + System.out.printf( + "Spans not ready yet, waiting %ds before retry (total wait: %ds)...%n", + BACKOFF_SECONDS, totalWait); + Thread.sleep(BACKOFF_SECONDS * 1000L); + totalWait += BACKOFF_SECONDS; + } + } + throw new RuntimeException( + "Timed out waiting for brainstore spans after " + MAX_TOTAL_WAIT_SECONDS + "s", + lastError); + } + + /** + * Assert that the spans produced by {@link SpanConverter} are a proper subset of the + * authoritative brainstore spans — i.e. every key/value present in a converted span also + * appears with the same value in the corresponding real brainstore span. + * + *

Spans are matched by {@code span_attributes.name} to avoid sensitivity to ordering + * differences between OTel export order and BTQL {@code created ASC}. + */ + @SuppressWarnings("unchecked") + private static void assertConverterMatchesBrainstore( + List> converted, + List> real, + String rootSpanId) { + if (converted.size() != real.size()) { + throw new AssertionError( + "SpanConverter produced " + + converted.size() + + " spans but brainstore has " + + real.size() + + " for root_span_id " + + rootSpanId); + } + // Index real spans by their span_attributes.name for order-independent matching + Map> realByName = new java.util.LinkedHashMap<>(); + for (Map span : real) { + Object sa = span.get("span_attributes"); + String name = sa instanceof Map ? (String) ((Map) sa).get("name") : null; + if (name == null) name = (String) span.get("name"); + if (name != null) realByName.put(name, span); + } + for (int i = 0; i < converted.size(); i++) { + Map conv = converted.get(i); + String name = (String) conv.get("name"); + Map realSpan = realByName.getOrDefault(name, real.get(i)); + String ctx = "SpanConverter[" + name + "]"; + // "name" is a synthetic top-level field added by SpanConverter for spec-assertion + // convenience; in real brainstore spans the name lives in span_attributes.name. + // "metrics" values are numeric and may vary across runs (e.g. reasoning token counts + // are non-deterministic); we only check that the same keys are present and non-null. + Map convWithoutName = new java.util.LinkedHashMap<>(conv); + convWithoutName.remove("name"); + convWithoutName.remove("metrics"); + assertIsSubset(convWithoutName, realSpan, ctx); + assertMetricsKeysPresent(conv, realSpan, ctx); + } + } + + /** + * Assert that every metric key present in the converted span also appears as a non-null number + * in the real brainstore span. Token counts are non-deterministic (especially for reasoning), + * so we only check presence and type rather than exact equality. + */ + @SuppressWarnings("unchecked") + private static void assertMetricsKeysPresent( + Map conv, Map realSpan, String ctx) { + Object convMetrics = conv.get("metrics"); + if (!(convMetrics instanceof Map)) return; + Object realMetrics = realSpan.get("metrics"); + if (!(realMetrics instanceof Map)) return; // brainstore may omit metrics; skip + Map convM = (Map) convMetrics; + Map realM = (Map) realMetrics; + for (String key : convM.keySet()) { + if (convM.get(key) == null) continue; + Object realVal = realM.get(key); + if (realVal == null) continue; // brainstore may compute differently; skip + if (!(realVal instanceof Number)) { + throw new AssertionError( + ctx + + ".metrics." + + key + + ": expected a Number but got " + + realVal.getClass().getSimpleName()); + } + } + } + + /** + * Assert that every key/value in {@code subset} is present with an equal value in {@code + * superset}, recursively for nested maps. Several cases are skipped: + * + *

    + *
  • Null values in {@code subset} — "don't care" + *
  • Null values in {@code superset} — brainstore may omit or transform certain fields (e.g. + * inline image data is not echoed back) + *
  • Values containing {@code braintrust_attachment} — the converter logs the raw data URL + * while brainstore stores an uploaded attachment reference; both are correct + *
+ */ + @SuppressWarnings("unchecked") + private static void assertIsSubset(Object subset, Object superset, String ctx) { + if (subset == null) return; + if (superset == null) return; // brainstore may omit or transform certain fields + // Skip when either side contains an attachment reference — the converter logs the raw + // data URL while brainstore stores an uploaded attachment reference; both are correct. + if (SpanValidator.containsBraintrustAttachment(subset)) return; + if (SpanValidator.containsBraintrustAttachment(superset)) return; + // Skip when brainstore has transformed a leaf value into a different type (e.g. a string + // ID that got replaced with a structured object, or vice versa). + boolean subIsColl = subset instanceof Map || subset instanceof List; + boolean supIsColl = superset instanceof Map || superset instanceof List; + if (subIsColl != supIsColl) return; + if (subset instanceof List) { + List subList = (List) subset; + List superList = (List) superset; + for (int i = 0; i < subList.size() && i < superList.size(); i++) { + assertIsSubset(subList.get(i), superList.get(i), ctx + "[" + i + "]"); + } + return; + } + if (!(subset instanceof Map)) { + // For string leaves, only assert non-null — actual content may vary between runs + // (e.g. model-generated text, reasoning summaries). For numbers and booleans, + // use exact equality since those are deterministic (types, finish reasons, etc.). + if (subset instanceof String) { + if (superset == null) { + throw new AssertionError(ctx + ": expected a non-null String but got null"); + } + } else { + SpanValidator.validateValue(superset, subset, ctx); + } + return; + } + Map subMap = (Map) subset; + Map superMap = (Map) superset; + for (Map.Entry entry : subMap.entrySet()) { + if (entry.getValue() == null) continue; + if ("id".equals(entry.getKey())) continue; // IDs are dynamic and non-deterministic + assertIsSubset( + entry.getValue(), superMap.get(entry.getKey()), ctx + "." + entry.getKey()); + } + } + + @SuppressWarnings("unchecked") + private List> fetchOnce(String rootSpanId, int numExpectedChildSpans) + throws Exception { + + String body = buildBtqlQuery(rootSpanId); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(btqlUrl)) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body)) + .timeout(Duration.ofSeconds(30)) + .build(); + + HttpResponse response = + httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new LookupException( + "BTQL query failed with status " + + response.statusCode() + + ": " + + response.body()); + } + + Map result = MAPPER.readValue(response.body(), MAP_TYPE); + List> allSpans = (List>) result.get("data"); + if (allSpans == null) { + throw new LookupException("BTQL response missing 'data' field"); + } + + // Filter out scorer spans injected by the Braintrust backend + List> childSpans = + allSpans.stream() + .filter( + s -> { + Object sa = s.get("span_attributes"); + if (sa instanceof Map) { + return !"scorer".equals(((Map) sa).get("purpose")); + } + return true; + }) + .toList(); + + int actual = childSpans.size(); + if (actual == 0) { + throw new LookupException("No child spans found yet for root_span_id: " + rootSpanId); + } + if (actual < numExpectedChildSpans) { + throw new LookupException( + "Expected " + + numExpectedChildSpans + + " child spans, only found " + + actual + + " so far"); + } + if (actual > numExpectedChildSpans) { + throw new RuntimeException( + "Expected " + + numExpectedChildSpans + + " child spans but found " + + actual + + " — too many (non-retriable)"); + } + + // Retry if any span is still incomplete (output or metrics not yet ingested). + // Braintrust may ingest the span skeleton before the payload fields are indexed. + for (Map span : childSpans) { + if (span.get("output") == null && span.get("metrics") == null) { + throw new LookupException( + "Span found but output/metrics not yet ingested (span_id: " + + span.get("span_id") + + ")"); + } + } + + return childSpans; + } + + /** + * Build the BTQL query JSON string. Uses LinkedHashMap throughout because Map.of() rejects null + * values (needed for the span_parents != null filter). + */ + private String buildBtqlQuery(String rootSpanId) throws Exception { + // span_parents != null literal node (Map.of rejects nulls, so use LinkedHashMap) + Map nullLiteral = new java.util.LinkedHashMap<>(); + nullLiteral.put("op", "literal"); + nullLiteral.put("value", null); + + Map query = new java.util.LinkedHashMap<>(); + query.put("query", buildQueryNode(rootSpanId, nullLiteral)); + query.put("use_columnstore", true); + query.put("use_brainstore", true); + query.put("brainstore_realtime", true); + + return MAPPER.writeValueAsString(query); + } + + private Map buildQueryNode(String rootSpanId, Map nullLiteral) { + Map q = new java.util.LinkedHashMap<>(); + q.put("select", List.of(Map.of("op", "star"))); + q.put( + "from", + Map.of( + "op", "function", + "name", Map.of("op", "ident", "name", List.of("project_logs")), + "args", List.of(Map.of("op", "literal", "value", projectId)))); + q.put( + "filter", + Map.of( + "op", + "and", + "left", + Map.of( + "op", "eq", + "left", Map.of("op", "ident", "name", List.of("root_span_id")), + "right", Map.of("op", "literal", "value", rootSpanId)), + "right", + Map.of( + "op", + "ne", + "left", + Map.of("op", "ident", "name", List.of("span_parents")), + "right", + nullLiteral))); + q.put( + "sort", + List.of( + Map.of( + "expr", + Map.of("op", "ident", "name", List.of("created")), + "dir", + "asc"))); + q.put("limit", 1000); + return q; + } + + /** Retriable error: spans not yet available, caller should retry. */ + private static class LookupException extends Exception { + LookupException(String msg) { + super(msg); + } + } + + /** Returns true when running in VCR replay mode (the default). */ + private static boolean isReplayMode() { + return TestHarness.getVcrMode().equals(VCR.VcrMode.REPLAY); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java new file mode 100644 index 00000000..d09fb291 --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpanValidator.java @@ -0,0 +1,285 @@ +package dev.braintrust.sdkspecimpl; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Validates a list of brainstore spans against the {@code expected_brainstore_spans} structure from + * a YAML spec. + * + *

Brainstore spans have the following top-level fields that the spec may assert on: + * + *

    + *
  • {@code metrics} — token counts, timing + *
  • {@code metadata} — model, provider + *
  • {@code span_attributes} — type, name + *
  • {@code input} — input messages + *
  • {@code output} — output choices / content + *
+ * + *

Spans arrive here already in brainstore format, produced either by {@link SpanConverter} + * (REPLAY mode) or {@link SpanFetcher} (RECORD / OFF mode). + */ +public class SpanValidator { + + /** + * Validate that the brainstore spans match the expected structures from the spec. + * + * @param brainstoreSpans spans in brainstore format (child LLM spans only, no root wrapper) + * @param expectedSpans the {@code expected_brainstore_spans} list from the YAML spec + * @param specName display name for error messages + */ + public static void validate( + List> brainstoreSpans, + List> expectedSpans, + String specName) { + + if (brainstoreSpans.size() < expectedSpans.size()) { + fail( + String.format( + "[%s] Expected at least %d brainstore spans but got %d", + specName, expectedSpans.size(), brainstoreSpans.size())); + } + + for (int i = 0; i < expectedSpans.size(); i++) { + validateSpan(brainstoreSpans.get(i), expectedSpans.get(i), specName + "[" + i + "]"); + } + } + + @SuppressWarnings("unchecked") + private static void validateSpan( + Map actual, Map expected, String context) { + for (Map.Entry entry : expected.entrySet()) { + String field = entry.getKey(); + Object expectedValue = entry.getValue(); + Object actualValue = actual.get(field); + validateValue(actualValue, expectedValue, context + "." + field); + } + } + + private static void validateMap( + Map actual, Map expected, String context) { + if (actual == null) { + fail(context + ": expected a Map but got null"); + } + for (Map.Entry entry : expected.entrySet()) { + String key = entry.getKey(); + Object actualVal = actual.get(key); + validateValue(actualVal, entry.getValue(), context + "." + key); + } + } + + @SuppressWarnings("unchecked") + static void validateValue(Object actual, Object expected, String context) { + // Skip assertions that reference braintrust_attachment — uploading attachments requires + // a real Braintrust storage backend, not exercised in the in-process runner. + if (containsBraintrustAttachment(expected)) { + return; + } + + if (expected instanceof SpecMatcher) { + assertMatcher(actual, (SpecMatcher) expected, context); + } else if (expected instanceof Map) { + if (!(actual instanceof Map)) { + fail( + String.format( + "%s: expected a Map but got %s (value: %s)", + context, + actual == null ? "null" : actual.getClass().getSimpleName(), + actual)); + } + validateMap((Map) actual, (Map) expected, context); + } else if (expected instanceof List) { + List expList = (List) expected; + if (!(actual instanceof List)) { + // output may be a single Map (e.g. Anthropic returns an object, not an array) + if (expList.size() == 1 && actual instanceof Map) { + validateValue(actual, expList.get(0), context + "[0]"); + return; + } + fail( + String.format( + "%s: expected a List but got %s (value: %s)", + context, + actual == null ? "null" : actual.getClass().getSimpleName(), + actual)); + } + List actualList = (List) actual; + if (actualList.size() < expList.size()) { + fail( + String.format( + "%s: expected at least %d items but got %d. actual=%s", + context, expList.size(), actualList.size(), actualList)); + } + for (int i = 0; i < expList.size(); i++) { + validateValue(actualList.get(i), expList.get(i), context + "[" + i + "]"); + } + } else { + // scalar: null expected means "don't care" + if (expected == null) return; + if (!valuesEqual(actual, expected)) { + fail( + String.format( + "%s: expected %s (%s) but got %s (%s)", + context, + expected, + expected.getClass().getSimpleName(), + actual, + actual == null ? "null" : actual.getClass().getSimpleName())); + } + } + } + + private static void assertMatcher(Object actual, SpecMatcher matcher, String context) { + if (matcher instanceof SpecMatcher.FnMatcher) { + assertFnMatcher(actual, (SpecMatcher.FnMatcher) matcher, context); + } else if (matcher instanceof SpecMatcher.StartsWithMatcher) { + SpecMatcher.StartsWithMatcher sw = (SpecMatcher.StartsWithMatcher) matcher; + if (!(actual instanceof String)) { + fail( + String.format( + "%s: starts_with(%s): expected a String but got %s (value: %s)", + context, + sw.prefix(), + actual == null ? "null" : actual.getClass().getSimpleName(), + actual)); + } + if (!((String) actual).startsWith(sw.prefix())) { + fail( + String.format( + "%s: starts_with(%s): value '%s' does not start with prefix", + context, sw.prefix(), actual)); + } + } else if (matcher instanceof SpecMatcher.OrMatcher) { + SpecMatcher.OrMatcher or = (SpecMatcher.OrMatcher) matcher; + List failures = new ArrayList<>(); + for (Object alternative : or.alternatives()) { + try { + validateValue(actual, alternative, context); + return; // matched + } catch (AssertionError e) { + failures.add(e.getMessage()); + } + } + fail( + String.format( + "%s: !or: no alternative matched. Failures:\n %s\nActual: %s", + context, String.join("\n ", failures), actual)); + } else { + fail(context + ": unknown SpecMatcher: " + matcher.getClass().getSimpleName()); + } + } + + private static void assertFnMatcher(Object actual, SpecMatcher.FnMatcher fn, String context) { + switch (fn.name()) { + case "is_non_negative_number" -> { + if (!(actual instanceof Number)) { + fail( + String.format( + "%s: is_non_negative_number: expected a Number but got %s" + + " (value: %s)", + context, + actual == null ? "null" : actual.getClass().getSimpleName(), + actual)); + } + double v = ((Number) actual).doubleValue(); + if (v < 0) { + fail( + String.format( + "%s: is_non_negative_number: value %s is negative", + context, v)); + } + } + case "is_non_empty_string" -> { + if (!(actual instanceof String) || ((String) actual).isEmpty()) { + fail( + String.format( + "%s: is_non_empty_string: expected non-empty String but got" + + " %s (value: %s)", + context, + actual == null ? "null" : actual.getClass().getSimpleName(), + actual)); + } + } + case "is_reasoning_message" -> { + // A reasoning summary is a list of {type: "summary_text", text: "..."} objects. + // An empty list is also acceptable (reasoning may not always occur). + if (actual == null) { + fail(context + ": is_reasoning_message: value is null"); + } + if (actual instanceof List) { + @SuppressWarnings("unchecked") + List items = (List) actual; + for (Object item : items) { + if (!(item instanceof Map)) { + fail( + context + + ": is_reasoning_message: list item is not a Map:" + + " " + + item); + } + @SuppressWarnings("unchecked") + Map m = (Map) item; + if (!"summary_text".equals(m.get("type"))) { + fail( + context + + ": is_reasoning_message: item type is not" + + " 'summary_text': " + + m.get("type")); + } + Object text = m.get("text"); + if (!(text instanceof String) || ((String) text).isBlank()) { + fail(context + ": is_reasoning_message: item text is empty: " + text); + } + } + } else if (actual instanceof String && ((String) actual).isEmpty()) { + fail(context + ": is_reasoning_message: value is empty string"); + } + // non-list, non-empty value is acceptable (e.g. a non-null string) + } + default -> { + // Python lambda-style predicates: just assert non-null / non-empty + if (actual == null) { + fail(String.format("%s: fn(%s): value is null", context, fn.name())); + } + if (actual instanceof String && ((String) actual).isEmpty()) { + fail(String.format("%s: fn(%s): value is empty string", context, fn.name())); + } + } + } + } + + // ---- helpers ---------------------------------------------------------------- + + @SuppressWarnings("unchecked") + static boolean containsBraintrustAttachment(Object value) { + if (value instanceof Map) { + Map map = (Map) value; + if ("braintrust_attachment".equals(map.get("type"))) return true; + for (Object v : map.values()) { + if (containsBraintrustAttachment(v)) return true; + } + } else if (value instanceof List) { + for (Object item : (List) value) { + if (containsBraintrustAttachment(item)) return true; + } + } else if (value instanceof SpecMatcher.OrMatcher) { + for (Object alt : ((SpecMatcher.OrMatcher) value).alternatives()) { + if (containsBraintrustAttachment(alt)) return true; + } + } + return false; + } + + private static boolean valuesEqual(Object actual, Object expected) { + if (actual == null && expected == null) return true; + if (actual == null || expected == null) return false; + if (actual instanceof Number && expected instanceof Number) { + return ((Number) actual).doubleValue() == ((Number) expected).doubleValue(); + } + return actual.equals(expected); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java new file mode 100644 index 00000000..5086ee6c --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecExecutor.java @@ -0,0 +1,359 @@ +package dev.braintrust.sdkspecimpl; + +import com.anthropic.client.AnthropicClient; +import com.anthropic.client.okhttp.AnthropicOkHttpClient; +import com.anthropic.models.messages.MessageCreateParams; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.Client; +import com.google.genai.types.GenerateContentConfig; +import com.google.genai.types.GenerateContentResponse; +import com.google.genai.types.HttpOptions; +import com.google.genai.types.Part; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.core.ObjectMappers; +import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.responses.Response; +import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseInputItem; +import com.openai.models.responses.ResponseOutputItem; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic; +import dev.braintrust.instrumentation.genai.BraintrustGenAI; +import dev.braintrust.instrumentation.langchain.BraintrustLangchain; +import dev.braintrust.instrumentation.openai.BraintrustOpenAI; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.openai.OpenAiChatModel; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +/** + * Executes LLM spec tests in-process using the Braintrust Java SDK instrumentation. + * + *

Each call to {@link #execute(LlmSpanSpec)} makes the real provider API calls (or uses VCR + * cassettes in replay mode) wrapped in an OTel root span, then returns. The caller can then collect + * exported spans from {@link TestHarness#awaitExportedSpans(int)}. + */ +public class SpecExecutor { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final Tracer tracer; + private final OpenAIClient openAIClient; + private final AnthropicClient anthropicClient; + private final Client geminiClient; + private final String openAiBaseUrl; + private final String openAiApiKey; + private final io.opentelemetry.api.OpenTelemetry otel; + + public SpecExecutor(TestHarness harness) { + OpenTelemetrySdk otelSdk = harness.openTelemetry(); + this.otel = otelSdk; + this.tracer = otelSdk.getTracer("btx"); + this.openAiBaseUrl = harness.openAiBaseUrl(); + this.openAiApiKey = harness.openAiApiKey(); + + this.openAIClient = + BraintrustOpenAI.wrapOpenAI( + otelSdk, + OpenAIOkHttpClient.builder() + .baseUrl(openAiBaseUrl) + .apiKey(openAiApiKey) + .build()); + + this.anthropicClient = + BraintrustAnthropic.wrap( + otelSdk, + AnthropicOkHttpClient.builder() + .baseUrl(harness.anthropicBaseUrl()) + .apiKey(harness.anthropicApiKey()) + .build()); + + String googleApiKey = harness.googleApiKey(); + var geminiBuilder = + new Client.Builder() + .apiKey(googleApiKey) + .httpOptions( + HttpOptions.builder().baseUrl(harness.googleBaseUrl()).build()); + this.geminiClient = BraintrustGenAI.wrap(otelSdk, geminiBuilder); + } + + /** + * Execute all requests defined in the spec, wrapped in a root span named after the spec. + * + * @return the OTel trace ID of the root span (hex string, e.g. {@code + * "e6f892e37dac9e3ef2f8906d6600d70c"}), which Braintrust stores as {@code root_span_id} + */ + public String execute(LlmSpanSpec spec) throws Exception { + Span rootSpan = tracer.spanBuilder(spec.name()).startSpan(); + rootSpan.setAttribute("client", spec.client()); + try (var ignored = rootSpan.makeCurrent()) { + // History is accumulated across multi-turn requests + List responsesHistory = new ArrayList<>(); + + for (Map request : spec.requests()) { + dispatchRequest( + spec.provider(), spec.endpoint(), spec.client(), request, responsesHistory); + } + } finally { + rootSpan.end(); + } + return rootSpan.getSpanContext().getTraceId(); + } + + private void dispatchRequest( + String provider, + String endpoint, + String client, + Map request, + List responsesHistory) + throws Exception { + if ("openai".equals(provider) && "/v1/chat/completions".equals(endpoint)) { + if ("langchain-openai".equals(client)) { + executeLangChainChatCompletion(request); + } else { + executeChatCompletion(request); + } + } else if ("openai".equals(provider) && "/v1/responses".equals(endpoint)) { + executeResponses(request, responsesHistory); + } else if ("anthropic".equals(provider) && "/v1/messages".equals(endpoint)) { + executeAnthropicMessages(request); + } else if ("google".equals(provider) && endpoint.contains(":generateContent")) { + executeGeminiGenerateContent(request, endpoint); + } else { + throw new UnsupportedOperationException( + "Provider " + + provider + + " endpoint " + + endpoint + + " client " + + client + + " not supported"); + } + } + + // ---- OpenAI chat/completions ------------------------------------------------ + + private void executeChatCompletion(Map request) throws Exception { + boolean streaming = Boolean.TRUE.equals(request.get("stream")); + // Serialize the whole request map to JSON, then let the SDK's mapper deserialize each + // field into the correct SDK type — no manual field extraction needed. + String json = MAPPER.writeValueAsString(request); + com.fasterxml.jackson.databind.JsonNode node = ObjectMappers.jsonMapper().readTree(json); + + var builder = ChatCompletionCreateParams.builder(); + if (node.has("model")) + builder.model(com.openai.models.ChatModel.of(node.get("model").asText())); + if (node.has("messages")) { + List msgs = + ObjectMappers.jsonMapper() + .convertValue( + node.get("messages"), + ObjectMappers.jsonMapper() + .getTypeFactory() + .constructCollectionType( + List.class, + com.openai.models.chat.completions + .ChatCompletionMessageParam.class)); + builder.messages(msgs); + } + if (node.has("tools")) { + List tools = + ObjectMappers.jsonMapper() + .convertValue( + node.get("tools"), + ObjectMappers.jsonMapper() + .getTypeFactory() + .constructCollectionType( + List.class, + com.openai.models.chat.completions + .ChatCompletionTool.class)); + builder.tools(tools); + } + if (node.has("temperature")) builder.temperature(node.get("temperature").asDouble()); + if (node.has("max_tokens")) builder.maxCompletionTokens(node.get("max_tokens").asLong()); + if (node.has("stream_options")) + builder.streamOptions( + ObjectMappers.jsonMapper() + .convertValue( + node.get("stream_options"), + com.openai.models.chat.completions.ChatCompletionStreamOptions + .class)); + + var params = builder.build(); + if (streaming) { + // Hold a reference to prevent GC-driven PhantomReachable cleanup before the stream + // is fully consumed, which would close the SSE stream early. + try (var stream = openAIClient.chat().completions().createStreaming(params)) { + stream.stream().forEach(chunk -> {}); + } + } else { + openAIClient.chat().completions().create(params); + } + } + + // ---- LangChain4j OpenAI chat/completions ------------------------------------ + + @SuppressWarnings("unchecked") + private void executeLangChainChatCompletion(Map request) throws Exception { + var modelBuilder = OpenAiChatModel.builder().baseUrl(openAiBaseUrl).apiKey(openAiApiKey); + if (request.containsKey("model")) { + modelBuilder.modelName((String) request.get("model")); + } + if (request.containsKey("temperature")) { + modelBuilder.temperature(((Number) request.get("temperature")).doubleValue()); + } + if (request.containsKey("max_tokens")) { + modelBuilder.maxTokens(((Number) request.get("max_tokens")).intValue()); + } + ChatModel model = BraintrustLangchain.wrap(otel, modelBuilder); + + List messages = new ArrayList<>(); + for (Map msg : (List>) request.get("messages")) { + String role = (String) msg.get("role"); + String content = msg.get("content") != null ? msg.get("content").toString() : ""; + switch (role) { + case "system" -> messages.add(SystemMessage.from(content)); + case "user" -> messages.add(UserMessage.from(content)); + default -> + throw new UnsupportedOperationException( + "langchain-openai: unsupported role: " + role); + } + } + model.chat(messages); + } + + // ---- OpenAI responses ------------------------------------------------------- + + private void executeResponses(Map request, List history) + throws Exception { + String json = MAPPER.writeValueAsString(request); + com.fasterxml.jackson.databind.JsonNode node = ObjectMappers.jsonMapper().readTree(json); + + // Deserialize this turn's input items + List thisInput = + ObjectMappers.jsonMapper() + .convertValue( + node.get("input"), + ObjectMappers.jsonMapper() + .getTypeFactory() + .constructCollectionType( + List.class, ResponseInputItem.class)); + + // Prepend accumulated history from previous turns + List fullInput = new ArrayList<>(history); + fullInput.addAll(thisInput); + + var builder = ResponseCreateParams.builder().inputOfResponse(fullInput); + if (node.has("model")) builder.model(node.get("model").asText()); + if (node.has("reasoning")) + builder.reasoning( + ObjectMappers.jsonMapper() + .convertValue( + node.get("reasoning"), com.openai.models.Reasoning.class)); + + Response response = openAIClient.responses().create(builder.build()); + + // Accumulate this turn's input + output into history for the next turn + history.addAll(thisInput); + for (ResponseOutputItem out : response.output()) { + String outJson = ObjectMappers.jsonMapper().writeValueAsString(out); + history.add(ObjectMappers.jsonMapper().readValue(outJson, ResponseInputItem.class)); + } + } + + // ---- Anthropic -------------------------------------------------------------- + + private void executeAnthropicMessages(Map request) throws Exception { + String json = MAPPER.writeValueAsString(request); + com.fasterxml.jackson.databind.JsonNode node = + com.anthropic.core.ObjectMappers.jsonMapper().readTree(json); + + var builder = MessageCreateParams.builder(); + if (node.has("model")) builder.model(node.get("model").asText()); + if (node.has("max_tokens")) builder.maxTokens(node.get("max_tokens").asLong()); + if (node.has("temperature")) builder.temperature(node.get("temperature").asDouble()); + if (node.has("system")) builder.system(node.get("system").asText()); + if (node.has("messages")) { + List msgs = + com.anthropic.core.ObjectMappers.jsonMapper() + .convertValue( + node.get("messages"), + com.anthropic.core.ObjectMappers.jsonMapper() + .getTypeFactory() + .constructCollectionType( + List.class, + com.anthropic.models.messages.MessageParam + .class)); + builder.messages(msgs); + } + + anthropicClient.messages().create(builder.build()); + } + + // ---- Google Gemini ---------------------------------------------------------- + + @SuppressWarnings("unchecked") + private void executeGeminiGenerateContent(Map request, String endpoint) + throws Exception { + String model = extractModelFromEndpoint(endpoint); + + List parts = new ArrayList<>(); + if (request.containsKey("contents")) { + for (Map content : + (List>) request.get("contents")) { + for (Map part : (List>) content.get("parts")) { + if (part.containsKey("text")) { + parts.add(Part.fromText((String) part.get("text"))); + } else if (part.containsKey("inline_data")) { + Map inline = (Map) part.get("inline_data"); + String mime = (String) inline.get("mime_type"); + byte[] bytes = Base64.getDecoder().decode((String) inline.get("data")); + parts.add(Part.fromBytes(bytes, mime)); + } + } + } + } + + com.google.genai.types.Content content = + com.google.genai.types.Content.fromParts(parts.toArray(new Part[0])); + + var configBuilder = GenerateContentConfig.builder(); + if (request.containsKey("generationConfig")) { + Map gc = (Map) request.get("generationConfig"); + if (gc.containsKey("temperature")) { + configBuilder.temperature(((Number) gc.get("temperature")).floatValue()); + } + if (gc.containsKey("maxOutputTokens")) { + configBuilder.maxOutputTokens(((Number) gc.get("maxOutputTokens")).intValue()); + } + } + + boolean streaming = + request.containsKey("stream") && Boolean.TRUE.equals(request.get("stream")); + if (streaming) { + for (GenerateContentResponse ignored : + geminiClient.models.generateContentStream( + model, content, configBuilder.build())) {} + } else { + geminiClient.models.generateContent(model, content, configBuilder.build()); + } + } + + private static String extractModelFromEndpoint(String endpoint) { + int modelsIndex = endpoint.indexOf("/models/"); + int colonIndex = endpoint.indexOf(":", modelsIndex); + if (modelsIndex == -1 || colonIndex == -1) { + throw new IllegalArgumentException("Invalid Gemini endpoint: " + endpoint); + } + return endpoint.substring(modelsIndex + "/models/".length(), colonIndex); + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java new file mode 100644 index 00000000..0902d8fc --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecLoader.java @@ -0,0 +1,169 @@ +package dev.braintrust.sdkspecimpl; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.AbstractConstruct; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; +import org.yaml.snakeyaml.nodes.Tag; + +/** + * Loads LLM span spec YAML files from btx/spec/llm_span/ and parses them into {@link LlmSpanSpec} + * objects. + * + *

Handles the custom YAML tags used in the spec files: + * + *

    + *
  • {@code !fn } - a named predicate (e.g. {@code is_non_negative_number}) + *
  • {@code !starts_with "prefix"} - asserts the value starts with a string + *
  • {@code !or [...]} - asserts the value matches at least one of a list of structures + *
+ */ +public class SpecLoader { + + private static final String SPEC_ROOT = "btx/spec/llm_span"; + + /** + * The clients supported by this Java runner, keyed by provider name. Each entry is the list of + * client identifiers that will be tested for that provider. When a provider is not listed here, + * it defaults to a single client whose name matches the provider (e.g. {@code "anthropic"}). + */ + static final Map> CLIENTS_BY_PROVIDER = + Map.of("openai", List.of("openai", "langchain-openai")); + + /** + * Returns the clients to test for the given provider. Defaults to {@code [providerName]} if the + * provider is not explicitly listed in {@link #CLIENTS_BY_PROVIDER}. + */ + public static List clientsForProvider(String provider) { + return CLIENTS_BY_PROVIDER.getOrDefault(provider, List.of(provider)); + } + + /** + * Load all LLM span specs that include "java" in their enabled_runners list, expanded into one + * {@link LlmSpanSpec} per supported client for the spec's provider. + */ + public static List loadAll() throws IOException { + Path root = Paths.get(SPEC_ROOT); + List specs = new ArrayList<>(); + try (Stream paths = Files.walk(root)) { + paths.filter(p -> p.toString().endsWith(".yaml")) + .sorted() + .forEach( + path -> { + try { + specs.addAll(load(path)); + } catch (IOException e) { + throw new RuntimeException("Failed to load spec: " + path, e); + } + }); + } + return specs; + } + + private static boolean isEnabledForJava(LlmSpanSpec spec) { + if (spec.enabledRunners() == null) { + return true; // no filter means all runners + } + return spec.enabledRunners().contains("java"); + } + + /** Load a YAML file and expand it into one {@link LlmSpanSpec} per supported client. */ + static List load(Path path) throws IOException { + Yaml yaml = buildYaml(); + try (InputStream is = Files.newInputStream(path)) { + @SuppressWarnings("unchecked") + Map raw = yaml.load(is); + // Build a representative spec first to check enabled_runners + String provider = (String) raw.get("provider"); + LlmSpanSpec representative = LlmSpanSpec.fromMap(raw, path.toString(), provider); + if (!isEnabledForJava(representative)) { + return List.of(); + } + return clientsForProvider(provider).stream() + .filter(client -> clientSupports(client, representative)) + .map(client -> LlmSpanSpec.fromMap(raw, path.toString(), client)) + .toList(); + } + } + + /** + * Returns true if the given client can execute the given spec. Used to filter out client/spec + * combinations that are unsupported — e.g. {@code langchain-openai} only supports basic chat + * completions, not tools, streaming, or the Responses API. + */ + @SuppressWarnings("unchecked") + private static boolean clientSupports(String client, LlmSpanSpec spec) { + if (!"langchain-openai".equals(client)) return true; + // langchain-openai only supports /v1/chat/completions with plain messages + if (!"/v1/chat/completions".equals(spec.endpoint())) return false; + for (Map req : spec.requests()) { + if (req.containsKey("tools")) return false; + if (req.containsKey("stream")) return false; + if (req.containsKey("stream_options")) return false; + } + return true; + } + + private static Yaml buildYaml() { + LoaderOptions opts = new LoaderOptions(); + opts.setTagInspector(tag -> true); // allow all custom tags + return new Yaml(new SpecConstructor(opts)); + } + + /** SnakeYAML Constructor that handles the custom spec tags. */ + private static class SpecConstructor extends Constructor { + SpecConstructor(LoaderOptions opts) { + super(Object.class, opts); + // !fn → FnMatcher(name) + yamlConstructors.put( + new Tag("!fn"), + new AbstractConstruct() { + @Override + public Object construct(Node node) { + String name = ((ScalarNode) node).getValue().trim(); + return new SpecMatcher.FnMatcher(name); + } + }); + // !starts_with "prefix" → StartsWithMatcher(prefix) + yamlConstructors.put( + new Tag("!starts_with"), + new AbstractConstruct() { + @Override + public Object construct(Node node) { + String prefix = ((ScalarNode) node).getValue().trim(); + // strip surrounding quotes if present + if (prefix.startsWith("\"") && prefix.endsWith("\"")) { + prefix = prefix.substring(1, prefix.length() - 1); + } + return new SpecMatcher.StartsWithMatcher(prefix); + } + }); + // !or [...] → OrMatcher(alternatives) + yamlConstructors.put( + new Tag("!or"), + new AbstractConstruct() { + @Override + public Object construct(Node node) { + SequenceNode seq = (SequenceNode) node; + List alternatives = new ArrayList<>(); + for (Node item : seq.getValue()) { + alternatives.add(constructObject(item)); + } + return new SpecMatcher.OrMatcher(alternatives); + } + }); + } + } +} diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java new file mode 100644 index 00000000..ab22e04d --- /dev/null +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/SpecMatcher.java @@ -0,0 +1,42 @@ +package dev.braintrust.sdkspecimpl; + +import java.util.List; + +/** + * Marker interface for custom YAML matchers used in expected_brainstore_spans assertions. + * + *

These correspond to the custom tags in the spec YAML files: + * + *

    + *
  • {@code !fn } - a named predicate (e.g. {@code is_non_negative_number}) + *
  • {@code !starts_with "prefix"} - asserts the value starts with a string + *
  • {@code !or [...]} - asserts the value matches at least one of a list of structures + *
+ */ +public interface SpecMatcher { + + /** + * A named predicate matcher. The name maps to one of the well-known predicate functions defined + * in the spec. + * + *

Supported names: + * + *

    + *
  • {@code is_non_negative_number} - value is a number >= 0 + *
  • {@code is_non_empty_string} - value is a non-empty string + *
  • {@code is_reasoning_message} - value is a non-null object (reasoning summary) + *
  • Any string starting with {@code lambda} - treated as a loose "truthy" check (we can't + * evaluate Python lambdas; we just assert the value is non-null/non-empty) + *
+ */ + record FnMatcher(String name) implements SpecMatcher {} + + /** Asserts the string value starts with the given prefix. */ + record StartsWithMatcher(String prefix) implements SpecMatcher {} + + /** + * Asserts the value matches at least one of the given alternatives. Each alternative is a plain + * Map/List structure (or another SpecMatcher). + */ + record OrMatcher(List alternatives) implements SpecMatcher {} +} diff --git a/settings.gradle b/settings.gradle index 97cccff0..fe81a65e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -23,3 +23,4 @@ include 'braintrust-java-agent:smoke-test:plain-java' include 'braintrust-java-agent:smoke-test:spring-boot' include 'braintrust-java-agent:smoke-test:tomcat' include 'braintrust-java-agent:smoke-test:wildfly' +include 'btx' diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json new file mode 100644 index 00000000..9be41f8f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json @@ -0,0 +1 @@ +{"model":"claude-haiku-4-5-20251001","id":"msg_017JUUeXKS2Merf4Sdm7yLNY","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json new file mode 100644 index 00000000..e3354304 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json @@ -0,0 +1,53 @@ +{ + "id" : "975e911d-181f-4a76-b9b2-7b055fb990b3", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"max_tokens\":128,\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"claude-haiku-4-5-20251001\",\"system\":\"You are a helpful assistant.\",\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-975e911d-181f-4a76-b9b2-7b055fb990b3.json", + "headers" : { + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "x-envoy-upstream-service-time" : "397", + "Server" : "cloudflare", + "vary" : "Accept-Encoding", + "anthropic-ratelimit-output-tokens-limit" : "800000", + "anthropic-ratelimit-output-tokens-reset" : "2026-03-27T00:59:03Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-03-27T00:59:03Z", + "anthropic-ratelimit-tokens-remaining" : "4800000", + "set-cookie" : "_cfuvid=vUGwaHeInAjF9XaMXJNZF8SjyHnfhyq0vJkmWqHq8rc-1774573143.0167425-1.0.1.1-4qfQSPrFsD7CCQYZEOUNjCMwMx1gIaUwUUYv8rL0VWA; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com", + "anthropic-ratelimit-requests-limit" : "4000", + "Content-Security-Policy" : "default-src 'none'; frame-ancestors 'none'", + "server-timing" : "x-originResponse;dur=399", + "anthropic-ratelimit-input-tokens-remaining" : "4000000", + "anthropic-ratelimit-requests-remaining" : "3999", + "Content-Type" : "application/json", + "CF-RAY" : "9e2a733fdf158399-SEA", + "anthropic-ratelimit-tokens-limit" : "4800000", + "cf-cache-status" : "DYNAMIC", + "request-id" : "req_011CZSgK9p1qTXYPnSmaDhUJ", + "anthropic-ratelimit-tokens-reset" : "2026-03-27T00:59:03Z", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Fri, 27 Mar 2026 00:59:03 GMT", + "X-Robots-Tag" : "none", + "anthropic-ratelimit-requests-reset" : "2026-03-27T00:59:03Z", + "anthropic-ratelimit-input-tokens-limit" : "4000000", + "anthropic-ratelimit-output-tokens-remaining" : "800000" + } + }, + "uuid" : "975e911d-181f-4a76-b9b2-7b055fb990b3", + "persistent" : true, + "insertionIndex" : 28 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json new file mode 100644 index 00000000..64d7ccd1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733646249033728","_xact_id":"1000196890678959441","audit_data":[{"_xact_id":"1000196890678959441","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:59:02.797Z","error":null,"expected":null,"facets":null,"id":"25f297da6a309119","input":[{"content":"What is the capital of France?","role":"user"},{"content":"You are a helpful assistant.","role":"system"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"claude-haiku-4-5-20251001","provider":"anthropic","request_base_uri":"","request_method":"POST","request_path":"v1/messages"},"metrics":{"completion_tokens":10,"end":1774573143.6300735,"prompt_tokens":20,"start":1774573142.7973282,"tokens":30},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":{"content":[{"text":"The capital of France is Paris.","type":"text"}],"id":"msg_017JUUeXKS2Merf4Sdm7yLNY","model":"claude-haiku-4-5-20251001","role":"assistant","stop_reason":"end_turn","stop_sequence":null,"type":"message","usage":{"cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"inference_geo":"not_available","input_tokens":20,"output_tokens":10,"service_tier":"standard"}},"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"d704f670032ab520baa58221f7f68f82","scores":null,"span_attributes":{"name":"anthropic.messages.create","type":"llm"},"span_id":"25f297da6a309119","span_parents":["54550a2b4b56e4e3"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890673142544","read_bytes":18439,"actual_xact_id":"1000196890678959441"},"freshness_state":{"last_processed_xact_id":"1000196890678959441","last_considered_xact_id":"1000196890678959441"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json new file mode 100644 index 00000000..17b94a3b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730689","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:58:58.846Z","error":null,"expected":null,"facets":null,"id":"57b73ed4d0f982c6","input":[{"content":"What is the weather like in Paris, France?","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"chat/completions"},"metrics":{"completion_tokens":16,"end":1774573139.5776687,"prompt_tokens":85,"start":1774573138.8462217,"tokens":101},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"finish_reason":"tool_calls","index":0,"logprobs":null,"message":{"annotations":[],"content":null,"refusal":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\"location\":\"Paris, France\"}","name":"get_weather"},"id":"call_7NeJ41zjpe5DRSpmrOVJvllk","type":"function"}]}}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"602b93faa2ba60738844c10818c6df8a","scores":null,"span_attributes":{"name":"Chat Completion","type":"llm"},"span_id":"57b73ed4d0f982c6","span_parents":["73466e7e434e1704"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890681701356","read_bytes":0,"actual_xact_id":"1000196890681701356"},"freshness_state":{"last_processed_xact_id":"1000196890681701356","last_considered_xact_id":"1000196890681701356"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5735ca27-504a-49e2-b510-b7de1bb16679.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5735ca27-504a-49e2-b510-b7de1bb16679.json new file mode 100644 index 00000000..e42ecc0d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5735ca27-504a-49e2-b510-b7de1bb16679.json @@ -0,0 +1 @@ +{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"exhausted_timeout","timeout_ms":500,"minimum_xact_id":"1000196890673142544","actual_xact_id":null},"freshness_state":{"last_processed_xact_id":"1000196890678959441","last_considered_xact_id":"1000196890673142544"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json new file mode 100644 index 00000000..90604b5f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730695","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:58:59.582Z","error":null,"expected":null,"facets":null,"id":"b3d999e0863ea84a","input":[{"content":"you are a thoughtful assistant","role":"system"},{"content":"Count from 1 to 10 slowly.","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"chat/completions"},"metrics":{"completion_tokens":40,"end":1774573142.9773357,"prompt_tokens":25,"start":1774573139.5821326,"time_to_first_token":0.004661458,"tokens":65},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"Sure! Here we go:\n\n1... \n2... \n3... \n4... \n5... \n6... \n7... \n8... \n9... \n10... \n\nTake your time!","refusal":null,"role":"assistant","tool_calls":[],"valid":true},"valid":true}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"a072e43365ff7f6043deb16ca4f5d3bc","scores":null,"span_attributes":{"name":"Chat Completion","type":"llm"},"span_id":"b3d999e0863ea84a","span_parents":["151afc244d427842"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890681701356","read_bytes":0,"actual_xact_id":"1000196890681701356"},"freshness_state":{"last_processed_xact_id":"1000196890681701356","last_considered_xact_id":"1000196890681701356"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json new file mode 100644 index 00000000..53cfaacb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730691","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:58:58.697Z","error":null,"expected":null,"facets":null,"id":"db24b792d29d758d","input":[{"content":"you are a helpful assistant","role":"system"},{"content":"[{type=text, text=What color is this image?}, {type=image_url, image_url={url=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==}}]","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"chat/completions"},"metrics":{"completion_tokens":38,"end":1774573140.2251115,"prompt_tokens":98,"start":1774573138.6977134,"tokens":136},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"annotations":[],"content":"The image you provided is a small, solid color image that appears to be a single shade of light blue. If you have any other questions or need further assistance, feel free to ask!","refusal":null,"role":"assistant"}}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"0bbdc534c257ca8a03e41a97dbb43cef","scores":null,"span_attributes":{"name":"Chat Completion","type":"llm"},"span_id":"db24b792d29d758d","span_parents":["7aae4a6690a3df78"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890673142544","read_bytes":23034,"actual_xact_id":"1000196890679629165"},"freshness_state":{"last_processed_xact_id":"1000196890678959441","last_considered_xact_id":"1000196890679629165"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json new file mode 100644 index 00000000..8c5fe20c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730692","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:59:00.236Z","error":null,"expected":null,"facets":null,"id":"c91333a23c41678b","input":[{"content":"you are a helpful assistant","role":"system"},{"content":[{"text":"What color is this image?","type":"text"},{"image_url":{"url":{"content_type":"image/png","filename":"file.png","key":"5db5dae3-5d0c-427c-9122-ec5c67868f09","type":"braintrust_attachment"}},"type":"image_url"}],"role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"chat/completions"},"metrics":{"completion_tokens":5,"end":1774573140.908804,"prompt_tokens":8522,"start":1774573140.236163,"tokens":8527},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"annotations":[],"content":"The image is red.","refusal":null,"role":"assistant"}}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"72c28ff867824fdb878ae9ce78c60c64","scores":null,"span_attributes":{"name":"Chat Completion","type":"llm"},"span_id":"c91333a23c41678b","span_parents":["f6e86f36e270bb59"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890673142544","read_bytes":23034,"actual_xact_id":"1000196890679629165"},"freshness_state":{"last_processed_xact_id":"1000196890678959441","last_considered_xact_id":"1000196890679629165"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json new file mode 100644 index 00000000..ae674bda --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730694","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:59:01.956Z","error":null,"expected":null,"facets":null,"id":"80ffb8ff30a393b8","input":{"config":{"temperature":0},"contents":[{"parts":[{"text":"What color is this image?"},{"image_url":{"url":{"content_type":"image/png","filename":"file.png","key":"b385947d-32cb-4559-b064-951669d7caaa","type":"braintrust_attachment"}}}],"role":"user"}],"model":"gemini-2.0-flash"},"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gemini-2.0-flash","provider":"gemini","temperature":0},"metrics":{"completion_tokens":5,"end":1774573142.7825167,"prompt_tokens":264,"start":1774573141.9561956,"tokens":269},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":{"candidates":[{"avgLogprobs":-0.05232394933700561,"content":{"parts":[{"text":"The image is red."}],"role":"model"},"finishReason":"STOP"}],"modelVersion":"gemini-2.0-flash","responseId":"VtbFae__DM-eqtsP9YODmA4","usageMetadata":{"candidatesTokenCount":5,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":5}],"promptTokenCount":264,"promptTokensDetails":[{"modality":"TEXT","tokenCount":6},{"modality":"IMAGE","tokenCount":258}],"totalTokenCount":269}},"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"f4f8152cf8a605b15c176979087c59a9","scores":null,"span_attributes":{"name":"generate_content","type":"llm"},"span_id":"80ffb8ff30a393b8","span_parents":["0746850979881fac"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890673142544","read_bytes":18439,"actual_xact_id":"1000196890678959441"},"freshness_state":{"last_processed_xact_id":"1000196890678959441","last_considered_xact_id":"1000196890678959441"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json new file mode 100644 index 00000000..8b3b5b6d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730688","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:58:58.697Z","error":null,"expected":null,"facets":null,"id":"f04f7c001877c104","input":[{"content":"you are a helpful assistant","role":"system"},{"content":"What is the capital of France?","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"chat/completions"},"metrics":{"completion_tokens":7,"end":1774573139.531713,"prompt_tokens":23,"start":1774573138.697783,"tokens":30},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"annotations":[],"content":"The capital of France is Paris.","refusal":null,"role":"assistant"}}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"c08458843dbf11bb36bb950075bcf9da","scores":null,"span_attributes":{"name":"Chat Completion","type":"llm"},"span_id":"f04f7c001877c104","span_parents":["e2cbb4180bb2e8a1"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890681300776","read_bytes":19589,"actual_xact_id":"1000196890681701356"},"freshness_state":{"last_processed_xact_id":"1000196890681701356","last_considered_xact_id":"1000196890681701356"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json new file mode 100644 index 00000000..eb8e23a9 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730690","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:58:59.554Z","error":null,"expected":null,"facets":null,"id":"87628e36e67d3108","input":[{"content":"you are a helpful assistant","role":"system"},{"content":"What is the capital of France?","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gpt-4o-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"chat/completions"},"metrics":{"completion_tokens":7,"end":1774573140.0722687,"prompt_tokens":23,"start":1774573139.5549815,"tokens":30},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"finish_reason":"stop","index":0,"logprobs":null,"message":{"annotations":[],"content":"The capital of France is Paris.","refusal":null,"role":"assistant"}}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"e78ebf34d05fa1a08d226553630aa1fe","scores":null,"span_attributes":{"name":"Chat Completion","type":"llm"},"span_id":"87628e36e67d3108","span_parents":["5bf11cc8b1ef0373"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890681300776","read_bytes":19589,"actual_xact_id":"1000196890681701356"},"freshness_state":{"last_processed_xact_id":"1000196890681701356","last_considered_xact_id":"1000196890681701356"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json new file mode 100644 index 00000000..c57d6b6f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733646249033729","_xact_id":"1000196890678959441","audit_data":[{"_xact_id":"1000196890678959441","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:59:00.086Z","error":null,"expected":null,"facets":null,"id":"8af4f9a72a1447db","input":[{"content":"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\n","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"o4-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"responses"},"metrics":{"completion_reasoning_tokens":704,"completion_tokens":918,"end":1774573149.2210836,"prompt_tokens":41,"start":1774573140.0861723,"tokens":959},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"id":"rs_07d340e4c7661be80069c5d654a830819081489fe3d0bb4302","summary":[{"text":"**Analyzing the sequence**\n\nThe user is asking about the sequence: 2, 6, 12, 20, 30, which are known as pronic or oblong numbers. Each term can be expressed with the formula n(n+1). Looking closely at the pattern, I notice that the differences between the terms increase by 2: 4, 6, 8, 10, and so on. So, for the nth term, I can confirm that a_n = n(n+1), or alternatively, a_n = n^2 + n, starting from n = 1.","type":"summary_text"},{"text":"**Summarizing the sequence**\n\nThe sequence is indexed from 1, where the nth term can be represented as a_n = n(n + 1). This pattern involves multiplying consecutive integers. The differences between terms—like 2 to 6 (+4), 6 to 12 (+6)—form an arithmetic progression starting at 4, with increments of 2. Essentially, this represents the sum of the first n even numbers. Therefore, the nth term formula is a_n = n(n + 1), which characterizes pronic numbers.","type":"summary_text"}],"type":"reasoning"},{"content":[{"annotations":[],"logprobs":[],"text":"The terms are \n2, 6, 12, 20, 30, … \n\nIf you look at the differences you get \n6–2=4, 12–6=6, 20–12=8, 30–20=10,… \nso you’re adding 4,6,8,10,… (i.e. the even numbers from 4 onward). \n\nEquivalently each term is the product of two consecutive integers: \n2=1·2, 6=2·3, 12=3·4, 20=4·5, 30=5·6,…\n\nHence the nth term (with n starting at 1) is \naₙ = n·(n+1) = n² + n.","type":"output_text"}],"id":"msg_07d340e4c7661be80069c5d65c9a0481908630c25ce29d3682","role":"assistant","status":"completed","type":"message"}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"424192964361b1dd759f53fe94864dfb","scores":null,"span_attributes":{"name":"responses","type":"llm"},"span_id":"8af4f9a72a1447db","span_parents":["15f967df70414905"],"tags":null},{"_async_scoring_state":null,"_pagination_key":"p07621733690140065792","_xact_id":"1000196890679629165","audit_data":[{"_xact_id":"1000196890679629165","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:59:09.236Z","error":null,"expected":null,"facets":null,"id":"04c8f2e27fdcc978","input":[{"content":"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\n","role":"user"},{"id":"rs_07d340e4c7661be80069c5d654a830819081489fe3d0bb4302","summary":[{"text":"**Analyzing the sequence**\n\nThe user is asking about the sequence: 2, 6, 12, 20, 30, which are known as pronic or oblong numbers. Each term can be expressed with the formula n(n+1). Looking closely at the pattern, I notice that the differences between the terms increase by 2: 4, 6, 8, 10, and so on. So, for the nth term, I can confirm that a_n = n(n+1), or alternatively, a_n = n^2 + n, starting from n = 1.","type":"summary_text"},{"text":"**Summarizing the sequence**\n\nThe sequence is indexed from 1, where the nth term can be represented as a_n = n(n + 1). This pattern involves multiplying consecutive integers. The differences between terms—like 2 to 6 (+4), 6 to 12 (+6)—form an arithmetic progression starting at 4, with increments of 2. Essentially, this represents the sum of the first n even numbers. Therefore, the nth term formula is a_n = n(n + 1), which characterizes pronic numbers.","type":"summary_text"}],"type":"reasoning"},{"content":[{"annotations":[],"logprobs":[],"text":"The terms are \n2, 6, 12, 20, 30, … \n\nIf you look at the differences you get \n6–2=4, 12–6=6, 20–12=8, 30–20=10,… \nso you’re adding 4,6,8,10,… (i.e. the even numbers from 4 onward). \n\nEquivalently each term is the product of two consecutive integers: \n2=1·2, 6=2·3, 12=3·4, 20=4·5, 30=5·6,…\n\nHence the nth term (with n starting at 1) is \naₙ = n·(n+1) = n² + n.","type":"output_text"}],"id":"msg_07d340e4c7661be80069c5d65c9a0481908630c25ce29d3682","role":"assistant","status":"completed","type":"message"},{"content":"Using the pattern you discovered, what would be the 10th term? And can you find the sum of the first 10 terms?","role":"user"}],"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"o4-mini","provider":"openai","request_base_uri":"http://localhost:53156","request_method":"POST","request_path":"responses"},"metrics":{"completion_reasoning_tokens":448,"completion_tokens":654,"end":1774573159.4095945,"prompt_tokens":241,"start":1774573149.2362018,"tokens":895},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":[{"id":"rs_07d340e4c7661be80069c5d65df04c8190bbc0af168a2cd996","summary":[{"text":"**Calculating terms and sums**\n\nThe user has asked about the sequence 2, 6, 12, 20, and 30, where the nth term is defined as a_n = n(n+1). For the 10th term, I calculate 10 * 11, which gives me 110. Now, to find the sum of the first 10 terms, I use the formula S_n = n(n+1)(n+2)/3. When n=10, this results in 10 * 11 * 12 / 3 = 440. So, the answers are the 10th term is 110 and the sum is 440!","type":"summary_text"},{"text":"**Finalizing results**\n\nFor the sum of the first 10 terms, I calculated S_10 = 10 * 11 * 12 / 3, which gives me 440. Therefore, the 10th term in the sequence is 110, and the total sum of the first 10 terms is 440. I could present these results in words and might also derive the sum formula for clarity. That's it! Everything seems to add up neatly.","type":"summary_text"}],"type":"reasoning"},{"content":[{"annotations":[],"logprobs":[],"text":"The nth term is \naₙ = n·(n+1). \n\nSo the 10th term is \na₁₀ = 10·11 = 110. \n\nFor the sum of the first n terms, \nSₙ = ∑_{k=1}^n k(k+1) \n = ∑ k² + ∑ k \n = [n(n+1)(2n+1)/6] + [n(n+1)/2] \n = n(n+1)(2n+1 + 3)/6 \n = n(n+1)(n+2)/3. \n\nHence \nS₁₀ = 10·11·12 / 3 = 440.","type":"output_text"}],"id":"msg_07d340e4c7661be80069c5d666c728819097330d1881281fb8","role":"assistant","status":"completed","type":"message"}],"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"424192964361b1dd759f53fe94864dfb","scores":null,"span_attributes":{"name":"responses","type":"llm"},"span_id":"04c8f2e27fdcc978","span_parents":["15f967df70414905"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890681701356","read_bytes":0,"actual_xact_id":"1000196890681701356"},"freshness_state":{"last_processed_xact_id":"1000196890681701356","last_considered_xact_id":"1000196890681701356"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json new file mode 100644 index 00000000..be45eb3b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json @@ -0,0 +1 @@ +{"data":[{"_async_scoring_state":null,"_pagination_key":"p07621733619951730693","_xact_id":"1000196890678558176","audit_data":[{"_xact_id":"1000196890678558176","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":null,"created":"2026-03-27T00:59:00.936Z","error":null,"expected":null,"facets":null,"id":"2f698be90d0ef5cd","input":{"config":{"temperature":0},"contents":[{"parts":[{"text":"What is the capital of France?"}],"role":"user"}],"model":"gemini-2.5-flash"},"is_root":false,"log_id":"g","metadata":{"braintrust.parent":"project_name:java-unit-test","model":"gemini-2.5-flash","provider":"gemini","temperature":0},"metrics":{"completion_tokens":8,"end":1774573141.9190414,"prompt_tokens":8,"start":1774573140.9365528,"tokens":37},"org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"output":{"candidates":[{"content":{"parts":[{"text":"The capital of France is **Paris**."}],"role":"model"},"finishReason":"STOP","index":0}],"modelVersion":"gemini-2.5-flash","responseId":"VdbFaaePDsizqtsPmJKAmQI","usageMetadata":{"candidatesTokenCount":8,"promptTokenCount":8,"promptTokensDetails":[{"modality":"TEXT","tokenCount":8}],"thoughtsTokenCount":21,"totalTokenCount":37}},"project_id":"6ae68365-7620-4630-921b-bac416634fc8","root_span_id":"72e6251738693193fbbba3b605aa05ca","scores":null,"span_attributes":{"name":"generate_content","type":"llm"},"span_id":"2f698be90d0ef5cd","span_parents":["d8622d7bcc64ae56"],"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you"},"_xact_id":{"type":"string","description":"The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter)"},"_pagination_key":{"type":["string","null"],"description":"A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore."},"created":{"type":"string","format":"date-time","description":"The timestamp the project logs event was created"},"org_id":{"type":"string","format":"uuid","description":"Unique id for the organization that the project belongs under"},"project_id":{"type":"string","format":"uuid","description":"Unique identifier for the project"},"log_id":{"type":"string","const":"g","description":"A literal 'g' which identifies the log as a project log"},"input":{"description":"The arguments that uniquely define a user input (an arbitrary, JSON serializable object)."},"output":{"description":"The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question."},"expected":{"description":"The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models."},"error":{"description":"The error that occurred, if any."},"scores":{"anyOf":[{"type":"object","additionalProperties":{"anyOf":[{"type":"number","minimum":0,"maximum":1},{"type":"null"}]},"properties":{}},{"type":"null"}]},"metadata":{"anyOf":[{"type":"object","properties":{"model":{"description":"The model used for this example","type":["string","null"]}},"additionalProperties":{}},{"type":"null"}]},"tags":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"metrics":{"anyOf":[{"type":"object","properties":{"start":{"description":"A unix timestamp recording when the section of code which produced the project logs event started","type":["number","null"]},"end":{"description":"A unix timestamp recording when the section of code which produced the project logs event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"caller_functionname":{"description":"This metric is deprecated"},"caller_filename":{"description":"This metric is deprecated"},"caller_lineno":{"description":"This metric is deprecated"}},"additionalProperties":{"type":"number"}},{"type":"null"}]},"context":{"anyOf":[{"type":"object","properties":{"caller_functionname":{"description":"The function in code which created the project logs event","type":["string","null"]},"caller_filename":{"description":"Name of the file in code where the project logs event was created","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"additionalProperties":{}},{"type":"null"}]},"span_id":{"type":"string","description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing"},"span_parents":{"anyOf":[{"type":"array","items":{"type":"string"}},{"type":"null"}]},"root_span_id":{"type":"string","description":"A unique identifier for the trace this project logs event belongs to"},"is_root":{"type":["boolean","null"],"description":"Whether this span is a root span"},"span_attributes":{"anyOf":[{"type":"object","properties":{"name":{"description":"Name of the span, for display purposes only","type":["string","null"]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]}},"additionalProperties":{},"description":"Human-identifying attributes of the span, such as name, type, etc."},{"type":"null"}]},"origin":{"anyOf":[{"type":"object","properties":{"object_type":{"description":"Type of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"},"object_id":{"description":"ID of the object the event is originating from.","type":"string","format":"uuid"},"id":{"description":"ID of the original event.","type":"string"},"_xact_id":{"description":"Transaction ID of the original event.","type":["string","null"]},"created":{"description":"Created timestamp of the original event. Used to help sort in the UI","type":["string","null"]}},"required":["object_type","object_id","id"],"description":"Reference to the original object and event this was copied from."},{"type":"null"}]},"comments":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"audit_data":{"anyOf":[{"type":"array","items":{}},{"type":"null"}]},"_async_scoring_state":{},"facets":{"anyOf":[{"type":"object","additionalProperties":{},"properties":{}},{"type":"null"}]},"classifications":{"anyOf":[{"type":"object","additionalProperties":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string","description":"Stable classification identifier"},"label":{"type":"string","description":"Original label of the classification item, which is useful for search and indexing purposes"},"confidence":{"type":["number","null"],"description":"Optional confidence score for the classification"},"metadata":{"anyOf":[{"type":"object","additionalProperties":{}},{"type":"null"}],"description":"Optional metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"type":"object","properties":{"type":{"type":"string","const":"function"},"id":{"type":"string"},"version":{"type":"string","description":"The version of the function"}},"required":["type","id"],"additionalProperties":false},{"type":"object","properties":{"type":{"type":"string","const":"global"},"name":{"type":"string"},"function_type":{"type":"string","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"default":"scorer","description":"The type of global function. Defaults to 'scorer'."}},"required":["type","name"],"additionalProperties":false}]},{"type":"null"}],"description":"Optional function identifier that produced the classification"}},"required":["id"],"additionalProperties":false}},"properties":{}},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000196890673142544","read_bytes":23034,"actual_xact_id":"1000196890679629165"},"freshness_state":{"last_processed_xact_id":"1000196890678959441","last_considered_xact_id":"1000196890679629165"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json new file mode 100644 index 00000000..f5f263bd --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json @@ -0,0 +1,42 @@ +{ + "id" : "1278a8a8-ca57-4fb7-9165-87d8d14c6c80", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"d704f670032ab520baa58221f7f68f82\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-1278a8a8-ca57-4fb7-9165-87d8d14c6c80.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wPEaBIAMEonQ=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "447", + "X-Amzn-Trace-Id" : "Root=1-69c5d667-60d4b90e7f25284816e6718e;Parent=5b21c4c70331037d;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "458", + "Date" : "Fri, 27 Mar 2026 00:59:20 GMT", + "Via" : "1.1 59e4792b9d6184bfa491a317b36590d2.cloudfront.net (CloudFront), 1.1 73b0c4a85645a8031ba157e0b3e28ffc.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d667000000004cda8322c2e33416", + "x-amzn-RequestId" : "75ae0eba-3026-4e82-a4a4-26a1b58f66b9", + "X-Amz-Cf-Id" : "lhbB0uFiLB9o3Pvbq6OyG080joylF9Dn_t6qApnQ2o7VhdY-v5NArQ==", + "Content-Type" : "application/json" + } + }, + "uuid" : "1278a8a8-ca57-4fb7-9165-87d8d14c6c80", + "persistent" : true, + "insertionIndex" : 155 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json new file mode 100644 index 00000000..600bc4b8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json @@ -0,0 +1,42 @@ +{ + "id" : "3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"602b93faa2ba60738844c10818c6df8a\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a252HGqooAMElZQ=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "234", + "X-Amzn-Trace-Id" : "Root=1-69c5d68d-5c63b91c5b7df2ec413dc2a0;Parent=4dc2c527d61486a5;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "248", + "Date" : "Fri, 27 Mar 2026 00:59:57 GMT", + "Via" : "1.1 b47176981720d8607d309e56e9510316.cloudfront.net (CloudFront), 1.1 4ac8d091dce10e726cfc5404bfed72b8.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d68d0000000064258fd1d3766fce", + "x-amzn-RequestId" : "0a7baa2b-909e-4f02-903b-31db91353a9b", + "X-Amz-Cf-Id" : "iqB_Jfh-3nbj_eZTEBA6M38JxAppWz3mpznPRr-bHwfmEGh0r9kvWw==", + "Content-Type" : "application/json" + } + }, + "uuid" : "3d5c63b6-4d8e-43f2-8b5d-7dcd428edad0", + "persistent" : true, + "insertionIndex" : 144 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5735ca27-504a-49e2-b510-b7de1bb16679.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5735ca27-504a-49e2-b510-b7de1bb16679.json new file mode 100644 index 00000000..12756417 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5735ca27-504a-49e2-b510-b7de1bb16679.json @@ -0,0 +1,45 @@ +{ + "id" : "5735ca27-504a-49e2-b510-b7de1bb16679", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"e78ebf34d05fa1a08d226553630aa1fe\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-5735ca27-504a-49e2-b510-b7de1bb16679.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wrEnxoAMEonQ=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "2128", + "X-Amzn-Trace-Id" : "Root=1-69c5d66a-2b8f8b6e7a934ef6515a47ed;Parent=25134711a532e2d3;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "2142", + "Date" : "Fri, 27 Mar 2026 00:59:24 GMT", + "Via" : "1.1 c9f68a0c96944962731456040c591f26.cloudfront.net (CloudFront), 1.1 4ac8d091dce10e726cfc5404bfed72b8.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d66a000000002b27cb513d5b8c70", + "x-amzn-RequestId" : "282494a8-94c3-47f9-9975-f26a020f7c2f", + "X-Amz-Cf-Id" : "dz_BgPCvPx9bb6Ayw-V679cdGv7g9JrsZ2qzLxNDR9it2Xzd-ya4Iw==", + "Content-Type" : "application/json" + } + }, + "uuid" : "5735ca27-504a-49e2-b510-b7de1bb16679", + "persistent" : true, + "scenarioName" : "scenario-1-btql", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-btql-2", + "insertionIndex" : 149 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json new file mode 100644 index 00000000..8b51648c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json @@ -0,0 +1,42 @@ +{ + "id" : "5814bec2-915f-48ed-988c-eb32ad7ea3e6", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"a072e43365ff7f6043deb16ca4f5d3bc\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-5814bec2-915f-48ed-988c-eb32ad7ea3e6.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a252CGJBoAMEilw=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "231", + "X-Amzn-Trace-Id" : "Root=1-69c5d68c-4a25d3985d17cbb1257e76d0;Parent=6c94bd56bce7239b;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "243", + "Date" : "Fri, 27 Mar 2026 00:59:56 GMT", + "Via" : "1.1 79a7455da856598d6db0b6edabec6574.cloudfront.net (CloudFront), 1.1 83d24992402f7b214901ab76fbdc11e2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d68c000000003365720aa8388c1d", + "x-amzn-RequestId" : "45f2587b-3386-410c-b63f-95699e6f04ee", + "X-Amz-Cf-Id" : "Rggw3LF5vLBT46xBDXJ-5P92RkY0rBnD6IxJ0OSR7ECIzPYOQme_jA==", + "Content-Type" : "application/json" + } + }, + "uuid" : "5814bec2-915f-48ed-988c-eb32ad7ea3e6", + "persistent" : true, + "insertionIndex" : 145 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json new file mode 100644 index 00000000..c401359b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json @@ -0,0 +1,42 @@ +{ + "id" : "5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"0bbdc534c257ca8a03e41a97dbb43cef\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wmGNroAMEMwg=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "255", + "X-Amzn-Trace-Id" : "Root=1-69c5d669-71a59dbd26dafa0e24dbc7dd;Parent=2216d63335197d40;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "266", + "Date" : "Fri, 27 Mar 2026 00:59:22 GMT", + "Via" : "1.1 59e4792b9d6184bfa491a317b36590d2.cloudfront.net (CloudFront), 1.1 0df7f27a01014ab815259ca2d88193c6.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d66900000000449922d31820a388", + "x-amzn-RequestId" : "789c13e6-d218-4613-905d-94f53116c349", + "X-Amz-Cf-Id" : "WmtGeqw8HmLG1fzu-0PBiJYbabskEUXc47HjNlc4LpgTY9SM9778SQ==", + "Content-Type" : "application/json" + } + }, + "uuid" : "5e4e1861-c5ab-48b9-b2c7-38d9d223a7f1", + "persistent" : true, + "insertionIndex" : 150 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json new file mode 100644 index 00000000..46a73277 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json @@ -0,0 +1,42 @@ +{ + "id" : "66a158e4-3f4e-407d-a3de-81d87935e2ca", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"72c28ff867824fdb878ae9ce78c60c64\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-66a158e4-3f4e-407d-a3de-81d87935e2ca.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wjFa0oAMEcWA=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "101", + "X-Amzn-Trace-Id" : "Root=1-69c5d669-51368ae676af441220d3a646;Parent=255b4cbb88349686;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "115", + "Date" : "Fri, 27 Mar 2026 00:59:21 GMT", + "Via" : "1.1 2582fcc2e5a8f93a556ac3ef26738abc.cloudfront.net (CloudFront), 1.1 ffe9646b2ea911744e2d51fc0715cedc.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d6690000000068172b642716b34b", + "x-amzn-RequestId" : "9228ca98-d2a1-4edc-820e-ffb1ee40fe03", + "X-Amz-Cf-Id" : "QgVa7MVEMBpM2nuF8TvKgzNt4Ej7oUFgVfp7c4KhS3M9glWXTNZrhw==", + "Content-Type" : "application/json" + } + }, + "uuid" : "66a158e4-3f4e-407d-a3de-81d87935e2ca", + "persistent" : true, + "insertionIndex" : 151 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json new file mode 100644 index 00000000..dee4320e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json @@ -0,0 +1,42 @@ +{ + "id" : "73a8916b-504b-46c6-90e2-2c7b93850eaf", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"f4f8152cf8a605b15c176979087c59a9\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-73a8916b-504b-46c6-90e2-2c7b93850eaf.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wXHcTIAMEegA=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "311", + "X-Amzn-Trace-Id" : "Root=1-69c5d668-0b3efe9d0b60740e0d576dd3;Parent=3a9aad0c89d6eaa7;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "323", + "Date" : "Fri, 27 Mar 2026 00:59:20 GMT", + "Via" : "1.1 79a7455da856598d6db0b6edabec6574.cloudfront.net (CloudFront), 1.1 83d24992402f7b214901ab76fbdc11e2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d6680000000074058f89de53edc8", + "x-amzn-RequestId" : "a22b6603-4036-46ab-a5f1-3294ee9fb740", + "X-Amz-Cf-Id" : "iCyZo8FR2NYiYRbkr5L76GIaBqoDAqUMCs0_BFj6Vy2_47GMHJajWQ==", + "Content-Type" : "application/json" + } + }, + "uuid" : "73a8916b-504b-46c6-90e2-2c7b93850eaf", + "persistent" : true, + "insertionIndex" : 154 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json new file mode 100644 index 00000000..f34af627 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json @@ -0,0 +1,42 @@ +{ + "id" : "846a96a6-6e1b-43af-8a2b-5a95416bf52c", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"c08458843dbf11bb36bb950075bcf9da\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-846a96a6-6e1b-43af-8a2b-5a95416bf52c.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a2512EDYIAMEt0g=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "292", + "X-Amzn-Trace-Id" : "Root=1-69c5d68b-69860f2e6c0b09b55f1606d8;Parent=0f7c265505c5cd4a;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "306", + "Date" : "Fri, 27 Mar 2026 00:59:55 GMT", + "Via" : "1.1 b7e07d6a19a4c8b2e410e9c1e173548c.cloudfront.net (CloudFront), 1.1 d525041695bdb6325f78ebba5c11b8a2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d68b0000000051fcf109c6ee29d9", + "x-amzn-RequestId" : "1666498f-11ab-41fd-a9eb-1cf0588adcd0", + "X-Amz-Cf-Id" : "j-b_Mumr-MFhe_YHA8n0P1bsRef9fm64GnVVjTNqa5obrT36rW4J8A==", + "Content-Type" : "application/json" + } + }, + "uuid" : "846a96a6-6e1b-43af-8a2b-5a95416bf52c", + "persistent" : true, + "insertionIndex" : 147 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json new file mode 100644 index 00000000..36bb4a8f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json @@ -0,0 +1,44 @@ +{ + "id" : "b36c1c17-c103-4979-ace7-8fbf8f1d5033", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"e78ebf34d05fa1a08d226553630aa1fe\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-b36c1c17-c103-4979-ace7-8fbf8f1d5033.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a251xFCTIAMELPw=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "315", + "X-Amzn-Trace-Id" : "Root=1-69c5d68a-4142abc75835e66c52fc2d35;Parent=13f3c56d97102906;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "327", + "Date" : "Fri, 27 Mar 2026 00:59:55 GMT", + "Via" : "1.1 940972e9e344075576fe20d5db482122.cloudfront.net (CloudFront), 1.1 65f2e9f7f1475de54aa452d3ceb9bcf6.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d68a0000000058fd0a0dadc9f67d", + "x-amzn-RequestId" : "ceed9702-b58f-4fbf-aa51-c0aba3d4030a", + "X-Amz-Cf-Id" : "hczUX_TCdVQB8XlB14PEN2qp9e_g-YBuuIX8cJ5hfnSUZYgbTvnCPA==", + "Content-Type" : "application/json" + } + }, + "uuid" : "b36c1c17-c103-4979-ace7-8fbf8f1d5033", + "persistent" : true, + "scenarioName" : "scenario-1-btql", + "requiredScenarioState" : "scenario-1-btql-2", + "insertionIndex" : 148 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json new file mode 100644 index 00000000..f2a375bf --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json @@ -0,0 +1,42 @@ +{ + "id" : "b55cd92c-7080-4b39-bf6b-25fe99e3c0e2", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"424192964361b1dd759f53fe94864dfb\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-b55cd92c-7080-4b39-bf6b-25fe99e3c0e2.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a2519EvBoAMEm0Q=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "217", + "X-Amzn-Trace-Id" : "Root=1-69c5d68c-61c961e144fe45f810fe1d77;Parent=090d6c75c8735d3c;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "234", + "Date" : "Fri, 27 Mar 2026 00:59:56 GMT", + "Via" : "1.1 79a7455da856598d6db0b6edabec6574.cloudfront.net (CloudFront), 1.1 65f2e9f7f1475de54aa452d3ceb9bcf6.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d68c000000004fb1bb480b7087b4", + "x-amzn-RequestId" : "88f8020f-6418-49f2-9b92-649f0d331da8", + "X-Amz-Cf-Id" : "HSipLL-dBPAybgfzTZmHzPaxXtw8NeJiYfJ2_nRxdWWV0DsGWdk9Fg==", + "Content-Type" : "application/json" + } + }, + "uuid" : "b55cd92c-7080-4b39-bf6b-25fe99e3c0e2", + "persistent" : true, + "insertionIndex" : 146 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json new file mode 100644 index 00000000..1ae903ea --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json @@ -0,0 +1,42 @@ +{ + "id" : "b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":{\"select\":[{\"op\":\"star\"}],\"from\":{\"name\":{\"name\":[\"project_logs\"],\"op\":\"ident\"},\"args\":[{\"value\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"op\":\"literal\"}],\"op\":\"function\"},\"filter\":{\"right\":{\"right\":{\"op\":\"literal\",\"value\":null},\"op\":\"ne\",\"left\":{\"name\":[\"span_parents\"],\"op\":\"ident\"}},\"op\":\"and\",\"left\":{\"right\":{\"value\":\"72e6251738693193fbbba3b605aa05ca\",\"op\":\"literal\"},\"op\":\"eq\",\"left\":{\"name\":[\"root_span_id\"],\"op\":\"ident\"}}},\"sort\":[{\"dir\":\"asc\",\"expr\":{\"name\":[\"created\"],\"op\":\"ident\"}}],\"limit\":1000},\"use_columnstore\":true,\"use_brainstore\":true,\"brainstore_realtime\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wdEqYIAMEWIw=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "348", + "X-Amzn-Trace-Id" : "Root=1-69c5d668-48383eb64c0ad8385c67d4f3;Parent=7a88a351999b507d;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "359", + "Date" : "Fri, 27 Mar 2026 00:59:21 GMT", + "Via" : "1.1 2582fcc2e5a8f93a556ac3ef26738abc.cloudfront.net (CloudFront), 1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d669000000000c794ef77b63175c", + "x-amzn-RequestId" : "de8f5bcd-1414-44a5-8d0c-b3243fbbed81", + "X-Amz-Cf-Id" : "VogwSispsQ1IN-td7uikuf_PUs1TZ_ov_sN3Ou47ZxGAyct-hPinUQ==", + "Content-Type" : "application/json" + } + }, + "uuid" : "b9ab8ed3-32ad-4900-a1d3-4b9257a75a3d", + "persistent" : true, + "insertionIndex" : 152 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e29fb36-3138-4598-a90e-fe0c742e653a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e29fb36-3138-4598-a90e-fe0c742e653a.json new file mode 100644 index 00000000..8ada99cc --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e29fb36-3138-4598-a90e-fe0c742e653a.json @@ -0,0 +1,39 @@ +{ + "id" : "2e29fb36-3138-4598-a90e-fe0c742e653a", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "Cs4dCrkBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAopCg9zZXJ2aWNlLnZlcnNpb24SFgoUMC4yLjEwLTk0Nzg5NWYtRElSVFkKIAoWdGVsZW1ldHJ5LnNkay5sYW5ndWFnZRIGCgRqYXZhCiUKEnRlbGVtZXRyeS5zZGsubmFtZRIPCg1vcGVudGVsZW1ldHJ5CiEKFXRlbGVtZXRyeS5zZGsudmVyc2lvbhIICgYxLjU5LjAS6xoKEQoPYnJhaW50cnVzdC1qYXZhEsEIChDXBPZwAyq1ILqlgiH39o+CEggl8pfaajCRGSIIVFUKK0tW5OMqGWFudGhyb3BpYy5tZXNzYWdlcy5jcmVhdGUwATlQH2IM34ygGEEZzwQ+34ygGEouChpicmFpbnRydXN0LnNwYW5fYXR0cmlidXRlcxIQCg57InR5cGUiOiJsbG0ifUoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKkgEKFWJyYWludHJ1c3QuaW5wdXRfanNvbhJ5CndbeyJjb250ZW50IjoiV2hhdCBpcyB0aGUgY2FwaXRhbCBvZiBGcmFuY2U/Iiwicm9sZSI6InVzZXIifSx7InJvbGUiOiJzeXN0ZW0iLCJjb250ZW50IjoiWW91IGFyZSBhIGhlbHBmdWwgYXNzaXN0YW50LiJ9XUpPChJicmFpbnRydXN0Lm1ldHJpY3MSOQo3eyJjb21wbGV0aW9uX3Rva2VucyI6MTAsInByb21wdF90b2tlbnMiOjIwLCJ0b2tlbnMiOjMwfUqiAQoTYnJhaW50cnVzdC5tZXRhZGF0YRKKAQqHAXsicHJvdmlkZXIiOiJhbnRocm9waWMiLCJyZXF1ZXN0X3BhdGgiOiJ2MS9tZXNzYWdlcyIsIm1vZGVsIjoiY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMSIsInJlcXVlc3RfYmFzZV91cmkiOiIiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUryAwoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhLXAwrUA3sibW9kZWwiOiJjbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxIiwiaWQiOiJtc2dfMDE3SlVVZVhLUzJNZXJmNFNkbTd5TE5ZIiwidHlwZSI6Im1lc3NhZ2UiLCJyb2xlIjoiYXNzaXN0YW50IiwiY29udGVudCI6W3sidHlwZSI6InRleHQiLCJ0ZXh0IjoiVGhlIGNhcGl0YWwgb2YgRnJhbmNlIGlzIFBhcmlzLiJ9XSwic3RvcF9yZWFzb24iOiJlbmRfdHVybiIsInN0b3Bfc2VxdWVuY2UiOm51bGwsInVzYWdlIjp7ImlucHV0X3Rva2VucyI6MjAsImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MCwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjAsImNhY2hlX2NyZWF0aW9uIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjAsImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9LCJvdXRwdXRfdG9rZW5zIjoxMCwic2VydmljZV90aWVyIjoic3RhbmRhcmQiLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSJ9fXoAhQEBAQAAEpESChBCQZKWQ2Gx3XWfU/6Uhk37EgiK9PmnKhRH2yIIFfln33BBSQUqCXJlc3BvbnNlczABOVQryWrejKAYQT3aRIvgjKAYSi4KGmJyYWludHJ1c3Quc3Bhbl9hdHRyaWJ1dGVzEhAKDnsidHlwZSI6ImxsbSJ9SjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEqpAQoVYnJhaW50cnVzdC5pbnB1dF9qc29uEo8BCowBW3siY29udGVudCI6Ikxvb2sgYXQgdGhpcyBzZXF1ZW5jZTogMiwgNiwgMTIsIDIwLCAzMC4gV2hhdCBpcyB0aGUgcGF0dGVybiBhbmQgd2hhdCB3b3VsZCBiZSB0aGUgZm9ybXVsYSBmb3IgdGhlIG50aCB0ZXJtP1xuIiwicm9sZSI6InVzZXIifV1KcwoSYnJhaW50cnVzdC5tZXRyaWNzEl0KW3siY29tcGxldGlvbl90b2tlbnMiOjkxOCwicHJvbXB0X3Rva2VucyI6NDEsInRva2VucyI6OTU5LCJjb21wbGV0aW9uX3JlYXNvbmluZ190b2tlbnMiOjcwNH1KoQEKE2JyYWludHJ1c3QubWV0YWRhdGESiQEKhgF7InByb3ZpZGVyIjoib3BlbmFpIiwicmVxdWVzdF9wYXRoIjoicmVzcG9uc2VzIiwibW9kZWwiOiJvNC1taW5pIiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHA6Ly9sb2NhbGhvc3Q6NTMxNTYiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqYDQoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhL9DAr6DFt7ImlkIjoicnNfMDdkMzQwZTRjNzY2MWJlODAwNjljNWQ2NTRhODMwODE5MDgxNDg5ZmUzZDBiYjQzMDIiLCJ0eXBlIjoicmVhc29uaW5nIiwic3VtbWFyeSI6W3sidHlwZSI6InN1bW1hcnlfdGV4dCIsInRleHQiOiIqKkFuYWx5emluZyB0aGUgc2VxdWVuY2UqKlxuXG5UaGUgdXNlciBpcyBhc2tpbmcgYWJvdXQgdGhlIHNlcXVlbmNlOiAyLCA2LCAxMiwgMjAsIDMwLCB3aGljaCBhcmUga25vd24gYXMgcHJvbmljIG9yIG9ibG9uZyBudW1iZXJzLiBFYWNoIHRlcm0gY2FuIGJlIGV4cHJlc3NlZCB3aXRoIHRoZSBmb3JtdWxhIG4obisxKS4gTG9va2luZyBjbG9zZWx5IGF0IHRoZSBwYXR0ZXJuLCBJIG5vdGljZSB0aGF0IHRoZSBkaWZmZXJlbmNlcyBiZXR3ZWVuIHRoZSB0ZXJtcyBpbmNyZWFzZSBieSAyOiA0LCA2LCA4LCAxMCwgYW5kIHNvIG9uLiBTbywgZm9yIHRoZSBudGggdGVybSwgSSBjYW4gY29uZmlybSB0aGF0IGFfbiA9IG4obisxKSwgb3IgYWx0ZXJuYXRpdmVseSwgYV9uID0gbl4yICsgbiwgc3RhcnRpbmcgZnJvbSBuID0gMS4ifSx7InR5cGUiOiJzdW1tYXJ5X3RleHQiLCJ0ZXh0IjoiKipTdW1tYXJpemluZyB0aGUgc2VxdWVuY2UqKlxuXG5UaGUgc2VxdWVuY2UgaXMgaW5kZXhlZCBmcm9tIDEsIHdoZXJlIHRoZSBudGggdGVybSBjYW4gYmUgcmVwcmVzZW50ZWQgYXMgYV9uID0gbihuICsgMSkuIFRoaXMgcGF0dGVybiBpbnZvbHZlcyBtdWx0aXBseWluZyBjb25zZWN1dGl2ZSBpbnRlZ2Vycy4gVGhlIGRpZmZlcmVuY2VzIGJldHdlZW4gdGVybXPigJRsaWtlIDIgdG8gNiAoKzQpLCA2IHRvIDEyICgrNinigJRmb3JtIGFuIGFyaXRobWV0aWMgcHJvZ3Jlc3Npb24gc3RhcnRpbmcgYXQgNCwgd2l0aCBpbmNyZW1lbnRzIG9mIDIuIEVzc2VudGlhbGx5LCB0aGlzIHJlcHJlc2VudHMgdGhlIHN1bSBvZiB0aGUgZmlyc3QgbiBldmVuIG51bWJlcnMuIFRoZXJlZm9yZSwgdGhlIG50aCB0ZXJtIGZvcm11bGEgaXMgYV9uID0gbihuICsgMSksIHdoaWNoIGNoYXJhY3Rlcml6ZXMgcHJvbmljIG51bWJlcnMuIn1dfSx7ImlkIjoibXNnXzA3ZDM0MGU0Yzc2NjFiZTgwMDY5YzVkNjVjOWEwNDgxOTA4NjMwYzI1Y2UyOWQzNjgyIiwidHlwZSI6Im1lc3NhZ2UiLCJzdGF0dXMiOiJjb21wbGV0ZWQiLCJjb250ZW50IjpbeyJ0eXBlIjoib3V0cHV0X3RleHQiLCJhbm5vdGF0aW9ucyI6W10sImxvZ3Byb2JzIjpbXSwidGV4dCI6IlRoZSB0ZXJtcyBhcmUgIFxuMizigIk2LOKAiTEyLOKAiTIwLOKAiTMwLCDigKYgIFxuXG5JZiB5b3UgbG9vayBhdCB0aGUgZGlmZmVyZW5jZXMgeW91IGdldCAgXG424oCTMj00LOKAgjEy4oCTNj02LOKAgjIw4oCTMTI9OCzigIIzMOKAkzIwPTEwLOKApiAgXG5zbyB5b3XigJlyZSBhZGRpbmcgNCw2LDgsMTAs4oCmIChpLmUuIHRoZSBldmVuIG51bWJlcnMgZnJvbSA0IG9ud2FyZCkuICBcblxuRXF1aXZhbGVudGx5IGVhY2ggdGVybSBpcyB0aGUgcHJvZHVjdCBvZiB0d28gY29uc2VjdXRpdmUgaW50ZWdlcnM6ICBcbjI9McK3MizigII2PTLCtzMs4oCCMTI9M8K3NCzigIIyMD00wrc1LOKAgjMwPTXCtzYs4oCmXG5cbkhlbmNlIHRoZSBudGggdGVybSAod2l0aCBuIHN0YXJ0aW5nIGF0IDEpIGlzICBcbmHigpkgPSBuwrcobisxKSA9IG7CsiArIG4uIn1dLCJyb2xlIjoiYXNzaXN0YW50In1degCFAQEBAAASoQEKDwoNc2RrLXNwZWMtaW1wbBKNAQoQ1wT2cAMqtSC6pYIh9/aPghIIVFUKK0tW5OMqCG1lc3NhZ2VzMAE5GG6jC9+MoBhB9xsHPt+MoBhKMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0ShUKBmNsaWVudBILCglhbnRocm9waWN6AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25utGMDIAMENjg=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c5d65d-2387c3dc293cd3770dcaf8f1;Parent=231d05f3586f7325;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Fri, 27 Mar 2026 00:59:10 GMT", + "Via" : "1.1 2582fcc2e5a8f93a556ac3ef26738abc.cloudfront.net (CloudFront), 1.1 170efbc424be9181bda5d0fcd6e41f30.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d65d0000000045564bc9f016a374", + "x-amzn-RequestId" : "5fcfc82d-de50-44f6-aebf-35eeecca54dd", + "X-Amz-Cf-Id" : "YccrUy4gy-fd5lx_4olxzScw4GK2_nSFqsDb0Q9_ci5yrhtGKJOpdQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "2e29fb36-3138-4598-a90e-fe0c742e653a", + "persistent" : true, + "insertionIndex" : 156 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-524370c8-e897-4896-ad3f-dba0c3314f81.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-524370c8-e897-4896-ad3f-dba0c3314f81.json new file mode 100644 index 00000000..a419c4c1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-524370c8-e897-4896-ad3f-dba0c3314f81.json @@ -0,0 +1,39 @@ +{ + "id" : "524370c8-e897-4896-ad3f-dba0c3314f81", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "Cp4hCrkBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAopCg9zZXJ2aWNlLnZlcnNpb24SFgoUMC4yLjEwLTk0Nzg5NWYtRElSVFkKIAoWdGVsZW1ldHJ5LnNkay5sYW5ndWFnZRIGCgRqYXZhCiUKEnRlbGVtZXRyeS5zZGsubmFtZRIPCg1vcGVudGVsZW1ldHJ5CiEKFXRlbGVtZXRyeS5zZGsudmVyc2lvbhIICgYxLjU5LjASvR4KEQoPYnJhaW50cnVzdC1qYXZhEqceChBCQZKWQ2Gx3XWfU/6Uhk37EggEyPLif9zJeCIIFfln33BBSQUqCXJlc3BvbnNlczABOUCJK4zgjKAYQSsxjerijKAYSi4KGmJyYWludHJ1c3Quc3Bhbl9hdHRyaWJ1dGVzEhAKDnsidHlwZSI6ImxsbSJ9SjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEqtDwoVYnJhaW50cnVzdC5pbnB1dF9qc29uEpMPCpAPW3siY29udGVudCI6Ikxvb2sgYXQgdGhpcyBzZXF1ZW5jZTogMiwgNiwgMTIsIDIwLCAzMC4gV2hhdCBpcyB0aGUgcGF0dGVybiBhbmQgd2hhdCB3b3VsZCBiZSB0aGUgZm9ybXVsYSBmb3IgdGhlIG50aCB0ZXJtP1xuIiwicm9sZSI6InVzZXIifSx7ImlkIjoicnNfMDdkMzQwZTRjNzY2MWJlODAwNjljNWQ2NTRhODMwODE5MDgxNDg5ZmUzZDBiYjQzMDIiLCJzdW1tYXJ5IjpbeyJ0ZXh0IjoiKipBbmFseXppbmcgdGhlIHNlcXVlbmNlKipcblxuVGhlIHVzZXIgaXMgYXNraW5nIGFib3V0IHRoZSBzZXF1ZW5jZTogMiwgNiwgMTIsIDIwLCAzMCwgd2hpY2ggYXJlIGtub3duIGFzIHByb25pYyBvciBvYmxvbmcgbnVtYmVycy4gRWFjaCB0ZXJtIGNhbiBiZSBleHByZXNzZWQgd2l0aCB0aGUgZm9ybXVsYSBuKG4rMSkuIExvb2tpbmcgY2xvc2VseSBhdCB0aGUgcGF0dGVybiwgSSBub3RpY2UgdGhhdCB0aGUgZGlmZmVyZW5jZXMgYmV0d2VlbiB0aGUgdGVybXMgaW5jcmVhc2UgYnkgMjogNCwgNiwgOCwgMTAsIGFuZCBzbyBvbi4gU28sIGZvciB0aGUgbnRoIHRlcm0sIEkgY2FuIGNvbmZpcm0gdGhhdCBhX24gPSBuKG4rMSksIG9yIGFsdGVybmF0aXZlbHksIGFfbiA9IG5eMiArIG4sIHN0YXJ0aW5nIGZyb20gbiA9IDEuIiwidHlwZSI6InN1bW1hcnlfdGV4dCJ9LHsidGV4dCI6IioqU3VtbWFyaXppbmcgdGhlIHNlcXVlbmNlKipcblxuVGhlIHNlcXVlbmNlIGlzIGluZGV4ZWQgZnJvbSAxLCB3aGVyZSB0aGUgbnRoIHRlcm0gY2FuIGJlIHJlcHJlc2VudGVkIGFzIGFfbiA9IG4obiArIDEpLiBUaGlzIHBhdHRlcm4gaW52b2x2ZXMgbXVsdGlwbHlpbmcgY29uc2VjdXRpdmUgaW50ZWdlcnMuIFRoZSBkaWZmZXJlbmNlcyBiZXR3ZWVuIHRlcm1z4oCUbGlrZSAyIHRvIDYgKCs0KSwgNiB0byAxMiAoKzYp4oCUZm9ybSBhbiBhcml0aG1ldGljIHByb2dyZXNzaW9uIHN0YXJ0aW5nIGF0IDQsIHdpdGggaW5jcmVtZW50cyBvZiAyLiBFc3NlbnRpYWxseSwgdGhpcyByZXByZXNlbnRzIHRoZSBzdW0gb2YgdGhlIGZpcnN0IG4gZXZlbiBudW1iZXJzLiBUaGVyZWZvcmUsIHRoZSBudGggdGVybSBmb3JtdWxhIGlzIGFfbiA9IG4obiArIDEpLCB3aGljaCBjaGFyYWN0ZXJpemVzIHByb25pYyBudW1iZXJzLiIsInR5cGUiOiJzdW1tYXJ5X3RleHQifV0sInR5cGUiOiJyZWFzb25pbmcifSx7ImlkIjoibXNnXzA3ZDM0MGU0Yzc2NjFiZTgwMDY5YzVkNjVjOWEwNDgxOTA4NjMwYzI1Y2UyOWQzNjgyIiwiY29udGVudCI6W3siYW5ub3RhdGlvbnMiOltdLCJ0ZXh0IjoiVGhlIHRlcm1zIGFyZSAgXG4yLOKAiTYs4oCJMTIs4oCJMjAs4oCJMzAsIOKApiAgXG5cbklmIHlvdSBsb29rIGF0IHRoZSBkaWZmZXJlbmNlcyB5b3UgZ2V0ICBcbjbigJMyPTQs4oCCMTLigJM2PTYs4oCCMjDigJMxMj04LOKAgjMw4oCTMjA9MTAs4oCmICBcbnNvIHlvdeKAmXJlIGFkZGluZyA0LDYsOCwxMCzigKYgKGkuZS4gdGhlIGV2ZW4gbnVtYmVycyBmcm9tIDQgb253YXJkKS4gIFxuXG5FcXVpdmFsZW50bHkgZWFjaCB0ZXJtIGlzIHRoZSBwcm9kdWN0IG9mIHR3byBjb25zZWN1dGl2ZSBpbnRlZ2VyczogIFxuMj0xwrcyLOKAgjY9MsK3MyzigIIxMj0zwrc0LOKAgjIwPTTCtzUs4oCCMzA9NcK3NizigKZcblxuSGVuY2UgdGhlIG50aCB0ZXJtICh3aXRoIG4gc3RhcnRpbmcgYXQgMSkgaXMgIFxuYeKCmSA9IG7CtyhuKzEpID0gbsKyICsgbi4iLCJ0eXBlIjoib3V0cHV0X3RleHQiLCJsb2dwcm9icyI6W119XSwicm9sZSI6ImFzc2lzdGFudCIsInN0YXR1cyI6ImNvbXBsZXRlZCIsInR5cGUiOiJtZXNzYWdlIn0seyJjb250ZW50IjoiVXNpbmcgdGhlIHBhdHRlcm4geW91IGRpc2NvdmVyZWQsIHdoYXQgd291bGQgYmUgdGhlIDEwdGggdGVybT8gQW5kIGNhbiB5b3UgZmluZCB0aGUgc3VtIG9mIHRoZSBmaXJzdCAxMCB0ZXJtcz8iLCJyb2xlIjoidXNlciJ9XUp0ChJicmFpbnRydXN0Lm1ldHJpY3MSXgpceyJjb21wbGV0aW9uX3Rva2VucyI6NjU0LCJwcm9tcHRfdG9rZW5zIjoyNDEsInRva2VucyI6ODk1LCJjb21wbGV0aW9uX3JlYXNvbmluZ190b2tlbnMiOjQ0OH1KoQEKE2JyYWludHJ1c3QubWV0YWRhdGESiQEKhgF7InByb3ZpZGVyIjoib3BlbmFpIiwicmVxdWVzdF9wYXRoIjoicmVzcG9uc2VzIiwibW9kZWwiOiJvNC1taW5pIiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHA6Ly9sb2NhbGhvc3Q6NTMxNTYiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqpCwoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhKOCwqLC1t7ImlkIjoicnNfMDdkMzQwZTRjNzY2MWJlODAwNjljNWQ2NWRmMDRjODE5MGJiYzBhZjE2OGEyY2Q5OTYiLCJ0eXBlIjoicmVhc29uaW5nIiwic3VtbWFyeSI6W3sidHlwZSI6InN1bW1hcnlfdGV4dCIsInRleHQiOiIqKkNhbGN1bGF0aW5nIHRlcm1zIGFuZCBzdW1zKipcblxuVGhlIHVzZXIgaGFzIGFza2VkIGFib3V0IHRoZSBzZXF1ZW5jZSAyLCA2LCAxMiwgMjAsIGFuZCAzMCwgd2hlcmUgdGhlIG50aCB0ZXJtIGlzIGRlZmluZWQgYXMgYV9uID0gbihuKzEpLiBGb3IgdGhlIDEwdGggdGVybSwgSSBjYWxjdWxhdGUgMTAgKiAxMSwgd2hpY2ggZ2l2ZXMgbWUgMTEwLiBOb3csIHRvIGZpbmQgdGhlIHN1bSBvZiB0aGUgZmlyc3QgMTAgdGVybXMsIEkgdXNlIHRoZSBmb3JtdWxhIFNfbiA9IG4obisxKShuKzIpLzMuIFdoZW4gbj0xMCwgdGhpcyByZXN1bHRzIGluIDEwICogMTEgKiAxMiAvIDMgPSA0NDAuIFNvLCB0aGUgYW5zd2VycyBhcmUgdGhlIDEwdGggdGVybSBpcyAxMTAgYW5kIHRoZSBzdW0gaXMgNDQwISJ9LHsidHlwZSI6InN1bW1hcnlfdGV4dCIsInRleHQiOiIqKkZpbmFsaXppbmcgcmVzdWx0cyoqXG5cbkZvciB0aGUgc3VtIG9mIHRoZSBmaXJzdCAxMCB0ZXJtcywgSSBjYWxjdWxhdGVkIFNfMTAgPSAxMCAqIDExICogMTIgLyAzLCB3aGljaCBnaXZlcyBtZSA0NDAuIFRoZXJlZm9yZSwgdGhlIDEwdGggdGVybSBpbiB0aGUgc2VxdWVuY2UgaXMgMTEwLCBhbmQgdGhlIHRvdGFsIHN1bSBvZiB0aGUgZmlyc3QgMTAgdGVybXMgaXMgNDQwLiBJIGNvdWxkIHByZXNlbnQgdGhlc2UgcmVzdWx0cyBpbiB3b3JkcyBhbmQgbWlnaHQgYWxzbyBkZXJpdmUgdGhlIHN1bSBmb3JtdWxhIGZvciBjbGFyaXR5LiBUaGF0J3MgaXQhIEV2ZXJ5dGhpbmcgc2VlbXMgdG8gYWRkIHVwIG5lYXRseS4ifV19LHsiaWQiOiJtc2dfMDdkMzQwZTRjNzY2MWJlODAwNjljNWQ2NjZjNzI4ODE5MDk3MzMwZDE4ODEyODFmYjgiLCJ0eXBlIjoibWVzc2FnZSIsInN0YXR1cyI6ImNvbXBsZXRlZCIsImNvbnRlbnQiOlt7InR5cGUiOiJvdXRwdXRfdGV4dCIsImFubm90YXRpb25zIjpbXSwibG9ncHJvYnMiOltdLCJ0ZXh0IjoiVGhlIG50aCB0ZXJtIGlzICBcbmHigpkgPSBuwrcobisxKS4gIFxuXG5TbyB0aGUgMTB0aCB0ZXJtIGlzICBcbmHigoHigoAgPSAxMMK3MTEgPSAxMTAuICBcblxuRm9yIHRoZSBzdW0gb2YgdGhlIGZpcnN0IG4gdGVybXMsICBcblPigpkgPSDiiJFfe2s9MX1ebiBrKGsrMSkgIFxuICAgPSDiiJEga8KyICsg4oiRIGsgIFxuICAgPSBbbihuKzEpKDJuKzEpLzZdICsgW24obisxKS8yXSAgXG4gICA9IG4obisxKSgybisxICsgMykvNiAgXG4gICA9IG4obisxKShuKzIpLzMuICBcblxuSGVuY2UgIFxuU+KCgeKCgCA9IDEwwrcxMcK3MTIgLyAzID0gNDQwLiJ9XSwicm9sZSI6ImFzc2lzdGFudCJ9XXoAhQEBAQAAEp8BCg8KDXNkay1zcGVjLWltcGwSiwEKEEJBkpZDYbHddZ9T/pSGTfsSCBX5Z99wQUkFKglyZWFzb25pbmcwATmgpf5p3oygGEH5UHTr4oygGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKEgoGY2xpZW50EggKBm9wZW5haXoAhQEBAQAA" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25wZG9ZoAMEY9w=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c5d668-675107351d644412378e1978;Parent=6d29ca53e8a642ae;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Fri, 27 Mar 2026 00:59:20 GMT", + "Via" : "1.1 d5e9313fa5148ebdba4664d3e2a90f58.cloudfront.net (CloudFront), 1.1 f8731007efc5ab360d90cee573a1e916.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d6680000000033e24f683106b0ca", + "x-amzn-RequestId" : "7c63e7e5-b763-4d34-92b6-463969ce589e", + "X-Amz-Cf-Id" : "p2RWtuG23UI9vUUssqciMDhSY_mEfBRY62v70Dj-GqUq_P2Lr1EvcA==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "524370c8-e897-4896-ad3f-dba0c3314f81", + "persistent" : true, + "insertionIndex" : 153 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-b9dc43a8-3404-4929-8f65-73cd2b2fe500.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-b9dc43a8-3404-4929-8f65-73cd2b2fe500.json new file mode 100644 index 00000000..6684cffb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-b9dc43a8-3404-4929-8f65-73cd2b2fe500.json @@ -0,0 +1,39 @@ +{ + "id" : "b9dc43a8-3404-4929-8f65-73cd2b2fe500", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CtRFCrkBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAopCg9zZXJ2aWNlLnZlcnNpb24SFgoUMC4yLjEwLTk0Nzg5NWYtRElSVFkKIAoWdGVsZW1ldHJ5LnNkay5sYW5ndWFnZRIGCgRqYXZhCiUKEnRlbGVtZXRyeS5zZGsubmFtZRIPCg1vcGVudGVsZW1ldHJ5CiEKFXRlbGVtZXRyeS5zZGsudmVyc2lvbhIICgYxLjU5LjAS8DoKEQoPYnJhaW50cnVzdC1qYXZhEooGChDAhFiEPb8Ruza7lQB1vPnaEgjwT3wAGHfBBCII4su0GAuy6KEqD0NoYXQgQ29tcGxldGlvbjADOX4HCBjejKAYQejJvEnejKAYSi4KGmJyYWludHJ1c3Quc3Bhbl9hdHRyaWJ1dGVzEhAKDnsidHlwZSI6ImxsbSJ9SjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEqRAQoVYnJhaW50cnVzdC5pbnB1dF9qc29uEngKdlt7InJvbGUiOiJzeXN0ZW0iLCJjb250ZW50IjoieW91IGFyZSBhIGhlbHBmdWwgYXNzaXN0YW50In0seyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTgoSYnJhaW50cnVzdC5tZXRyaWNzEjgKNnsiY29tcGxldGlvbl90b2tlbnMiOjcsInByb21wdF90b2tlbnMiOjIzLCJ0b2tlbnMiOjMwfUqsAQoTYnJhaW50cnVzdC5tZXRhZGF0YRKUAQqRAXsicHJvdmlkZXIiOiJvcGVuYWkiLCJyZXF1ZXN0X3BhdGgiOiJjaGF0L2NvbXBsZXRpb25zIiwibW9kZWwiOiJncHQtNG8tbWluaSIsInJlcXVlc3RfYmFzZV91cmkiOiJodHRwOi8vbG9jYWxob3N0OjUzMTU2IiwicmVxdWVzdF9tZXRob2QiOiJQT1NUIn1KvQEKFmJyYWludHJ1c3Qub3V0cHV0X2pzb24SogEKnwFbeyJpbmRleCI6MCwibWVzc2FnZSI6eyJyb2xlIjoiYXNzaXN0YW50IiwiY29udGVudCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4iLCJyZWZ1c2FsIjpudWxsLCJhbm5vdGF0aW9ucyI6W119LCJsb2dwcm9icyI6bnVsbCwiZmluaXNoX3JlYXNvbiI6InN0b3AifV16AIUBAQEAABLaBgoQYCuT+qK6YHOIRMEIGMbfihIIV7c+1ND5gsYiCHNGbn5DThcEKg9DaGF0IENvbXBsZXRpb24wATnmBOEg3oygGEG+A3pM3oygGEouChpicmFpbnRydXN0LnNwYW5fYXR0cmlidXRlcxIQCg57InR5cGUiOiJsbG0ifUoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKYwoVYnJhaW50cnVzdC5pbnB1dF9qc29uEkoKSFt7ImNvbnRlbnQiOiJXaGF0IGlzIHRoZSB3ZWF0aGVyIGxpa2UgaW4gUGFyaXMsIEZyYW5jZT8iLCJyb2xlIjoidXNlciJ9XUpQChJicmFpbnRydXN0Lm1ldHJpY3MSOgo4eyJjb21wbGV0aW9uX3Rva2VucyI6MTYsInByb21wdF90b2tlbnMiOjg1LCJ0b2tlbnMiOjEwMX1KpwEKE2JyYWludHJ1c3QubWV0YWRhdGESjwEKjAF7InByb3ZpZGVyIjoib3BlbmFpIiwicmVxdWVzdF9wYXRoIjoiY2hhdC9jb21wbGV0aW9ucyIsIm1vZGVsIjoiZ3B0LTRvIiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHA6Ly9sb2NhbGhvc3Q6NTMxNTYiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUq/AgoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhKkAgqhAlt7ImluZGV4IjowLCJtZXNzYWdlIjp7InJvbGUiOiJhc3Npc3RhbnQiLCJjb250ZW50IjpudWxsLCJ0b29sX2NhbGxzIjpbeyJpZCI6ImNhbGxfN05lSjQxempwZTVEUlNwbXJPVkp2bGxrIiwidHlwZSI6ImZ1bmN0aW9uIiwiZnVuY3Rpb24iOnsibmFtZSI6ImdldF93ZWF0aGVyIiwiYXJndW1lbnRzIjoie1wibG9jYXRpb25cIjpcIlBhcmlzLCBGcmFuY2VcIn0ifX1dLCJyZWZ1c2FsIjpudWxsLCJhbm5vdGF0aW9ucyI6W119LCJsb2dwcm9icyI6bnVsbCwiZmluaXNoX3JlYXNvbiI6InRvb2xfY2FsbHMifV16AIUBAQEAABKKBgoQ546/NNBfoaCNImVTYwqh/hIIh2KONuZ9MQgiCFvxHMix7wNzKg9DaGF0IENvbXBsZXRpb24wATms1R9L3oygGEEhBPVp3oygGEouChpicmFpbnRydXN0LnNwYW5fYXR0cmlidXRlcxIQCg57InR5cGUiOiJsbG0ifUoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKkQEKFWJyYWludHJ1c3QuaW5wdXRfanNvbhJ4CnZbeyJjb250ZW50IjoieW91IGFyZSBhIGhlbHBmdWwgYXNzaXN0YW50Iiwicm9sZSI6InN5c3RlbSJ9LHsiY29udGVudCI6IldoYXQgaXMgdGhlIGNhcGl0YWwgb2YgRnJhbmNlPyIsInJvbGUiOiJ1c2VyIn1dSk4KEmJyYWludHJ1c3QubWV0cmljcxI4CjZ7ImNvbXBsZXRpb25fdG9rZW5zIjo3LCJwcm9tcHRfdG9rZW5zIjoyMywidG9rZW5zIjozMH1KrAEKE2JyYWludHJ1c3QubWV0YWRhdGESlAEKkQF7InByb3ZpZGVyIjoib3BlbmFpIiwicmVxdWVzdF9wYXRoIjoiY2hhdC9jb21wbGV0aW9ucyIsIm1vZGVsIjoiZ3B0LTRvLW1pbmkiLCJyZXF1ZXN0X2Jhc2VfdXJpIjoiaHR0cDovL2xvY2FsaG9zdDo1MzE1NiIsInJlcXVlc3RfbWV0aG9kIjoiUE9TVCJ9Sr0BChZicmFpbnRydXN0Lm91dHB1dF9qc29uEqIBCp8BW3siaW5kZXgiOjAsIm1lc3NhZ2UiOnsicm9sZSI6ImFzc2lzdGFudCIsImNvbnRlbnQiOiJUaGUgY2FwaXRhbCBvZiBGcmFuY2UgaXMgUGFyaXMuIiwicmVmdXNhbCI6bnVsbCwiYW5ub3RhdGlvbnMiOltdfSwibG9ncHJvYnMiOm51bGwsImZpbmlzaF9yZWFzb24iOiJzdG9wIn1degCFAQEBAAASywgKEAu9xTTCV8qKA+Qal9u0PO8SCNskt5LSnXWNIgh6rkpmkKPfeCoPQ2hhdCBDb21wbGV0aW9uMAM5i/YGGN6MoBhBojURc96MoBhKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SrwCChVicmFpbnRydXN0LmlucHV0X2pzb24SogIKnwJbeyJyb2xlIjoic3lzdGVtIiwiY29udGVudCI6InlvdSBhcmUgYSBoZWxwZnVsIGFzc2lzdGFudCJ9LHsicm9sZSI6InVzZXIiLCJjb250ZW50IjoiW3t0eXBlPXRleHQsIHRleHQ9V2hhdCBjb2xvciBpcyB0aGlzIGltYWdlP30sIHt0eXBlPWltYWdlX3VybCwgaW1hZ2VfdXJsPXt1cmw9ZGF0YTppbWFnZS9wbmc7YmFzZTY0LGlWQk9SdzBLR2dvQUFBQU5TVWhFVWdBQUFBRUFBQUFCQ0FZQUFBQWZGY1NKQUFBQURVbEVRVlI0Mm1QOHo4RHdId0FGQlFJQVg4angwZ0FBQUFCSlJVNUVya0pnZ2c9PX19XSJ9XUpQChJicmFpbnRydXN0Lm1ldHJpY3MSOgo4eyJjb21wbGV0aW9uX3Rva2VucyI6MzgsInByb21wdF90b2tlbnMiOjk4LCJ0b2tlbnMiOjEzNn1KrAEKE2JyYWludHJ1c3QubWV0YWRhdGESlAEKkQF7InByb3ZpZGVyIjoib3BlbmFpIiwicmVxdWVzdF9wYXRoIjoiY2hhdC9jb21wbGV0aW9ucyIsIm1vZGVsIjoiZ3B0LTRvLW1pbmkiLCJyZXF1ZXN0X2Jhc2VfdXJpIjoiaHR0cDovL2xvY2FsaG9zdDo1MzE1NiIsInJlcXVlc3RfbWV0aG9kIjoiUE9TVCJ9StECChZicmFpbnRydXN0Lm91dHB1dF9qc29uErYCCrMCW3siaW5kZXgiOjAsIm1lc3NhZ2UiOnsicm9sZSI6ImFzc2lzdGFudCIsImNvbnRlbnQiOiJUaGUgaW1hZ2UgeW91IHByb3ZpZGVkIGlzIGEgc21hbGwsIHNvbGlkIGNvbG9yIGltYWdlIHRoYXQgYXBwZWFycyB0byBiZSBhIHNpbmdsZSBzaGFkZSBvZiBsaWdodCBibHVlLiBJZiB5b3UgaGF2ZSBhbnkgb3RoZXIgcXVlc3Rpb25zIG9yIG5lZWQgZnVydGhlciBhc3Npc3RhbmNlLCBmZWVsIGZyZWUgdG8gYXNrISIsInJlZnVzYWwiOm51bGwsImFubm90YXRpb25zIjpbXX0sImxvZ3Byb2JzIjpudWxsLCJmaW5pc2hfcmVhc29uIjoic3RvcCJ9XXoAhQEBAQAAErgHChBywo/4Z4JP24eK6c54xgxkEgjJEzOiPEFniyII9uhvNuJwu1kqD0NoYXQgQ29tcGxldGlvbjABOb7WuXPejKAYQXeH0ZvejKAYSi4KGmJyYWludHJ1c3Quc3Bhbl9hdHRyaWJ1dGVzEhAKDnsidHlwZSI6ImxsbSJ9SjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdErJAgoVYnJhaW50cnVzdC5pbnB1dF9qc29uEq8CCqwCW3siY29udGVudCI6InlvdSBhcmUgYSBoZWxwZnVsIGFzc2lzdGFudCIsInJvbGUiOiJzeXN0ZW0ifSx7ImNvbnRlbnQiOlt7InRleHQiOiJXaGF0IGNvbG9yIGlzIHRoaXMgaW1hZ2U/IiwidHlwZSI6InRleHQifSx7ImltYWdlX3VybCI6eyJ1cmwiOiJkYXRhOmltYWdlL3BuZztiYXNlNjQsaVZCT1J3MEtHZ29BQUFBTlNVaEVVZ0FBQUFFQUFBQUJDQVlBQUFBZkZjU0pBQUFBRFVsRVFWUjQybVA4ejhEd0h3QUZCUUlBWDhqeDBnQUFBQUJKUlU1RXJrSmdnZz09In0sInR5cGUiOiJpbWFnZV91cmwifV0sInJvbGUiOiJ1c2VyIn1dSlIKEmJyYWludHJ1c3QubWV0cmljcxI8Cjp7ImNvbXBsZXRpb25fdG9rZW5zIjo1LCJwcm9tcHRfdG9rZW5zIjo4NTIyLCJ0b2tlbnMiOjg1Mjd9SqwBChNicmFpbnRydXN0Lm1ldGFkYXRhEpQBCpEBeyJwcm92aWRlciI6Im9wZW5haSIsInJlcXVlc3RfcGF0aCI6ImNoYXQvY29tcGxldGlvbnMiLCJtb2RlbCI6ImdwdC00by1taW5pIiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHA6Ly9sb2NhbGhvc3Q6NTMxNTYiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqvAQoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhKUAQqRAVt7ImluZGV4IjowLCJtZXNzYWdlIjp7InJvbGUiOiJhc3Npc3RhbnQiLCJjb250ZW50IjoiVGhlIGltYWdlIGlzIHJlZC4iLCJyZWZ1c2FsIjpudWxsLCJhbm5vdGF0aW9ucyI6W119LCJsb2dwcm9icyI6bnVsbCwiZmluaXNoX3JlYXNvbiI6InN0b3AifV16AIUBAQEAABKoBwoQcuYlFzhpMZP7u6O2BaoFyhIIL2mL6Q0O9c0iCNhiLXvMZK5WKhBnZW5lcmF0ZV9jb250ZW50MAM5V/F4nd6MoBhBjoYI2N6MoBhKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SqcBChVicmFpbnRydXN0LmlucHV0X2pzb24SjQEKigF7ImNvbnRlbnRzIjpbeyJwYXJ0cyI6W3sidGV4dCI6IldoYXQgaXMgdGhlIGNhcGl0YWwgb2YgRnJhbmNlPyJ9XSwicm9sZSI6InVzZXIifV0sIm1vZGVsIjoiZ2VtaW5pLTIuNS1mbGFzaCIsImNvbmZpZyI6eyJ0ZW1wZXJhdHVyZSI6MC4wfX1KTQoSYnJhaW50cnVzdC5tZXRyaWNzEjcKNXsiY29tcGxldGlvbl90b2tlbnMiOjgsInByb21wdF90b2tlbnMiOjgsInRva2VucyI6Mzd9SlsKE2JyYWludHJ1c3QubWV0YWRhdGESRApCeyJwcm92aWRlciI6ImdlbWluaSIsInRlbXBlcmF0dXJlIjowLjAsIm1vZGVsIjoiZ2VtaW5pLTIuNS1mbGFzaCJ9SpUDChZicmFpbnRydXN0Lm91dHB1dF9qc29uEvoCCvcCeyJjYW5kaWRhdGVzIjpbeyJjb250ZW50Ijp7InBhcnRzIjpbeyJ0ZXh0IjoiVGhlIGNhcGl0YWwgb2YgRnJhbmNlIGlzICoqUGFyaXMqKi4ifV0sInJvbGUiOiJtb2RlbCJ9LCJmaW5pc2hSZWFzb24iOiJTVE9QIiwiaW5kZXgiOjB9XSwidXNhZ2VNZXRhZGF0YSI6eyJwcm9tcHRUb2tlbkNvdW50Ijo4LCJjYW5kaWRhdGVzVG9rZW5Db3VudCI6OCwidG90YWxUb2tlbkNvdW50IjozNywicHJvbXB0VG9rZW5zRGV0YWlscyI6W3sibW9kYWxpdHkiOiJURVhUIiwidG9rZW5Db3VudCI6OH1dLCJ0aG91Z2h0c1Rva2VuQ291bnQiOjIxfSwibW9kZWxWZXJzaW9uIjoiZ2VtaW5pLTIuNS1mbGFzaCIsInJlc3BvbnNlSWQiOiJWZGJGYWFlUERzaXpxdHNQbUpLQW1RSSJ9egIYAYUBAQEAABKPCQoQ9PgVLPimBbFcF2l5CHxZqRIIgP+4/zCjk7giCAdGhQl5iB+sKhBnZW5lcmF0ZV9jb250ZW50MAM5rHQ/2t6MoBhB5B2AC9+MoBhKLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SrQCChVicmFpbnRydXN0LmlucHV0X2pzb24SmgIKlwJ7ImNvbnRlbnRzIjpbeyJwYXJ0cyI6W3sidGV4dCI6IldoYXQgY29sb3IgaXMgdGhpcyBpbWFnZT8ifSx7ImlubGluZURhdGEiOnsiZGF0YSI6ImlWQk9SdzBLR2dvQUFBQU5TVWhFVWdBQUFBRUFBQUFCQ0FZQUFBQWZGY1NKQUFBQURVbEVRVlI0Mm1QOHo4RHdId0FGQlFJQVg4angwZ0FBQUFCSlJVNUVya0pnZ2c9PSIsIm1pbWVUeXBlIjoiaW1hZ2UvcG5nIn19XSwicm9sZSI6InVzZXIifV0sIm1vZGVsIjoiZ2VtaW5pLTIuMC1mbGFzaCIsImNvbmZpZyI6eyJ0ZW1wZXJhdHVyZSI6MC4wfX1KUAoSYnJhaW50cnVzdC5tZXRyaWNzEjoKOHsiY29tcGxldGlvbl90b2tlbnMiOjUsInByb21wdF90b2tlbnMiOjI2NCwidG9rZW5zIjoyNjl9SlsKE2JyYWludHJ1c3QubWV0YWRhdGESRApCeyJwcm92aWRlciI6ImdlbWluaSIsInRlbXBlcmF0dXJlIjowLjAsIm1vZGVsIjoiZ2VtaW5pLTIuMC1mbGFzaCJ9SuwDChZicmFpbnRydXN0Lm91dHB1dF9qc29uEtEDCs4DeyJjYW5kaWRhdGVzIjpbeyJjb250ZW50Ijp7InBhcnRzIjpbeyJ0ZXh0IjoiVGhlIGltYWdlIGlzIHJlZC4ifV0sInJvbGUiOiJtb2RlbCJ9LCJmaW5pc2hSZWFzb24iOiJTVE9QIiwiYXZnTG9ncHJvYnMiOi0wLjA1MjMyMzk0OTMzNzAwNTYxfV0sInVzYWdlTWV0YWRhdGEiOnsicHJvbXB0VG9rZW5Db3VudCI6MjY0LCJjYW5kaWRhdGVzVG9rZW5Db3VudCI6NSwidG90YWxUb2tlbkNvdW50IjoyNjksInByb21wdFRva2Vuc0RldGFpbHMiOlt7Im1vZGFsaXR5IjoiVEVYVCIsInRva2VuQ291bnQiOjZ9LHsibW9kYWxpdHkiOiJJTUFHRSIsInRva2VuQ291bnQiOjI1OH1dLCJjYW5kaWRhdGVzVG9rZW5zRGV0YWlscyI6W3sibW9kYWxpdHkiOiJURVhUIiwidG9rZW5Db3VudCI6NX1dfSwibW9kZWxWZXJzaW9uIjoiZ2VtaW5pLTIuMC1mbGFzaCIsInJlc3BvbnNlSWQiOiJWdGJGYWVfX0RNLWVxdHNQOVlPRG1BNCJ9egIYAYUBAQEAABKdBwoQoHLkM2X/f2BD3rFspPXTvBIIs9mZ4IY+qEoiCBUa/CRNQnhCKg9DaGF0IENvbXBsZXRpb24wATmXIL5M3oygGEFD0hwX34ygGEouChpicmFpbnRydXN0LnNwYW5fYXR0cmlidXRlcxIQCg57InR5cGUiOiJsbG0ifUoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKkAEKFWJyYWludHJ1c3QuaW5wdXRfanNvbhJ3CnVbeyJjb250ZW50IjoieW91IGFyZSBhIHRob3VnaHRmdWwgYXNzaXN0YW50Iiwicm9sZSI6InN5c3RlbSJ9LHsiY29udGVudCI6IkNvdW50IGZyb20gMSB0byAxMCBzbG93bHkuIiwicm9sZSI6InVzZXIifV1KcQoSYnJhaW50cnVzdC5tZXRyaWNzElsKWXsiY29tcGxldGlvbl90b2tlbnMiOjQwLCJwcm9tcHRfdG9rZW5zIjoyNSwidG9rZW5zIjo2NSwidGltZV90b19maXJzdF90b2tlbiI6MC4wMDQ2NjE0NTh9SqwBChNicmFpbnRydXN0Lm1ldGFkYXRhEpQBCpEBeyJwcm92aWRlciI6Im9wZW5haSIsInJlcXVlc3RfcGF0aCI6ImNoYXQvY29tcGxldGlvbnMiLCJtb2RlbCI6ImdwdC00by1taW5pIiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHA6Ly9sb2NhbGhvc3Q6NTMxNTYiLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUquAgoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhKTAgqQAlt7ImZpbmlzaF9yZWFzb24iOiJzdG9wIiwiaW5kZXgiOjAsImxvZ3Byb2JzIjpudWxsLCJtZXNzYWdlIjp7ImNvbnRlbnQiOiJTdXJlISBIZXJlIHdlIGdvOlxuXG4xLi4uICBcbjIuLi4gIFxuMy4uLiAgXG40Li4uICBcbjUuLi4gIFxuNi4uLiAgXG43Li4uICBcbjguLi4gIFxuOS4uLiAgXG4xMC4uLiAgXG5cblRha2UgeW91ciB0aW1lISIsInJlZnVzYWwiOm51bGwsInJvbGUiOiJhc3Npc3RhbnQiLCJ0b29sX2NhbGxzIjpbXSwidmFsaWQiOnRydWV9LCJ2YWxpZCI6dHJ1ZX1degCFAQEBAAASogkKDwoNc2RrLXNwZWMtaW1wbBKXAQoQwIRYhD2/Ebs2u5UAdbz52hII4su0GAuy6KEqC2NvbXBsZXRpb25zMAE5yB+wFt6MoBhBA2doSt6MoBhKMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0ShwKBmNsaWVudBISChBsYW5nY2hhaW4tb3BlbmFpegCFAQEBAAAShwEKEGArk/qiumBziETBCBjG34oSCHNGbn5DThcEKgV0b29sczABOcgfsBbejKAYQZw3f0zejKAYSjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEoSCgZjbGllbnQSCAoGb3BlbmFpegCFAQEBAAASjQEKEOeOvzTQX6GgjSJlU2MKof4SCFvxHMix7wNzKgtjb21wbGV0aW9uczABORgHbUrejKAYQabC+WnejKAYSjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEoSCgZjbGllbnQSCAoGb3BlbmFpegCFAQEBAAASlwEKEAu9xTTCV8qKA+Qal9u0PO8SCHquSmaQo994KgthdHRhY2htZW50czABOfBCsBbejKAYQYonFHPejKAYSjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEocCgZjbGllbnQSEgoQbGFuZ2NoYWluLW9wZW5haXoAhQEBAQAAEo0BChBywo/4Z4JP24eK6c54xgxkEgj26G824nC7WSoLYXR0YWNobWVudHMwATmAABVz3oygGEHZpdab3oygGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKEgoGY2xpZW50EggKBm9wZW5haXoAhQEBAQAAEpIBChBy5iUXOGkxk/u7o7YFqgXKEgjYYi17zGSuVioQZ2VuZXJhdGVfY29udGVudDABOeB22ZvejKAYQd1XKdrejKAYSjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEoSCgZjbGllbnQSCAoGZ29vZ2xlegCFAQEBAAASjQEKEPT4FSz4pgWxXBdpeQh8WakSCAdGhQl5iB+sKgthdHRhY2htZW50czABOejoKtrejKAYQSbRoAvfjKAYSjIKEWJyYWludHJ1c3QucGFyZW50Eh0KG3Byb2plY3RfbmFtZTpqYXZhLXVuaXQtdGVzdEoSCgZjbGllbnQSCAoGZ29vZ2xlegCFAQEBAAASiwEKEKBy5DNl/39gQ96xbKT107wSCBUa/CRNQnhCKglzdHJlYW1pbmcwATmoL4BM3oygGEE9zR0X34ygGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3RKEgoGY2xpZW50EggKBm9wZW5haXoAhQEBAQAA" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "a25tzEOqIAMEALQ=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c5d657-7aca19893e8f656e202fd61a;Parent=5a8cfc235bf4e6a1;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Fri, 27 Mar 2026 00:59:04 GMT", + "Via" : "1.1 db84db36e16ca0c80b0992006d731900.cloudfront.net (CloudFront), 1.1 d9d466ed70d93f34739969f91577ec74.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c5d65700000000008fa3e022c64449", + "x-amzn-RequestId" : "12344457-93a5-4093-a56a-4b376d61b045", + "X-Amz-Cf-Id" : "A0PEra2otOOVXpmaBnz5VL4QfDxtn_qhADDTcmrr_t69lk62O5PRRA==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "b9dc43a8-3404-4929-8f65-73cd2b2fe500", + "persistent" : true, + "insertionIndex" : 157 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json b/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json new file mode 100644 index 00000000..5353895e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json @@ -0,0 +1,39 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The image is red." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.052323949337005612 + } + ], + "usageMetadata": { + "promptTokenCount": 264, + "candidatesTokenCount": 5, + "totalTokenCount": 269, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + }, + { + "modality": "IMAGE", + "tokenCount": 258 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 5 + } + ] + }, + "modelVersion": "gemini-2.0-flash", + "responseId": "VtbFae__DM-eqtsP9YODmA4" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json b/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json new file mode 100644 index 00000000..44fb5600 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/google/__files/v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json @@ -0,0 +1,30 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The capital of France is **Paris**." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 8, + "candidatesTokenCount": 8, + "totalTokenCount": 37, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 8 + } + ], + "thoughtsTokenCount": 21 + }, + "modelVersion": "gemini-2.5-flash", + "responseId": "VdbFaaePDsizqtsPmJKAmQI" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json b/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json new file mode 100644 index 00000000..d8b57c34 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json @@ -0,0 +1,37 @@ +{ + "id" : "bdc53e3b-3373-4000-8eca-72e602162a16", + "name" : "v1beta_models_gemini-2.0-flashgeneratecontent", + "request" : { + "url" : "/v1beta/models/gemini-2.0-flash:generateContent", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json; charset=UTF-8" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"What color is this image?\"},{\"inlineData\":{\"data\":\"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==\",\"mimeType\":\"image/png\"}}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1beta_models_gemini-2.0-flashgeneratecontent-bdc53e3b-3373-4000-8eca-72e602162a16.json", + "headers" : { + "X-Frame-Options" : "SAMEORIGIN", + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "scaffolding on HTTPServer2", + "X-Content-Type-Options" : "nosniff", + "Server-Timing" : "gfet4t7; dur=713", + "Vary" : [ "Origin", "X-Origin", "Referer" ], + "X-Gemini-Service-Tier" : "standard", + "X-XSS-Protection" : "0", + "Date" : "Fri, 27 Mar 2026 00:59:02 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "bdc53e3b-3373-4000-8eca-72e602162a16", + "persistent" : true, + "insertionIndex" : 3 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json b/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json new file mode 100644 index 00000000..1ca0b67d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/google/mappings/v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json @@ -0,0 +1,37 @@ +{ + "id" : "408179ff-16f0-4c71-9c25-346c1e21cca2", + "name" : "v1beta_models_gemini-2.5-flashgeneratecontent", + "request" : { + "url" : "/v1beta/models/gemini-2.5-flash:generateContent", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json; charset=UTF-8" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"What is the capital of France?\"}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1beta_models_gemini-2.5-flashgeneratecontent-408179ff-16f0-4c71-9c25-346c1e21cca2.json", + "headers" : { + "X-Frame-Options" : "SAMEORIGIN", + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "scaffolding on HTTPServer2", + "X-Content-Type-Options" : "nosniff", + "Server-Timing" : "gfet4t7; dur=714", + "Vary" : [ "Origin", "X-Origin", "Referer" ], + "X-Gemini-Service-Tier" : "standard", + "X-XSS-Protection" : "0", + "Date" : "Fri, 27 Mar 2026 00:59:01 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "408179ff-16f0-4c71-9c25-346c1e21cca2", + "persistent" : true, + "insertionIndex" : 4 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json new file mode 100644 index 00000000..12e605a2 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-DNppLtNIXQuyWfQhcpeMcwp0DajS3", + "object": "chat.completion", + "created": 1774573139, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 23, + "completion_tokens": 7, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_2e9401d89c" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.txt new file mode 100644 index 00000000..b1923803 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.txt @@ -0,0 +1,88 @@ +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ftLYJvpIL"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"Sure"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"5LVFkST"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"eCGE5fWuMy"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" Here"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"VAyO8P"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" we"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"0jSMotAm"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" go"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"UJrVpClB"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":":\n\n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"7gfq6v"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"1"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"IGfs2DXxye"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"9MpZhamy"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"FDhPmSK"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"2"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"7vcrbsvYRl"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"1NURn7gA"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"03UYaGd"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"3"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"XdsvPC2OEp"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Cg7D3RGR"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"vMq8a7B"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"4"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"fn3cRB9qDV"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"aXrVN7Ny"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Zh2UAzE"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"5"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"j41a4bzmDB"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"9l1JrU10"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"3oPTFk8"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"6"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"R5JzXrwTFv"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"upHgYFvl"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"L0D2BX1"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"7"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"zOoBi3VnVq"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"QnliFkkF"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"yFdl2tE"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"8"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"j5Clu7MrFQ"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ArrIODGx"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"sVwXgWR"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"9"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"IHYEE8hCzA"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"fPP9gkAs"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ZDE54Qt"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"10"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"gURJOdBxw"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"..."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"785Lcqac"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" \n\n"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Qf4kX"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"Take"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"O2CQJ8y"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" your"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"eEWGYS"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":" time"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"o5SfUQ"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"rYGK0vH0bR"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"BYmou"} + +data: {"id":"chatcmpl-DNppLc0DmdxvknPLG1ih4A5PFxfeO","object":"chat.completion.chunk","created":1774573139,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_e738e3044b","choices":[],"usage":{"prompt_tokens":25,"completion_tokens":40,"total_tokens":65,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"XpxgnSVxLy"} + +data: [DONE] + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json new file mode 100644 index 00000000..bf9df5fb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-DNppLlAvDZS4GlDlnoKdLlmwaKpHW", + "object": "chat.completion", + "created": 1774573139, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The image you provided is a small, solid color image that appears to be a single shade of light blue. If you have any other questions or need further assistance, feel free to ask!", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 98, + "completion_tokens": 38, + "total_tokens": 136, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_2e9401d89c" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json new file mode 100644 index 00000000..29a45d51 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json @@ -0,0 +1,46 @@ +{ + "id": "chatcmpl-DNppLjpi2Oa96vjgDDzd8vIcy92E1", + "object": "chat.completion", + "created": 1774573139, + "model": "gpt-4o-2024-08-06", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_7NeJ41zjpe5DRSpmrOVJvllk", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\":\"Paris, France\"}" + } + } + ], + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 85, + "completion_tokens": 16, + "total_tokens": 101, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_ab062d0c27" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json new file mode 100644 index 00000000..9005aaa2 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-DNppMQyvFpZTamJlJw8K4qnoxr1QC", + "object": "chat.completion", + "created": 1774573140, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The image is red.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 8522, + "completion_tokens": 5, + "total_tokens": 8527, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_e738e3044b" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json new file mode 100644 index 00000000..972bc6be --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-DNppLabItURg5em0UyIVrInxboPco", + "object": "chat.completion", + "created": 1774573139, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 23, + "completion_tokens": 7, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_2e9401d89c" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0aac5720-a309-46ee-bb02-5d466bc02763.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0aac5720-a309-46ee-bb02-5d466bc02763.json new file mode 100644 index 00000000..f7fd3651 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-0aac5720-a309-46ee-bb02-5d466bc02763.json @@ -0,0 +1,85 @@ +{ + "id": "resp_07d340e4c7661be80069c5d65d69088190a11cb06cbfa91eaa", + "object": "response", + "created_at": 1774573149, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774573159, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "output": [ + { + "id": "rs_07d340e4c7661be80069c5d65df04c8190bbc0af168a2cd996", + "type": "reasoning", + "summary": [ + { + "type": "summary_text", + "text": "**Calculating terms and sums**\n\nThe user has asked about the sequence 2, 6, 12, 20, and 30, where the nth term is defined as a_n = n(n+1). For the 10th term, I calculate 10 * 11, which gives me 110. Now, to find the sum of the first 10 terms, I use the formula S_n = n(n+1)(n+2)/3. When n=10, this results in 10 * 11 * 12 / 3 = 440. So, the answers are the 10th term is 110 and the sum is 440!" + }, + { + "type": "summary_text", + "text": "**Finalizing results**\n\nFor the sum of the first 10 terms, I calculated S_10 = 10 * 11 * 12 / 3, which gives me 440. Therefore, the 10th term in the sequence is 110, and the total sum of the first 10 terms is 440. I could present these results in words and might also derive the sum formula for clarity. That's it! Everything seems to add up neatly." + } + ] + }, + { + "id": "msg_07d340e4c7661be80069c5d666c728819097330d1881281fb8", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The nth term is \na\u2099 = n\u00b7(n+1). \n\nSo the 10th term is \na\u2081\u2080 = 10\u00b711 = 110. \n\nFor the sum of the first n terms, \nS\u2099 = \u2211_{k=1}^n k(k+1) \n = \u2211 k\u00b2 + \u2211 k \n = [n(n+1)(2n+1)/6] + [n(n+1)/2] \n = n(n+1)(2n+1 + 3)/6 \n = n(n+1)(n+2)/3. \n\nHence \nS\u2081\u2080 = 10\u00b711\u00b712 / 3 = 440." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "high", + "summary": "detailed" + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 241, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 654, + "output_tokens_details": { + "reasoning_tokens": 448 + }, + "total_tokens": 895 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json new file mode 100644 index 00000000..cafbe0e7 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json @@ -0,0 +1,85 @@ +{ + "id": "resp_07d340e4c7661be80069c5d6544eec819087342d5334ba4a3b", + "object": "response", + "created_at": 1774573140, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774573148, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "output": [ + { + "id": "rs_07d340e4c7661be80069c5d654a830819081489fe3d0bb4302", + "type": "reasoning", + "summary": [ + { + "type": "summary_text", + "text": "**Analyzing the sequence**\n\nThe user is asking about the sequence: 2, 6, 12, 20, 30, which are known as pronic or oblong numbers. Each term can be expressed with the formula n(n+1). Looking closely at the pattern, I notice that the differences between the terms increase by 2: 4, 6, 8, 10, and so on. So, for the nth term, I can confirm that a_n = n(n+1), or alternatively, a_n = n^2 + n, starting from n = 1." + }, + { + "type": "summary_text", + "text": "**Summarizing the sequence**\n\nThe sequence is indexed from 1, where the nth term can be represented as a_n = n(n + 1). This pattern involves multiplying consecutive integers. The differences between terms\u2014like 2 to 6 (+4), 6 to 12 (+6)\u2014form an arithmetic progression starting at 4, with increments of 2. Essentially, this represents the sum of the first n even numbers. Therefore, the nth term formula is a_n = n(n + 1), which characterizes pronic numbers." + } + ] + }, + { + "id": "msg_07d340e4c7661be80069c5d65c9a0481908630c25ce29d3682", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The terms are \n2,\u20096,\u200912,\u200920,\u200930, \u2026 \n\nIf you look at the differences you get \n6\u20132=4,\u200212\u20136=6,\u200220\u201312=8,\u200230\u201320=10,\u2026 \nso you\u2019re adding 4,6,8,10,\u2026 (i.e. the even numbers from 4 onward). \n\nEquivalently each term is the product of two consecutive integers: \n2=1\u00b72,\u20026=2\u00b73,\u200212=3\u00b74,\u200220=4\u00b75,\u200230=5\u00b76,\u2026\n\nHence the nth term (with n starting at 1) is \na\u2099 = n\u00b7(n+1) = n\u00b2 + n." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "high", + "summary": "detailed" + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 41, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 918, + "output_tokens_details": { + "reasoning_tokens": 704 + }, + "total_tokens": 959 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json new file mode 100644 index 00000000..f6908cec --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json @@ -0,0 +1,49 @@ +{ + "id" : "0d0a384b-29c5-4f05-adeb-c0410a7a4690", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"you are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-0d0a384b-29c5-4f05-adeb-c0410a7a4690.json", + "headers" : { + "x-request-id" : "req_57658aa5e99a471696f70b1ad6479d1b", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e2a732b2e31a4ad-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999980", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:59:00 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=B0Vmg2c_xmsM4UHr.ZI5ipwEGdU9OkaahtQ.Jy7ti2k-1774573139.703917-1.0.1.1-bI3u5XQBkNaD3cyUrye096KPiKCKYkV9L1XwJKn9JreK.gFHRQiexS4D_OftDRIHXN3FRuSAOYFddLxNkn6W93ULITLlz09zAA_V2Dc2Xq2MV4LEvpJkRhrrGyzXEyFW; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:29:00 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "289", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "0d0a384b-29c5-4f05-adeb-c0410a7a4690", + "persistent" : true, + "insertionIndex" : 27 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.json new file mode 100644 index 00000000..090187da --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.json @@ -0,0 +1,49 @@ +{ + "id" : "1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"you are a thoughtful assistant\",\"role\":\"system\"},{\"content\":\"Count from 1 to 10 slowly.\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"max_completion_tokens\":800,\"stream_options\":{\"include_usage\":true},\"temperature\":0.0,\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2.txt", + "headers" : { + "x-request-id" : "req_8e94ccf780424a82ba5566098ceb6994", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e2a732b3aa930ab-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999982", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:59:02 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=_2dc725D8KtWYYP1YIxwIRZNSVReh_Pz6m8DVYNocns-1774573139.7119591-1.0.1.1-EpHduBHgVAOKH3icJOMHCutPbU.L6fsjekz_K.rgfBfaGFosAA72bIjOMM3QqrVNIw1myXpIGF3FLRLLDensPpA7lser1FKnVg4WpNCOqpaLVBtdTsHb23oFI6EcwiAj; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:29:02 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "2497", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "1f71b4fb-3d5c-43c0-96b3-f3a9f5ca04a2", + "persistent" : true, + "insertionIndex" : 24 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json new file mode 100644 index 00000000..d3aa1ddb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json @@ -0,0 +1,49 @@ +{ + "id" : "24baf998-377f-4686-9378-3cdec0cff11d", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"system\",\n \"content\" : \"you are a helpful assistant\"\n }, {\n \"role\" : \"user\",\n \"content\" : \"[{type=text, text=What color is this image?}, {type=image_url, image_url={url=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==}}]\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : false\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-24baf998-377f-4686-9378-3cdec0cff11d.json", + "headers" : { + "x-request-id" : "req_faa8da573e2f4103af6de518ed1d2e8a", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e2a73266e7383bf-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999940", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:59:00 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=jv7MgD7s29ueBzz0GkW0IlmtLxNkD5osqE.wK6dG16k-1774573138.943997-1.0.1.1-rgqFLe4efIdAd8UmMqpL_.pOuJtdmVKSJFd.MAHcg01IERapXRH2VIDw3EfooRPKZsWr63NW5YeCiVTGylYdXkHB.QIiuLOtBcNFPnwOzn9pcXU7svhlNHbg47K2QKre; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:29:00 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "1015", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "24baf998-377f-4686-9378-3cdec0cff11d", + "persistent" : true, + "insertionIndex" : 26 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json new file mode 100644 index 00000000..3a620505 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json @@ -0,0 +1,49 @@ +{ + "id" : "8bf4505e-8fa8-4269-81b2-447008162038", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"What is the weather like in Paris, France?\",\"role\":\"user\"}],\"model\":\"gpt-4o\",\"max_completion_tokens\":500,\"temperature\":0.0,\"tools\":[{\"function\":{\"name\":\"get_weather\",\"description\":\"Get the current weather for a location\",\"parameters\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and state, e.g. San Francisco, CA\"},\"unit\":{\"type\":\"string\",\"enum\":[\"celsius\",\"fahrenheit\"],\"description\":\"The unit of temperature\"}},\"required\":[\"location\"]}},\"type\":\"function\"}]}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-8bf4505e-8fa8-4269-81b2-447008162038.json", + "headers" : { + "x-request-id" : "req_c1ff495e43f24967bbd951bfabdf00c0", + "x-ratelimit-limit-tokens" : "30000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e2a73272a0f757b-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "6ms", + "x-ratelimit-remaining-tokens" : "29999987", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "9999", + "Date" : "Fri, 27 Mar 2026 00:58:59 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=YPNwD_8s0872FF8lftTGobIK5BB6SaQZPXBY7lLFduo-1774573139.0629938-1.0.1.1-9DeqjLlVj.tg7zlnp4VeRDsPJpv_QCrQsGD_JZXMkb0aclK_8KPWb98NvvwbtSDbgc998LiDUdTMRBDesUoHZA.PkOBI0tY0kp5MDfBP6fbuTXU5rLgV7EqhgdlDMuCh; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:28:59 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "10000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "290", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "8bf4505e-8fa8-4269-81b2-447008162038", + "persistent" : true, + "insertionIndex" : 29 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json new file mode 100644 index 00000000..dc2ff822 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json @@ -0,0 +1,52 @@ +{ + "id" : "9b40daa1-1c04-4843-aef0-9795d7b198cc", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"you are a helpful assistant\",\"role\":\"system\"},{\"content\":[{\"text\":\"What color is this image?\",\"type\":\"text\"},{\"image_url\":{\"url\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==\"},\"type\":\"image_url\"}],\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-9b40daa1-1c04-4843-aef0-9795d7b198cc.json", + "headers" : { + "Server" : "cloudflare", + "x-ratelimit-reset-input-images" : "1ms", + "x-ratelimit-reset-tokens" : "0s", + "x-ratelimit-limit-input-images" : "50000", + "set-cookie" : "__cf_bm=tz1gpktLscKzslpW.YHgwDwt_rHFk8QUaZlElAsWbtQ-1774573140.364704-1.0.1.1-Kp590ESrImXGK5dCjdx4fHudd1lv3U0GPbBP7IzqwyD8j8cqjK.pVriwfernI3EE_Tx8Ok6ehtIHZWHaQ1ZusABRgWLR6QInMpHWEEkmQle6NU6WZDJlJv1NqUm5ThAo; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:29:00 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-remaining-input-images" : "49999", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json", + "x-request-id" : "req_4694f19e28164b3f9f459de1355d5a83", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "CF-Ray" : "9e2a732f4aafb9eb-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999217", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:59:00 GMT", + "access-control-expose-headers" : "X-Request-ID", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "448", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "9b40daa1-1c04-4843-aef0-9795d7b198cc", + "persistent" : true, + "insertionIndex" : 25 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json new file mode 100644 index 00000000..772a4bca --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json @@ -0,0 +1,49 @@ +{ + "id" : "b7b2ea82-c026-4354-9a44-5c1ce620c016", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"system\",\n \"content\" : \"you are a helpful assistant\"\n }, {\n \"role\" : \"user\",\n \"content\" : \"What is the capital of France?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : false\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-b7b2ea82-c026-4354-9a44-5c1ce620c016.json", + "headers" : { + "x-request-id" : "req_20ff658b03244c10be8c63341f3c9480", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e2a732668e99b72-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999980", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:58:59 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=RhQYBWuL_A03FRuZ0CdMIopLnbN5S3HknZMpUE.UUC4-1774573138.9433095-1.0.1.1-yxpacoAKtbWi5MTYKo9ruZaV8jpJDKq55tiOVguQHmV20F10_SVBEuvVUl7_Y_2oSXrF0NCL6JWpsuiyjPWtY7UptYMYWEeUdYMWhLjPaZ.eQ.SOagdrULLcB1rty0hj; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:28:59 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "286", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "b7b2ea82-c026-4354-9a44-5c1ce620c016", + "persistent" : true, + "insertionIndex" : 28 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0aac5720-a309-46ee-bb02-5d466bc02763.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0aac5720-a309-46ee-bb02-5d466bc02763.json new file mode 100644 index 00000000..c161fc88 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-0aac5720-a309-46ee-bb02-5d466bc02763.json @@ -0,0 +1,47 @@ +{ + "id" : "0aac5720-a309-46ee-bb02-5d466bc02763", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"content\":\"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\",\"role\":\"user\"},{\"id\":\"rs_07d340e4c7661be80069c5d654a830819081489fe3d0bb4302\",\"summary\":[{\"text\":\"**Analyzing the sequence**\\n\\nThe user is asking about the sequence: 2, 6, 12, 20, 30, which are known as pronic or oblong numbers. Each term can be expressed with the formula n(n+1). Looking closely at the pattern, I notice that the differences between the terms increase by 2: 4, 6, 8, 10, and so on. So, for the nth term, I can confirm that a_n = n(n+1), or alternatively, a_n = n^2 + n, starting from n = 1.\",\"type\":\"summary_text\"},{\"text\":\"**Summarizing the sequence**\\n\\nThe sequence is indexed from 1, where the nth term can be represented as a_n = n(n + 1). This pattern involves multiplying consecutive integers. The differences between terms—like 2 to 6 (+4), 6 to 12 (+6)—form an arithmetic progression starting at 4, with increments of 2. Essentially, this represents the sum of the first n even numbers. Therefore, the nth term formula is a_n = n(n + 1), which characterizes pronic numbers.\",\"type\":\"summary_text\"}],\"type\":\"reasoning\"},{\"id\":\"msg_07d340e4c7661be80069c5d65c9a0481908630c25ce29d3682\",\"content\":[{\"annotations\":[],\"text\":\"The terms are \\n2, 6, 12, 20, 30, … \\n\\nIf you look at the differences you get \\n6–2=4, 12–6=6, 20–12=8, 30–20=10,… \\nso you’re adding 4,6,8,10,… (i.e. the even numbers from 4 onward). \\n\\nEquivalently each term is the product of two consecutive integers: \\n2=1·2, 6=2·3, 12=3·4, 20=4·5, 30=5·6,…\\n\\nHence the nth term (with n starting at 1) is \\naₙ = n·(n+1) = n² + n.\",\"type\":\"output_text\",\"logprobs\":[]}],\"role\":\"assistant\",\"status\":\"completed\",\"type\":\"message\"},{\"content\":\"Using the pattern you discovered, what would be the 10th term? And can you find the sum of the first 10 terms?\",\"role\":\"user\"}],\"model\":\"o4-mini\",\"reasoning\":{\"effort\":\"high\",\"summary\":\"detailed\"}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-0aac5720-a309-46ee-bb02-5d466bc02763.json", + "headers" : { + "x-request-id" : "req_d6ac419df9e040e684afdbde2bee6c48", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "CF-RAY" : "9e2a73677f406a17-SEA", + "Server" : "cloudflare", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999552", + "cf-cache-status" : "DYNAMIC", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:59:19 GMT", + "x-ratelimit-reset-tokens" : "0s", + "set-cookie" : "__cf_bm=xHW1yDAEXvZ8IBNA.0HMo8vs2hkc4HFATv9rn_jV_7M-1774573149.3595676-1.0.1.1-WvVJWZPBnYCCvkAP3yvHLvBIoxgExioVSV9aj1nx8_OryxqusPYfJRDCODZfIjHpsO9u_lJfDpce2XhSf7U4He.PIU2IdRD0cIs56kQyB2lf5cNtdB0egBrXEEtEPpv0; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:29:19 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit-requests" : "30000", + "openai-processing-ms" : "9800", + "openai-version" : "2020-10-01", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "0aac5720-a309-46ee-bb02-5d466bc02763", + "persistent" : true, + "insertionIndex" : 22 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json new file mode 100644 index 00000000..0fa1b136 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json @@ -0,0 +1,47 @@ +{ + "id" : "e4e75ad7-0ead-4360-8025-e390931f8ef2", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"content\":\"Look at this sequence: 2, 6, 12, 20, 30. What is the pattern and what would be the formula for the nth term?\\n\",\"role\":\"user\"}],\"model\":\"o4-mini\",\"reasoning\":{\"effort\":\"high\",\"summary\":\"detailed\"}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-e4e75ad7-0ead-4360-8025-e390931f8ef2.json", + "headers" : { + "x-request-id" : "req_d8de8510a3804db1a558d58d92e62fc0", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "CF-RAY" : "9e2a732e9d09e17a-SEA", + "Server" : "cloudflare", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999752", + "cf-cache-status" : "DYNAMIC", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Fri, 27 Mar 2026 00:59:09 GMT", + "x-ratelimit-reset-tokens" : "0s", + "set-cookie" : "__cf_bm=w6IfxuJivQWxXQcX4jttbaJ4sLJdEn6MzkBzfYcYhSw-1774573140.2587576-1.0.1.1-I.RiqRv5Hon4UaH0rDuyn3eKnvfwS.fhR6TjM2ObmWlNyH91SXdpiJJ0y_aiAkCmdW9v7mMtWJbkMle51kQrSIwyT5EP4B3HMUpNTW8xjtLCZJhfNuGbMxecnpLy9gvN; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 27 Mar 2026 01:29:09 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "x-ratelimit-limit-requests" : "30000", + "openai-processing-ms" : "8733", + "openai-version" : "2020-10-01", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "e4e75ad7-0ead-4360-8025-e390931f8ef2", + "persistent" : true, + "insertionIndex" : 23 +} \ No newline at end of file