From 5959d74ebf22f6fe3b7aa081752f0dce23944906 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Fri, 21 Aug 2026 04:14:33 +0000 Subject: [PATCH 1/3] test: characterize ambiguous union behavior --- scripts/union-characterization | 9 + scripts/union-characterization-differential | 20 + .../fixtures/union_characterization/README.md | 17 + .../union_characterization/cases.json | 320 ++++++++++ .../fixtures/union_characterization/main.json | 545 ++++++++++++++++++ .../union_characterization/published.json | 535 +++++++++++++++++ .../internal/type/union_characterization.rb | 180 ++++++ .../type/union_characterization_test.rb | 37 ++ 8 files changed, 1663 insertions(+) create mode 100755 scripts/union-characterization create mode 100755 scripts/union-characterization-differential create mode 100644 test/fixtures/union_characterization/README.md create mode 100644 test/fixtures/union_characterization/cases.json create mode 100644 test/fixtures/union_characterization/main.json create mode 100644 test/fixtures/union_characterization/published.json create mode 100644 test/openai/internal/type/union_characterization.rb create mode 100644 test/openai/internal/type/union_characterization_test.rb diff --git a/scripts/union-characterization b/scripts/union-characterization new file mode 100755 index 000000000..04aa5595d --- /dev/null +++ b/scripts/union-characterization @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "openai" + +require_relative "../test/openai/internal/type/union_characterization" + +puts(JSON.pretty_generate(OpenAI::Test::UnionCharacterization.observe)) diff --git a/scripts/union-characterization-differential b/scripts/union-characterization-differential new file mode 100755 index 000000000..825a4aaaa --- /dev/null +++ b/scripts/union-characterization-differential @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd -- "$(dirname -- "$0")/.." + +tmp_dir=$(mktemp -d) +trap 'rm -rf -- "$tmp_dir"' EXIT + +for release in v0.78.0 v0.80.0; do + mkdir "$tmp_dir/$release" + git archive "$release" | tar -x -C "$tmp_dir/$release" + ruby -I"$tmp_dir/$release/lib" scripts/union-characterization >"$tmp_dir/$release.json" + diff -u test/fixtures/union_characterization/published.json "$tmp_dir/$release.json" +done + +ruby -Ilib scripts/union-characterization >"$tmp_dir/main.json" +diff -u test/fixtures/union_characterization/main.json "$tmp_dir/main.json" + +echo "Union characterization matches v0.78.0, v0.80.0, and main baselines." diff --git a/test/fixtures/union_characterization/README.md b/test/fixtures/union_characterization/README.md new file mode 100644 index 000000000..03efe3a7f --- /dev/null +++ b/test/fixtures/union_characterization/README.md @@ -0,0 +1,17 @@ +# Union characterization corpus + +This corpus records current behavior; it does not declare every selected class to be semantically correct. + +- `discriminator_evidence` is an explicit fixture fact: missing, known and unique, known but duplicated, or explicitly unknown. +- `exact_candidates` lists every generated variant whose independent coercion has no `no` or `maybe` evidence. +- `structural_candidates` lists every generated variant whose independent coercion has no `no` evidence. +- `best_ranked_fallbacks` applies the current `Union#coerce` fallback score only when no exact candidate exists. +- `compatibility_selected` is the class selected by the current runtime. It is a compatibility observation, not an endorsement. +- `serialized_body` proves that accepted request shorthand and unknown fields retain their wire representation regardless of materialization. +- `public_boundary` records context-specific behavior when it differs from the shared union. It is currently available for the Realtime connection introduced after v0.80.0. + +The fixtures deliberately pair wire-string hashes with idiomatic Ruby symbol hashes. Their different candidate sets are part of the observed contract. + +`published.json` is the shared v0.78.0/v0.80.0 release baseline; the two releases produce identical observations for this corpus. `main.json` is the current generated/runtime baseline. Run `scripts/union-characterization-differential` under the repository Ruby to regenerate every observation in temporary tag archives and verify the checked-in files. + +The two published baselines agree for this corpus. Current `main` retains the request behavior and serialized bodies, but newer generated Responses/Beta variants change which known event wins structural fallback for an explicit unknown stream tag. Current Realtime adds a public connection boundary that returns `UnknownServerEvent`, even though direct shared-union coercion can still select a known event for symbolized input. diff --git a/test/fixtures/union_characterization/cases.json b/test/fixtures/union_characterization/cases.json new file mode 100644 index 000000000..2135c4993 --- /dev/null +++ b/test/fixtures/union_characterization/cases.json @@ -0,0 +1,320 @@ +[ + { + "id": "responses_string_message_without_type", + "union": "responses_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "missing" + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "role": "user", + "content": "hello" + }, + "serialization": "responses_request" + }, + { + "id": "responses_typed_message_without_type", + "union": "responses_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "missing" + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + }, + "serialization": "responses_request" + }, + { + "id": "responses_typed_message_without_type_symbols", + "union": "responses_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "missing" + }, + "uniquely_resolvable": false, + "input_form": "ruby_symbols", + "input": { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + }, + "serialization": "responses_request" + }, + { + "id": "responses_typed_message_with_duplicate_tag", + "union": "responses_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3 + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + }, + "serialization": "responses_request" + }, + { + "id": "responses_typed_message_with_duplicate_tag_symbols", + "union": "responses_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3 + }, + "uniquely_resolvable": false, + "input_form": "ruby_symbols", + "input": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + }, + "serialization": "responses_request" + }, + { + "id": "responses_typed_message_with_unknown_field", + "union": "responses_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "missing" + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ], + "future_field": { + "preserved": true + } + }, + "serialization": "responses_request" + }, + { + "id": "beta_typed_message_without_type", + "union": "beta_response_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "missing" + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + }, + "serialization": "beta_responses_request" + }, + { + "id": "beta_typed_message_with_duplicate_tag_symbols", + "union": "beta_response_input_item", + "context": "request", + "discriminator": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3 + }, + "uniquely_resolvable": false, + "input_form": "ruby_symbols", + "input": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + }, + "serialization": "beta_responses_request" + }, + { + "id": "responses_unknown_stream_event", + "union": "responses_stream_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0 + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "serialization": "union" + }, + { + "id": "responses_unknown_stream_event_symbols", + "union": "responses_stream_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0 + }, + "uniquely_resolvable": false, + "input_form": "ruby_symbols", + "input": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "serialization": "union" + }, + { + "id": "beta_unknown_stream_event", + "union": "beta_response_stream_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0 + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "serialization": "union" + }, + { + "id": "beta_unknown_stream_event_symbols", + "union": "beta_response_stream_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0 + }, + "uniquely_resolvable": false, + "input_form": "ruby_symbols", + "input": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "serialization": "union" + }, + { + "id": "realtime_unknown_server_event", + "union": "realtime_server_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0 + }, + "uniquely_resolvable": false, + "input_form": "wire", + "input": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "serialization": "union", + "public_boundary": "realtime_connection" + }, + { + "id": "realtime_unknown_server_event_symbols", + "union": "realtime_server_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0 + }, + "uniquely_resolvable": false, + "input_form": "ruby_symbols", + "input": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "serialization": "union", + "public_boundary": "realtime_connection" + }, + { + "id": "realtime_known_unique_server_event", + "union": "realtime_server_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "known_unique", + "matching_variants": 1 + }, + "uniquely_resolvable": true, + "input_form": "wire", + "input": { + "type": "conversation.created", + "event_id": "evt_1", + "conversation": { + "id": "conv_1", + "object": "realtime.conversation" + } + }, + "serialization": "union" + }, + { + "id": "realtime_known_unique_server_event_symbols", + "union": "realtime_server_event", + "context": "response_stream", + "discriminator": { + "property": "type", + "state": "known_unique", + "matching_variants": 1 + }, + "uniquely_resolvable": true, + "input_form": "ruby_symbols", + "input": { + "type": "conversation.created", + "event_id": "evt_1", + "conversation": { + "id": "conv_1", + "object": "realtime.conversation" + } + }, + "serialization": "union" + } +] diff --git a/test/fixtures/union_characterization/main.json b/test/fixtures/union_characterization/main.json new file mode 100644 index 000000000..1a4a792b1 --- /dev/null +++ b/test/fixtures/union_characterization/main.json @@ -0,0 +1,545 @@ +{ + "responses_string_message_without_type": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 2, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": "hello" + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_without_type": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 2, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_without_type_symbols": { + "context": "request", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", + "selected_exactness": { + "yes": 9, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_with_duplicate_tag": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3, + "value": "message" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", + "selected_exactness": { + "yes": 3, + "no": 2, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_with_duplicate_tag_symbols": { + "context": "request", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3, + "value": "message" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", + "selected_exactness": { + "yes": 9, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_with_unknown_field": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 2, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ], + "future_field": { + "preserved": true + } + } + ], + "model": "gpt-4o" + } + }, + "beta_typed_message_without_type": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 3, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "beta_typed_message_with_duplicate_tag_symbols": { + "context": "request", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3, + "value": "message" + }, + "exact_candidates": [ + "OpenAI::Models::Beta::BetaEasyInputMessage", + "OpenAI::Models::Beta::BetaResponseInputItem::Message" + ], + "structural_candidates": [ + "OpenAI::Models::Beta::BetaEasyInputMessage", + "OpenAI::Models::Beta::BetaResponseInputItem::Message", + "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaEasyInputMessage", + "selected_exactness": { + "yes": 9, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_unknown_stream_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [ + "OpenAI::Models::Responses::ResponseErrorEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseErrorEvent", + "selected_exactness": { + "yes": 2, + "no": 2, + "maybe": 2 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + } + }, + "responses_unknown_stream_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseAudioDoneEvent", + "OpenAI::Models::Responses::ResponseAudioTranscriptDoneEvent" + ], + "best_ranked_fallbacks": [ + "OpenAI::Models::Responses::ResponseImageGenCallPartialImageEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseImageGenCallPartialImageEvent", + "selected_exactness": { + "yes": 6, + "no": 4, + "maybe": 1 + }, + "serialized_body": { + "sequence_number": 9, + "type": "future.unmodeled.event" + } + }, + "beta_unknown_stream_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [ + "OpenAI::Models::Beta::BetaResponseErrorEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaResponseErrorEvent", + "selected_exactness": { + "yes": 3, + "no": 2, + "maybe": 2 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + } + }, + "beta_unknown_stream_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [ + "OpenAI::Models::Beta::BetaResponseAudioDoneEvent", + "OpenAI::Models::Beta::BetaResponseAudioTranscriptDoneEvent" + ], + "best_ranked_fallbacks": [ + "OpenAI::Models::Beta::BetaResponseImageGenCallPartialImageEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaResponseImageGenCallPartialImageEvent", + "selected_exactness": { + "yes": 7, + "no": 4, + "maybe": 1 + }, + "serialized_body": { + "sequence_number": 9, + "type": "future.unmodeled.event" + } + }, + "realtime_unknown_server_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "Hash", + "selected_error": "ArgumentError", + "selected_exactness": { + "yes": 0, + "no": 1, + "maybe": 0 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "public_boundary": { + "available": true, + "selected": "OpenAI::Models::Realtime::UnknownServerEvent", + "body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + } + } + }, + "realtime_unknown_server_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [ + "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent", + "selected_exactness": { + "yes": 4, + "no": 2, + "maybe": 1 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "public_boundary": { + "available": true, + "selected": "OpenAI::Models::Realtime::UnknownServerEvent", + "body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + } + } + }, + "realtime_known_unique_server_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "known_unique", + "matching_variants": 1, + "value": "conversation.created" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [], + "uniquely_resolvable": true, + "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", + "selected_exactness": { + "yes": 2, + "no": 2, + "maybe": 0 + }, + "serialized_body": { + "type": "conversation.created", + "event_id": "evt_1", + "conversation": { + "id": "conv_1", + "object": "realtime.conversation" + } + } + }, + "realtime_known_unique_server_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "known_unique", + "matching_variants": 1, + "value": "conversation.created" + }, + "exact_candidates": [ + "OpenAI::Models::Realtime::ConversationCreatedEvent" + ], + "structural_candidates": [ + "OpenAI::Models::Realtime::ConversationCreatedEvent", + "OpenAI::Models::Realtime::InputAudioBufferClearedEvent" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": true, + "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", + "selected_exactness": { + "yes": 6, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "conversation": { + "id": "conv_1", + "object": "realtime.conversation" + }, + "event_id": "evt_1", + "type": "conversation.created" + } + } +} diff --git a/test/fixtures/union_characterization/published.json b/test/fixtures/union_characterization/published.json new file mode 100644 index 000000000..9f8cc1c5c --- /dev/null +++ b/test/fixtures/union_characterization/published.json @@ -0,0 +1,535 @@ +{ + "responses_string_message_without_type": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 2, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": "hello" + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_without_type": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 2, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_without_type_symbols": { + "context": "request", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", + "selected_exactness": { + "yes": 9, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_with_duplicate_tag": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3, + "value": "message" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", + "selected_exactness": { + "yes": 3, + "no": 2, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_with_duplicate_tag_symbols": { + "context": "request", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3, + "value": "message" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", + "selected_exactness": { + "yes": 9, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_typed_message_with_unknown_field": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 2, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ], + "future_field": { + "preserved": true + } + } + ], + "model": "gpt-4o" + } + }, + "beta_typed_message_without_type": { + "context": "request", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "missing" + }, + "exact_candidates": [ + "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" + ], + "structural_candidates": [ + "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger", + "selected_exactness": { + "yes": 3, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "beta_typed_message_with_duplicate_tag_symbols": { + "context": "request", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "known_duplicate", + "matching_variants": 3, + "value": "message" + }, + "exact_candidates": [ + "OpenAI::Models::Beta::BetaEasyInputMessage", + "OpenAI::Models::Beta::BetaResponseInputItem::Message" + ], + "structural_candidates": [ + "OpenAI::Models::Beta::BetaEasyInputMessage", + "OpenAI::Models::Beta::BetaResponseInputItem::Message", + "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaEasyInputMessage", + "selected_exactness": { + "yes": 9, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "hello" + } + ] + } + ], + "model": "gpt-4o" + } + }, + "responses_unknown_stream_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [ + "OpenAI::Models::Responses::ResponseErrorEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseErrorEvent", + "selected_exactness": { + "yes": 2, + "no": 2, + "maybe": 2 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + } + }, + "responses_unknown_stream_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [ + "OpenAI::Models::Responses::ResponseAudioDoneEvent", + "OpenAI::Models::Responses::ResponseAudioTranscriptDoneEvent" + ], + "best_ranked_fallbacks": [ + "OpenAI::Models::Responses::ResponseReasoningSummaryPartDoneEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Responses::ResponseReasoningSummaryPartDoneEvent", + "selected_exactness": { + "yes": 3, + "no": 4, + "maybe": 1 + }, + "serialized_body": { + "sequence_number": 9, + "type": "future.unmodeled.event" + } + }, + "beta_unknown_stream_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [ + "OpenAI::Models::Beta::BetaResponseErrorEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaResponseErrorEvent", + "selected_exactness": { + "yes": 3, + "no": 2, + "maybe": 2 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + } + }, + "beta_unknown_stream_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [ + "OpenAI::Models::Beta::BetaResponseAudioDoneEvent", + "OpenAI::Models::Beta::BetaResponseAudioTranscriptDoneEvent" + ], + "best_ranked_fallbacks": [ + "OpenAI::Models::Beta::BetaResponseReasoningSummaryPartDoneEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Beta::BetaResponseReasoningSummaryPartDoneEvent", + "selected_exactness": { + "yes": 4, + "no": 4, + "maybe": 1 + }, + "serialized_body": { + "sequence_number": 9, + "type": "future.unmodeled.event" + } + }, + "realtime_unknown_server_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [], + "uniquely_resolvable": false, + "compatibility_selected": "Hash", + "selected_error": "ArgumentError", + "selected_exactness": { + "yes": 0, + "no": 1, + "maybe": 0 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "public_boundary": { + "available": false + } + }, + "realtime_unknown_server_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "explicitly_unknown", + "matching_variants": 0, + "value": "future.unmodeled.event" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [ + "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent" + ], + "uniquely_resolvable": false, + "compatibility_selected": "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent", + "selected_exactness": { + "yes": 4, + "no": 2, + "maybe": 1 + }, + "serialized_body": { + "type": "future.unmodeled.event", + "sequence_number": 9 + }, + "public_boundary": { + "available": false + } + }, + "realtime_known_unique_server_event": { + "context": "response_stream", + "input_form": "wire", + "discriminator_evidence": { + "property": "type", + "state": "known_unique", + "matching_variants": 1, + "value": "conversation.created" + }, + "exact_candidates": [], + "structural_candidates": [], + "best_ranked_fallbacks": [], + "uniquely_resolvable": true, + "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", + "selected_exactness": { + "yes": 2, + "no": 2, + "maybe": 0 + }, + "serialized_body": { + "type": "conversation.created", + "event_id": "evt_1", + "conversation": { + "id": "conv_1", + "object": "realtime.conversation" + } + } + }, + "realtime_known_unique_server_event_symbols": { + "context": "response_stream", + "input_form": "ruby_symbols", + "discriminator_evidence": { + "property": "type", + "state": "known_unique", + "matching_variants": 1, + "value": "conversation.created" + }, + "exact_candidates": [ + "OpenAI::Models::Realtime::ConversationCreatedEvent" + ], + "structural_candidates": [ + "OpenAI::Models::Realtime::ConversationCreatedEvent", + "OpenAI::Models::Realtime::InputAudioBufferClearedEvent" + ], + "best_ranked_fallbacks": [], + "uniquely_resolvable": true, + "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", + "selected_exactness": { + "yes": 6, + "no": 0, + "maybe": 0 + }, + "serialized_body": { + "conversation": { + "id": "conv_1", + "object": "realtime.conversation" + }, + "event_id": "evt_1", + "type": "conversation.created" + } + } +} diff --git a/test/openai/internal/type/union_characterization.rb b/test/openai/internal/type/union_characterization.rb new file mode 100644 index 000000000..e9a2e7e8e --- /dev/null +++ b/test/openai/internal/type/union_characterization.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require "json" +require "uri" + +module OpenAI + module Test + end +end + +module OpenAI::Test::UnionCharacterization + FIXTURE_PATH = File.expand_path("../../../fixtures/union_characterization/cases.json", __dir__) + SYMBOL_VALUE_KEYS = %w[role type].freeze + UNION_TARGETS = { + "responses_input_item" => OpenAI::Responses::ResponseInputItem, + "beta_response_input_item" => OpenAI::Beta::BetaResponseInputItem, + "responses_stream_event" => OpenAI::Responses::ResponseStreamEvent, + "beta_response_stream_event" => OpenAI::Beta::BetaResponseStreamEvent, + "realtime_server_event" => OpenAI::Realtime::RealtimeServerEvent + }.freeze + + module_function + + def observe = fixtures.to_h { |fixture| [fixture.fetch("id"), observe_case(fixture)] } + + def fixtures = JSON.parse(File.read(FIXTURE_PATH)) + + def observe_case(fixture) + union = UNION_TARGETS.fetch(fixture.fetch("union")) + input = fixture_input(fixture) + candidates = union.variants.map { observe_candidate(_1, input) } + exact_candidates = candidates.filter_map do |candidate| + candidate.fetch("target") if candidate.fetch("exactness").values_at("no", "maybe") == [0, 0] + end + + structural_candidates = candidates.filter_map do |candidate| + candidate.fetch("target") if candidate.fetch("exactness").fetch("no").zero? + end + + evidence = discriminator_evidence(fixture.fetch("discriminator"), input) + + state = OpenAI::Internal::Type::Converter.new_coerce_state + selected = OpenAI::Internal::Type::Converter.coerce(union, duplicate(input), state: state) + + { + "context" => fixture.fetch("context"), + "input_form" => fixture.fetch("input_form"), + "discriminator_evidence" => evidence, + "exact_candidates" => exact_candidates, + "structural_candidates" => structural_candidates, + "best_ranked_fallbacks" => exact_candidates.empty? ? best_ranked_fallbacks(candidates) : [], + "uniquely_resolvable" => fixture.fetch("uniquely_resolvable"), + "compatibility_selected" => target_name(selected.class), + "selected_error" => state.fetch(:error)&.class&.name, + "selected_exactness" => stringify_keys(state.fetch(:exactness)), + "serialized_body" => normalize(serialize(fixture, union, selected, input)), + "public_boundary" => observe_public_boundary(fixture, input) + }.compact + end + + def observe_candidate(target, input) + state = OpenAI::Internal::Type::Converter.new_coerce_state + OpenAI::Internal::Type::Converter.coerce(target, duplicate(input), state: state) + { + "target" => target_name(target), + "exactness" => stringify_keys(state.fetch(:exactness)), + "error" => state.fetch(:error)&.class&.name + } + end + + def discriminator_evidence(expected, input) + property = expected.fetch("property") + value = input.fetch(property.to_sym) { input.fetch(property, OpenAI::Internal::OMIT) } + evidence = {"property" => property, **expected.except("property")} + if expected.fetch("state") == "missing" + raise ArgumentError.new(expected) unless value == OpenAI::Internal::OMIT + + return evidence + end + + raise ArgumentError.new(expected) if value == OpenAI::Internal::OMIT + + normalized = value.is_a?(Symbol) ? value.to_s : value + {**evidence, "value" => normalized} + end + + def best_ranked_fallbacks(candidates) + alternatives = candidates.select { _1.fetch("exactness").fetch("maybe").positive? } + return [] if alternatives.empty? + + best_score = alternatives.map { fallback_score(_1) }.min + alternatives.filter_map { _1.fetch("target") if fallback_score(_1) == best_score } + end + + def fallback_score(candidate) + yes, no, maybe = candidate.fetch("exactness").values_at("yes", "no", "maybe") + [-yes, -maybe, no] + end + + def serialize(fixture, union, selected, input) + case fixture.fetch("serialization") + when "responses_request" + params = OpenAI::Responses::ResponseCreateParams.new(input: [duplicate(input)], model: "gpt-4o") + OpenAI::Responses::ResponseCreateParams.dump_request(params).first + when "beta_responses_request" + params = OpenAI::Beta::ResponseCreateParams.new(input: [duplicate(input)], model: "gpt-4o") + OpenAI::Beta::ResponseCreateParams.dump_request(params).first + when "union" + OpenAI::Internal::Type::Converter.dump(union, selected) + else + raise ArgumentError.new(fixture) + end + end + + def observe_public_boundary(fixture, input) + return unless fixture["public_boundary"] == "realtime_connection" + return {"available" => false} unless defined?(OpenAI::Realtime::Connection) + + connection = OpenAI::Realtime::Connection.new(socket: nil, url: URI("wss://example.test")) + event = connection.parse_event(JSON.generate(input)) + { + "available" => true, + "selected" => target_name(event.class), + "body" => normalize(event.to_h) + } + end + + def fixture_input(fixture) + input = duplicate(fixture.fetch("input")) + case fixture.fetch("input_form") + when "ruby_symbols" + symbolize_ruby_input(input) + when "wire" + input + else + raise ArgumentError.new(fixture) + end + end + + def symbolize_ruby_input(value, parent_key: nil) + case value + in Hash + value.to_h { |key, item| [key.to_sym, symbolize_ruby_input(item, parent_key: key)] } + in Array + value.map { symbolize_ruby_input(_1, parent_key: parent_key) } + in String if SYMBOL_VALUE_KEYS.include?(parent_key) + value.to_sym + else + value + end + end + + def duplicate(value) + case value + in Hash + value.to_h { |key, item| [key, duplicate(item)] } + in Array + value.map { duplicate(_1) } + else + value + end + end + + def normalize(value) + case value + in Hash + value.to_h { |key, item| [key.to_s, normalize(item)] } + in Array + value.map { normalize(_1) } + in Symbol + value.to_s + else + value + end + end + + def stringify_keys(value) = value.to_h { |key, item| [key.to_s, item] } + + def target_name(target) = target.name || target.inspect +end diff --git a/test/openai/internal/type/union_characterization_test.rb b/test/openai/internal/type/union_characterization_test.rb new file mode 100644 index 000000000..97fec4771 --- /dev/null +++ b/test/openai/internal/type/union_characterization_test.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require_relative "../../test_helper" +require_relative "union_characterization" + +class OpenAI::Test::UnionCharacterizationTest < Minitest::Test + EXPECTED_PATH = File.expand_path("../../../fixtures/union_characterization/main.json", __dir__) + + def test_current_union_characterization_matches_the_recorded_main_baseline + expected = JSON.parse(File.read(EXPECTED_PATH)) + + assert_equal(expected, OpenAI::Test::UnionCharacterization.observe) + end + + def test_ambiguous_and_unknown_values_are_not_labeled_uniquely_resolvable + observations = OpenAI::Test::UnionCharacterization.observe + ambiguous_ids = observations.keys.grep(/responses|beta/) + ambiguous_ids.concat(%w[realtime_unknown_server_event realtime_unknown_server_event_symbols]) + + ambiguous_ids.each { refute(observations.fetch(_1).fetch("uniquely_resolvable"), _1) } + %w[realtime_known_unique_server_event realtime_known_unique_server_event_symbols].each do |id| + assert(observations.fetch(id).fetch("uniquely_resolvable"), id) + end + end + + def test_realtime_boundary_preserves_an_explicitly_unknown_event + cases = OpenAI::Test::UnionCharacterization.observe + + %w[realtime_unknown_server_event realtime_unknown_server_event_symbols].each do |id| + boundary = cases.fetch(id).fetch("public_boundary") + + assert_equal(true, boundary.fetch("available")) + assert_equal("OpenAI::Models::Realtime::UnknownServerEvent", boundary.fetch("selected")) + assert_equal("future.unmodeled.event", boundary.dig("body", "type")) + end + end +end From d90571e5487a3b43dd553ab9d2eefe553b8b3798 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Fri, 21 Aug 2026 04:38:26 +0000 Subject: [PATCH 2/3] test: trim union characterization baselines --- scripts/union-characterization-differential | 17 +- .../fixtures/union_characterization/README.md | 4 +- .../fixtures/union_characterization/main.json | 545 ------------------ .../union_characterization/published.json | 535 ----------------- .../type/union_characterization_test.rb | 95 ++- 5 files changed, 103 insertions(+), 1093 deletions(-) delete mode 100644 test/fixtures/union_characterization/main.json delete mode 100644 test/fixtures/union_characterization/published.json diff --git a/scripts/union-characterization-differential b/scripts/union-characterization-differential index 825a4aaaa..4bd572d89 100755 --- a/scripts/union-characterization-differential +++ b/scripts/union-characterization-differential @@ -11,10 +11,21 @@ for release in v0.78.0 v0.80.0; do mkdir "$tmp_dir/$release" git archive "$release" | tar -x -C "$tmp_dir/$release" ruby -I"$tmp_dir/$release/lib" scripts/union-characterization >"$tmp_dir/$release.json" - diff -u test/fixtures/union_characterization/published.json "$tmp_dir/$release.json" done +echo "Comparing published releases v0.78.0 and v0.80.0..." +diff -u "$tmp_dir/v0.78.0.json" "$tmp_dir/v0.80.0.json" + ruby -Ilib scripts/union-characterization >"$tmp_dir/main.json" -diff -u test/fixtures/union_characterization/main.json "$tmp_dir/main.json" -echo "Union characterization matches v0.78.0, v0.80.0, and main baselines." +echo "Published releases agree. Their differential from the current checkout follows:" +if diff -u --label v0.78.0-and-v0.80.0 --label current-checkout \ + "$tmp_dir/v0.80.0.json" "$tmp_dir/main.json" +then + echo "No differences." +else + status=$? + if ((status > 1)); then + exit "$status" + fi +fi diff --git a/test/fixtures/union_characterization/README.md b/test/fixtures/union_characterization/README.md index 03efe3a7f..0b57087dd 100644 --- a/test/fixtures/union_characterization/README.md +++ b/test/fixtures/union_characterization/README.md @@ -10,8 +10,8 @@ This corpus records current behavior; it does not declare every selected class t - `serialized_body` proves that accepted request shorthand and unknown fields retain their wire representation regardless of materialization. - `public_boundary` records context-specific behavior when it differs from the shared union. It is currently available for the Realtime connection introduced after v0.80.0. -The fixtures deliberately pair wire-string hashes with idiomatic Ruby symbol hashes. Their different candidate sets are part of the observed contract. +The fixtures deliberately pair wire-string hashes with idiomatic Ruby symbol hashes. Their different candidate sets are part of the observed contract. Focused tests retain the compatibility-sensitive selections and serialized bodies without checking in full generated output snapshots. -`published.json` is the shared v0.78.0/v0.80.0 release baseline; the two releases produce identical observations for this corpus. `main.json` is the current generated/runtime baseline. Run `scripts/union-characterization-differential` under the repository Ruby to regenerate every observation in temporary tag archives and verify the checked-in files. +Run `scripts/union-characterization` to print every current observation. Run `scripts/union-characterization-differential` under the repository Ruby to generate observations from temporary v0.78.0 and v0.80.0 tag archives, verify that those published releases agree, and print their differential from the current checkout. The two published baselines agree for this corpus. Current `main` retains the request behavior and serialized bodies, but newer generated Responses/Beta variants change which known event wins structural fallback for an explicit unknown stream tag. Current Realtime adds a public connection boundary that returns `UnknownServerEvent`, even though direct shared-union coercion can still select a known event for symbolized input. diff --git a/test/fixtures/union_characterization/main.json b/test/fixtures/union_characterization/main.json deleted file mode 100644 index 1a4a792b1..000000000 --- a/test/fixtures/union_characterization/main.json +++ /dev/null @@ -1,545 +0,0 @@ -{ - "responses_string_message_without_type": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 2, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": "hello" - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_without_type": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 2, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_without_type_symbols": { - "context": "request", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message", - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message", - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", - "selected_exactness": { - "yes": 9, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_with_duplicate_tag": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "known_duplicate", - "matching_variants": 3, - "value": "message" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", - "selected_exactness": { - "yes": 3, - "no": 2, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_with_duplicate_tag_symbols": { - "context": "request", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "known_duplicate", - "matching_variants": 3, - "value": "message" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message", - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", - "selected_exactness": { - "yes": 9, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_with_unknown_field": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 2, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ], - "future_field": { - "preserved": true - } - } - ], - "model": "gpt-4o" - } - }, - "beta_typed_message_without_type": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 3, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "beta_typed_message_with_duplicate_tag_symbols": { - "context": "request", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "known_duplicate", - "matching_variants": 3, - "value": "message" - }, - "exact_candidates": [ - "OpenAI::Models::Beta::BetaEasyInputMessage", - "OpenAI::Models::Beta::BetaResponseInputItem::Message" - ], - "structural_candidates": [ - "OpenAI::Models::Beta::BetaEasyInputMessage", - "OpenAI::Models::Beta::BetaResponseInputItem::Message", - "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaEasyInputMessage", - "selected_exactness": { - "yes": 9, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_unknown_stream_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [ - "OpenAI::Models::Responses::ResponseErrorEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseErrorEvent", - "selected_exactness": { - "yes": 2, - "no": 2, - "maybe": 2 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - } - }, - "responses_unknown_stream_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseAudioDoneEvent", - "OpenAI::Models::Responses::ResponseAudioTranscriptDoneEvent" - ], - "best_ranked_fallbacks": [ - "OpenAI::Models::Responses::ResponseImageGenCallPartialImageEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseImageGenCallPartialImageEvent", - "selected_exactness": { - "yes": 6, - "no": 4, - "maybe": 1 - }, - "serialized_body": { - "sequence_number": 9, - "type": "future.unmodeled.event" - } - }, - "beta_unknown_stream_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [ - "OpenAI::Models::Beta::BetaResponseErrorEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaResponseErrorEvent", - "selected_exactness": { - "yes": 3, - "no": 2, - "maybe": 2 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - } - }, - "beta_unknown_stream_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [ - "OpenAI::Models::Beta::BetaResponseAudioDoneEvent", - "OpenAI::Models::Beta::BetaResponseAudioTranscriptDoneEvent" - ], - "best_ranked_fallbacks": [ - "OpenAI::Models::Beta::BetaResponseImageGenCallPartialImageEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaResponseImageGenCallPartialImageEvent", - "selected_exactness": { - "yes": 7, - "no": 4, - "maybe": 1 - }, - "serialized_body": { - "sequence_number": 9, - "type": "future.unmodeled.event" - } - }, - "realtime_unknown_server_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "Hash", - "selected_error": "ArgumentError", - "selected_exactness": { - "yes": 0, - "no": 1, - "maybe": 0 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - }, - "public_boundary": { - "available": true, - "selected": "OpenAI::Models::Realtime::UnknownServerEvent", - "body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - } - } - }, - "realtime_unknown_server_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [ - "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent", - "selected_exactness": { - "yes": 4, - "no": 2, - "maybe": 1 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - }, - "public_boundary": { - "available": true, - "selected": "OpenAI::Models::Realtime::UnknownServerEvent", - "body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - } - } - }, - "realtime_known_unique_server_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "known_unique", - "matching_variants": 1, - "value": "conversation.created" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [], - "uniquely_resolvable": true, - "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", - "selected_exactness": { - "yes": 2, - "no": 2, - "maybe": 0 - }, - "serialized_body": { - "type": "conversation.created", - "event_id": "evt_1", - "conversation": { - "id": "conv_1", - "object": "realtime.conversation" - } - } - }, - "realtime_known_unique_server_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "known_unique", - "matching_variants": 1, - "value": "conversation.created" - }, - "exact_candidates": [ - "OpenAI::Models::Realtime::ConversationCreatedEvent" - ], - "structural_candidates": [ - "OpenAI::Models::Realtime::ConversationCreatedEvent", - "OpenAI::Models::Realtime::InputAudioBufferClearedEvent" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": true, - "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", - "selected_exactness": { - "yes": 6, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "conversation": { - "id": "conv_1", - "object": "realtime.conversation" - }, - "event_id": "evt_1", - "type": "conversation.created" - } - } -} diff --git a/test/fixtures/union_characterization/published.json b/test/fixtures/union_characterization/published.json deleted file mode 100644 index 9f8cc1c5c..000000000 --- a/test/fixtures/union_characterization/published.json +++ /dev/null @@ -1,535 +0,0 @@ -{ - "responses_string_message_without_type": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 2, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": "hello" - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_without_type": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 2, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_without_type_symbols": { - "context": "request", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message", - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message", - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", - "selected_exactness": { - "yes": 9, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_with_duplicate_tag": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "known_duplicate", - "matching_variants": 3, - "value": "message" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", - "selected_exactness": { - "yes": 3, - "no": 2, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_with_duplicate_tag_symbols": { - "context": "request", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "known_duplicate", - "matching_variants": 3, - "value": "message" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::EasyInputMessage", - "OpenAI::Models::Responses::ResponseInputItem::Message", - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::EasyInputMessage", - "selected_exactness": { - "yes": 9, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_typed_message_with_unknown_field": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 2, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ], - "future_field": { - "preserved": true - } - } - ], - "model": "gpt-4o" - } - }, - "beta_typed_message_without_type": { - "context": "request", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "missing" - }, - "exact_candidates": [ - "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" - ], - "structural_candidates": [ - "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger", - "selected_exactness": { - "yes": 3, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "beta_typed_message_with_duplicate_tag_symbols": { - "context": "request", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "known_duplicate", - "matching_variants": 3, - "value": "message" - }, - "exact_candidates": [ - "OpenAI::Models::Beta::BetaEasyInputMessage", - "OpenAI::Models::Beta::BetaResponseInputItem::Message" - ], - "structural_candidates": [ - "OpenAI::Models::Beta::BetaEasyInputMessage", - "OpenAI::Models::Beta::BetaResponseInputItem::Message", - "OpenAI::Models::Beta::BetaResponseInputItem::CompactionTrigger" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaEasyInputMessage", - "selected_exactness": { - "yes": 9, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "input": [ - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "hello" - } - ] - } - ], - "model": "gpt-4o" - } - }, - "responses_unknown_stream_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [ - "OpenAI::Models::Responses::ResponseErrorEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseErrorEvent", - "selected_exactness": { - "yes": 2, - "no": 2, - "maybe": 2 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - } - }, - "responses_unknown_stream_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [ - "OpenAI::Models::Responses::ResponseAudioDoneEvent", - "OpenAI::Models::Responses::ResponseAudioTranscriptDoneEvent" - ], - "best_ranked_fallbacks": [ - "OpenAI::Models::Responses::ResponseReasoningSummaryPartDoneEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Responses::ResponseReasoningSummaryPartDoneEvent", - "selected_exactness": { - "yes": 3, - "no": 4, - "maybe": 1 - }, - "serialized_body": { - "sequence_number": 9, - "type": "future.unmodeled.event" - } - }, - "beta_unknown_stream_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [ - "OpenAI::Models::Beta::BetaResponseErrorEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaResponseErrorEvent", - "selected_exactness": { - "yes": 3, - "no": 2, - "maybe": 2 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - } - }, - "beta_unknown_stream_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [ - "OpenAI::Models::Beta::BetaResponseAudioDoneEvent", - "OpenAI::Models::Beta::BetaResponseAudioTranscriptDoneEvent" - ], - "best_ranked_fallbacks": [ - "OpenAI::Models::Beta::BetaResponseReasoningSummaryPartDoneEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Beta::BetaResponseReasoningSummaryPartDoneEvent", - "selected_exactness": { - "yes": 4, - "no": 4, - "maybe": 1 - }, - "serialized_body": { - "sequence_number": 9, - "type": "future.unmodeled.event" - } - }, - "realtime_unknown_server_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [], - "uniquely_resolvable": false, - "compatibility_selected": "Hash", - "selected_error": "ArgumentError", - "selected_exactness": { - "yes": 0, - "no": 1, - "maybe": 0 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - }, - "public_boundary": { - "available": false - } - }, - "realtime_unknown_server_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "explicitly_unknown", - "matching_variants": 0, - "value": "future.unmodeled.event" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [ - "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent" - ], - "uniquely_resolvable": false, - "compatibility_selected": "OpenAI::Models::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent", - "selected_exactness": { - "yes": 4, - "no": 2, - "maybe": 1 - }, - "serialized_body": { - "type": "future.unmodeled.event", - "sequence_number": 9 - }, - "public_boundary": { - "available": false - } - }, - "realtime_known_unique_server_event": { - "context": "response_stream", - "input_form": "wire", - "discriminator_evidence": { - "property": "type", - "state": "known_unique", - "matching_variants": 1, - "value": "conversation.created" - }, - "exact_candidates": [], - "structural_candidates": [], - "best_ranked_fallbacks": [], - "uniquely_resolvable": true, - "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", - "selected_exactness": { - "yes": 2, - "no": 2, - "maybe": 0 - }, - "serialized_body": { - "type": "conversation.created", - "event_id": "evt_1", - "conversation": { - "id": "conv_1", - "object": "realtime.conversation" - } - } - }, - "realtime_known_unique_server_event_symbols": { - "context": "response_stream", - "input_form": "ruby_symbols", - "discriminator_evidence": { - "property": "type", - "state": "known_unique", - "matching_variants": 1, - "value": "conversation.created" - }, - "exact_candidates": [ - "OpenAI::Models::Realtime::ConversationCreatedEvent" - ], - "structural_candidates": [ - "OpenAI::Models::Realtime::ConversationCreatedEvent", - "OpenAI::Models::Realtime::InputAudioBufferClearedEvent" - ], - "best_ranked_fallbacks": [], - "uniquely_resolvable": true, - "compatibility_selected": "OpenAI::Models::Realtime::ConversationCreatedEvent", - "selected_exactness": { - "yes": 6, - "no": 0, - "maybe": 0 - }, - "serialized_body": { - "conversation": { - "id": "conv_1", - "object": "realtime.conversation" - }, - "event_id": "evt_1", - "type": "conversation.created" - } - } -} diff --git a/test/openai/internal/type/union_characterization_test.rb b/test/openai/internal/type/union_characterization_test.rb index 97fec4771..739e8dcd3 100644 --- a/test/openai/internal/type/union_characterization_test.rb +++ b/test/openai/internal/type/union_characterization_test.rb @@ -4,14 +4,6 @@ require_relative "union_characterization" class OpenAI::Test::UnionCharacterizationTest < Minitest::Test - EXPECTED_PATH = File.expand_path("../../../fixtures/union_characterization/main.json", __dir__) - - def test_current_union_characterization_matches_the_recorded_main_baseline - expected = JSON.parse(File.read(EXPECTED_PATH)) - - assert_equal(expected, OpenAI::Test::UnionCharacterization.observe) - end - def test_ambiguous_and_unknown_values_are_not_labeled_uniquely_resolvable observations = OpenAI::Test::UnionCharacterization.observe ambiguous_ids = observations.keys.grep(/responses|beta/) @@ -23,6 +15,73 @@ def test_ambiguous_and_unknown_values_are_not_labeled_uniquely_resolvable end end + def test_valid_responses_message_has_multiple_exact_candidates + observation = OpenAI::Test::UnionCharacterization.observe.fetch( + "responses_typed_message_without_type_symbols" + ) + + assert_equal("missing", observation.dig("discriminator_evidence", "state")) + assert_equal( + [ + "OpenAI::Models::Responses::EasyInputMessage", + "OpenAI::Models::Responses::ResponseInputItem::Message", + "OpenAI::Models::Responses::ResponseInputItem::CompactionTrigger" + ], + observation.fetch("exact_candidates") + ) + assert_equal("OpenAI::Models::Responses::EasyInputMessage", observation.fetch("compatibility_selected")) + end + + def test_duplicate_message_tag_does_not_make_the_responses_message_unique + observation = OpenAI::Test::UnionCharacterization.observe.fetch( + "responses_typed_message_with_duplicate_tag_symbols" + ) + + assert_equal("known_duplicate", observation.dig("discriminator_evidence", "state")) + assert_equal(3, observation.dig("discriminator_evidence", "matching_variants")) + assert_equal(2, observation.fetch("exact_candidates").length) + refute(observation.fetch("uniquely_resolvable")) + end + + def test_request_shorthand_and_unknown_fields_retain_their_wire_representation + observations = OpenAI::Test::UnionCharacterization.observe + responses_body = observations.fetch("responses_typed_message_without_type_symbols").fetch("serialized_body") + beta_body = observations.fetch("beta_typed_message_without_type").fetch("serialized_body") + unknown_field_body = observations.fetch("responses_typed_message_with_unknown_field").fetch("serialized_body") + + expected_message = { + "role" => "user", + "content" => [{"type" => "input_text", "text" => "hello"}] + } + assert_equal({"input" => [expected_message], "model" => "gpt-4o"}, responses_body) + assert_equal(responses_body, beta_body) + assert_equal({"preserved" => true}, unknown_field_body.dig("input", 0, "future_field")) + end + + def test_explicitly_unknown_stream_tags_retain_current_fallbacks + expected_selections = { + "responses_unknown_stream_event" => "OpenAI::Models::Responses::ResponseErrorEvent", + "responses_unknown_stream_event_symbols" => "OpenAI::Models::Responses::ResponseImageGenCallPartialImageEvent", + "beta_unknown_stream_event" => "OpenAI::Models::Beta::BetaResponseErrorEvent", + "beta_unknown_stream_event_symbols" => "OpenAI::Models::Beta::BetaResponseImageGenCallPartialImageEvent" + } + observations = OpenAI::Test::UnionCharacterization.observe + + expected_selections.each do |id, selected| + observation = observations.fetch(id) + + assert_equal("explicitly_unknown", observation.dig("discriminator_evidence", "state"), id) + assert_empty(observation.fetch("exact_candidates"), id) + assert_equal([selected], observation.fetch("best_ranked_fallbacks"), id) + assert_equal(selected, observation.fetch("compatibility_selected"), id) + assert_equal( + {"type" => "future.unmodeled.event", "sequence_number" => 9}, + observation.fetch("serialized_body"), + id + ) + end + end + def test_realtime_boundary_preserves_an_explicitly_unknown_event cases = OpenAI::Test::UnionCharacterization.observe @@ -34,4 +93,24 @@ def test_realtime_boundary_preserves_an_explicitly_unknown_event assert_equal("future.unmodeled.event", boundary.dig("body", "type")) end end + + def test_known_realtime_tag_selects_the_unique_discriminator_variant + observations = OpenAI::Test::UnionCharacterization.observe + expected_exact_candidates = { + "realtime_known_unique_server_event" => [], + "realtime_known_unique_server_event_symbols" => ["OpenAI::Models::Realtime::ConversationCreatedEvent"] + } + + expected_exact_candidates.each do |id, exact_candidates| + observation = observations.fetch(id) + + assert_equal("known_unique", observation.dig("discriminator_evidence", "state"), id) + assert_equal(exact_candidates, observation.fetch("exact_candidates"), id) + assert_equal( + "OpenAI::Models::Realtime::ConversationCreatedEvent", + observation.fetch("compatibility_selected"), + id + ) + end + end end From e4cc7dcd44f4f1e3e33338a8579fdf9648eb828d Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Fri, 21 Aug 2026 04:54:53 +0000 Subject: [PATCH 3/3] fix: harden union characterization scripts --- scripts/union-characterization | 6 +- scripts/union-characterization-differential | 22 ++++- .../fixtures/union_characterization/README.md | 2 +- test/scripts/union_characterization_test.rb | 92 +++++++++++++++++++ 4 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 test/scripts/union_characterization_test.rb diff --git a/scripts/union-characterization b/scripts/union-characterization index 04aa5595d..2d347c689 100755 --- a/scripts/union-characterization +++ b/scripts/union-characterization @@ -2,7 +2,11 @@ # frozen_string_literal: true require "json" -require "openai" + +openai_lib = File.expand_path(ARGV.shift || File.expand_path("../lib", __dir__)) +abort("Usage: scripts/union-characterization [openai-lib-directory]") unless ARGV.empty? + +require File.join(openai_lib, "openai") require_relative "../test/openai/internal/type/union_characterization" diff --git a/scripts/union-characterization-differential b/scripts/union-characterization-differential index 4bd572d89..06322fdec 100755 --- a/scripts/union-characterization-differential +++ b/scripts/union-characterization-differential @@ -4,19 +4,33 @@ set -euo pipefail cd -- "$(dirname -- "$0")/.." +releases=(v0.78.0 v0.80.0) +missing_releases=() +for release in "${releases[@]}"; do + if ! git rev-parse --verify --quiet "refs/tags/${release}^{commit}" >/dev/null; then + missing_releases+=("$release") + fi +done + +if ((${#missing_releases[@]} > 0)); then + printf 'Missing required release tags: %s\n' "${missing_releases[*]}" >&2 + printf 'Fetch them with `git fetch --tags` before running this differential.\n' >&2 + exit 1 +fi + tmp_dir=$(mktemp -d) trap 'rm -rf -- "$tmp_dir"' EXIT -for release in v0.78.0 v0.80.0; do +for release in "${releases[@]}"; do mkdir "$tmp_dir/$release" - git archive "$release" | tar -x -C "$tmp_dir/$release" - ruby -I"$tmp_dir/$release/lib" scripts/union-characterization >"$tmp_dir/$release.json" + git archive "refs/tags/$release" | tar -x -C "$tmp_dir/$release" + ruby scripts/union-characterization "$tmp_dir/$release/lib" >"$tmp_dir/$release.json" done echo "Comparing published releases v0.78.0 and v0.80.0..." diff -u "$tmp_dir/v0.78.0.json" "$tmp_dir/v0.80.0.json" -ruby -Ilib scripts/union-characterization >"$tmp_dir/main.json" +ruby scripts/union-characterization >"$tmp_dir/main.json" echo "Published releases agree. Their differential from the current checkout follows:" if diff -u --label v0.78.0-and-v0.80.0 --label current-checkout \ diff --git a/test/fixtures/union_characterization/README.md b/test/fixtures/union_characterization/README.md index 0b57087dd..2f86f24cd 100644 --- a/test/fixtures/union_characterization/README.md +++ b/test/fixtures/union_characterization/README.md @@ -12,6 +12,6 @@ This corpus records current behavior; it does not declare every selected class t The fixtures deliberately pair wire-string hashes with idiomatic Ruby symbol hashes. Their different candidate sets are part of the observed contract. Focused tests retain the compatibility-sensitive selections and serialized bodies without checking in full generated output snapshots. -Run `scripts/union-characterization` to print every current observation. Run `scripts/union-characterization-differential` under the repository Ruby to generate observations from temporary v0.78.0 and v0.80.0 tag archives, verify that those published releases agree, and print their differential from the current checkout. +Run `scripts/union-characterization` to print every current observation using this checkout's `lib`. Run `scripts/union-characterization-differential` under the repository Ruby to generate observations from temporary v0.78.0 and v0.80.0 tag archives, verify that those published releases agree, and print their differential from the current checkout. The differential reports any unavailable release tags together before attempting to archive them; fetch the tags when working from a shallow or no-tags checkout. The two published baselines agree for this corpus. Current `main` retains the request behavior and serialized bodies, but newer generated Responses/Beta variants change which known event wins structural fallback for an explicit unknown stream tag. Current Realtime adds a public connection boundary that returns `UnknownServerEvent`, even though direct shared-union coercion can still select a known event for symbolized input. diff --git a/test/scripts/union_characterization_test.rb b/test/scripts/union_characterization_test.rb new file mode 100644 index 000000000..19197ea5e --- /dev/null +++ b/test/scripts/union_characterization_test.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require "json" +require "minitest/autorun" +require "open3" +require "pathname" +require "rbconfig" +require "tmpdir" + +class UnionCharacterizationScriptsTest < Minitest::Test + ROOT = Pathname(__dir__).join("../..").expand_path + CHARACTERIZATION_SCRIPT = ROOT.join("scripts/union-characterization").to_s + DIFFERENTIAL_SCRIPT = ROOT.join("scripts/union-characterization-differential").to_s + + def test_characterization_loads_the_checkout_before_other_load_paths + Dir.mktmpdir("openai-union-characterization-test") do |directory| + Pathname(directory).join("openai.rb").write("raise \"loaded non-checkout openai\"\n") + + stdout, stderr, status = Open3.capture3( + RbConfig.ruby, + "-I#{directory}", + CHARACTERIZATION_SCRIPT + ) + + assert_predicate(status, :success?, stderr) + assert_includes(JSON.parse(stdout), "responses_typed_message_without_type_symbols") + end + end + + def test_characterization_accepts_an_explicit_relative_lib_directory + stdout, stderr, status = Open3.capture3( + RbConfig.ruby, + CHARACTERIZATION_SCRIPT, + "lib", + chdir: ROOT + ) + + assert_predicate(status, :success?, stderr) + assert_includes(JSON.parse(stdout), "responses_typed_message_without_type_symbols") + end + + def test_differential_reports_all_missing_release_refs_before_archiving + Dir.mktmpdir("openai-union-characterization-git-test") do |directory| + initialize_repository(directory) + + git_dir = Pathname(directory).join(".git").to_s + stdout, stderr, status = Open3.capture3({"GIT_DIR" => git_dir}, DIFFERENTIAL_SCRIPT) + + refute_predicate(status, :success?) + assert_empty(stdout) + assert_includes(stderr, "Missing required release tags: v0.78.0 v0.80.0") + assert_includes(stderr, "git fetch --tags") + refute_includes(stderr, "not a valid object name") + end + end + + def test_differential_does_not_accept_branches_named_like_release_tags + Dir.mktmpdir("openai-union-characterization-branch-test") do |directory| + initialize_repository(directory) + run_git(directory, "commit", "--allow-empty", "--message", "initial") + run_git(directory, "branch", "v0.78.0") + run_git(directory, "branch", "v0.80.0") + + git_dir = Pathname(directory).join(".git").to_s + _stdout, stderr, status = Open3.capture3({"GIT_DIR" => git_dir}, DIFFERENTIAL_SCRIPT) + + refute_predicate(status, :success?) + assert_includes(stderr, "Missing required release tags: v0.78.0 v0.80.0") + end + end + + private + + def initialize_repository(directory) + _stdout, stderr, status = Open3.capture3("git", "init", "--quiet", directory) + assert_predicate(status, :success?, stderr) + end + + def run_git(directory, *arguments) + _stdout, stderr, status = Open3.capture3( + "git", + "-C", + directory, + "-c", + "user.name=Union Characterization Test", + "-c", + "user.email=union-characterization@example.test", + *arguments + ) + assert_predicate(status, :success?, stderr) + end +end