Skip to content

Update OpenAI to 2.9.1 #7349

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 1 commit into from
Mar 3, 2026
Merged

Update OpenAI to 2.9.1 #7349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion eng/packages/General.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="$(MicrosoftExtensionsVectorDataAbstractionsVersion)" />
<PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
<PackageVersion Include="ModelContextProtocol.Core" Version="0.4.0-preview.3" />
<PackageVersion Include="OpenAI" Version="2.8.0" />
<PackageVersion Include="OpenAI" Version="2.9.1" />
<PackageVersion Include="Polly" Version="8.4.2" />
<PackageVersion Include="Polly.Core" Version="8.4.2" />
<PackageVersion Include="Polly.Extensions" Version="8.4.2" />
Expand Down
8 changes: 4 additions & 4 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIRealtimeExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ namespace OpenAI.Realtime;
[Experimental(DiagnosticIds.Experiments.AIOpenAIRealtime)]
public static class MicrosoftExtensionsAIRealtimeExtensions
{
/// <summary>Creates an OpenAI <see cref="ConversationFunctionTool"/> from an <see cref="AIFunctionDeclaration"/>.</summary>
/// <summary>Creates an OpenAI <see cref="RealtimeFunctionTool"/> from an <see cref="AIFunctionDeclaration"/>.</summary>
/// <param name="function">The function to convert.</param>
/// <returns>An OpenAI <see cref="ConversationFunctionTool"/> representing <paramref name="function"/>.</returns>
/// <returns>An OpenAI <see cref="RealtimeFunctionTool"/> representing <paramref name="function"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="function"/> is <see langword="null"/>.</exception>
public static ConversationFunctionTool AsOpenAIConversationFunctionTool(this AIFunctionDeclaration function) =>
OpenAIRealtimeConversationClient.ToOpenAIConversationFunctionTool(Throw.IfNull(function));
public static RealtimeFunctionTool AsOpenAIRealtimeFunctionTool(this AIFunctionDeclaration function) =>
OpenAIRealtimeConversationClient.ToOpenAIRealtimeFunctionTool(Throw.IfNull(function));
Comment thread
stephentoub marked this conversation as resolved.
}
12 changes: 8 additions & 4 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/MicrosoftExtensionsAIResponsesExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ public static ResponseResult AsOpenAIResponseResult(this ChatResponse response,
ConversationOptions = OpenAIClientExtensions.IsConversationId(response.ConversationId) ? new(response.ConversationId) : null,
CreatedAt = response.CreatedAt ?? default,
Id = response.ResponseId,
Instructions = options?.Instructions,
MaxOutputTokenCount = options?.MaxOutputTokens,
Model = response.ModelId ?? options?.ModelId,
ParallelToolCallsEnabled = options?.AllowMultipleToolCalls ?? true,
Expand All @@ -108,6 +107,11 @@ public static ResponseResult AsOpenAIResponseResult(this ChatResponse response,
Usage = OpenAIResponsesChatClient.ToResponseTokenUsage(response.Usage),
};

if (options?.Instructions is { Length: > 0 })
{
result.Instructions.Add(ResponseItem.CreateDeveloperMessageItem(options.Instructions));
}

foreach (var responseItem in OpenAIResponsesChatClient.ToOpenAIResponseItems(response.Messages, options))
{
result.OutputItems.Add(responseItem);
Expand All @@ -122,7 +126,7 @@ public static ResponseResult AsOpenAIResponseResult(this ChatResponse response,
/// <remarks>
/// <see cref="ResponseTool"/> does not derive from <see cref="AITool"/>, so it cannot be added directly to a list of <see cref="AITool"/>s.
/// Instead, this method wraps the provided <see cref="ResponseTool"/> in an <see cref="AITool"/> and adds that to the list.
/// The <see cref="IChatClient"/> returned by <see cref="OpenAIClientExtensions.AsIChatClient(ResponsesClient)"/> will
/// The <see cref="IChatClient"/> returned by <see cref="OpenAIClientExtensions.AsIChatClient(ResponsesClient, string)"/> will
/// be able to unwrap the <see cref="ResponseTool"/> when it processes the list of tools and use the provided <paramref name="tool"/> as-is.
/// </remarks>
public static void Add(this IList<AITool> tools, ResponseTool tool)
Expand All @@ -138,7 +142,7 @@ public static void Add(this IList<AITool> tools, ResponseTool tool)
/// <remarks>
/// <para>
/// The returned tool is only suitable for use with the <see cref="IChatClient"/> returned by
/// <see cref="OpenAIClientExtensions.AsIChatClient(ResponsesClient)"/> (or <see cref="IChatClient"/>s that delegate
/// <see cref="OpenAIClientExtensions.AsIChatClient(ResponsesClient, string)"/> (or <see cref="IChatClient"/>s that delegate
/// to such an instance). It is likely to be ignored by any other <see cref="IChatClient"/> implementation.
/// </para>
/// <para>
Expand All @@ -147,7 +151,7 @@ public static void Add(this IList<AITool> tools, ResponseTool tool)
/// <see cref="HostedFileSearchTool"/>, those types should be preferred instead of this method, as they are more portable,
/// capable of being respected by any <see cref="IChatClient"/> implementation. This method does not attempt to
/// map the supplied <see cref="ResponseTool"/> to any of those types, it simply wraps it as-is:
/// the <see cref="IChatClient"/> returned by <see cref="OpenAIClientExtensions.AsIChatClient(ResponsesClient)"/> will
/// the <see cref="IChatClient"/> returned by <see cref="OpenAIClientExtensions.AsIChatClient(ResponsesClient, string)"/> will
/// be able to unwrap the <see cref="ResponseTool"/> when it processes the list of tools.
/// </para>
/// </remarks>
Expand Down
2 changes: 1 addition & 1 deletion src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -681,7 +681,7 @@ ChatResponseFormatJson jsonFormat when OpenAIClientExtensions.StrictSchemaTransf
private static ChatReasoningEffortLevel? ToOpenAIChatReasoningEffortLevel(ReasoningEffort? effort) =>
effort switch
{
ReasoningEffort.None => new ChatReasoningEffortLevel("none"),
ReasoningEffort.None => ChatReasoningEffortLevel.None,
ReasoningEffort.Low => ChatReasoningEffortLevel.Low,
ReasoningEffort.Medium => ChatReasoningEffortLevel.Medium,
ReasoningEffort.High => ChatReasoningEffortLevel.High,
Expand Down
5 changes: 3 additions & 2 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,12 @@ public static IChatClient AsIChatClient(this ChatClient chatClient) =>

/// <summary>Gets an <see cref="IChatClient"/> for use with this <see cref="ResponsesClient"/>.</summary>
/// <param name="responseClient">The client.</param>
/// <param name="defaultModelId">The default model ID to use for the chat client.</param>
/// <returns>An <see cref="IChatClient"/> that can be used to converse via the <see cref="ResponsesClient"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="responseClient"/> is <see langword="null"/>.</exception>
[Experimental(DiagnosticIds.Experiments.AIOpenAIResponses)]
public static IChatClient AsIChatClient(this ResponsesClient responseClient) =>
new OpenAIResponsesChatClient(responseClient);
public static IChatClient AsIChatClient(this ResponsesClient responseClient, string? defaultModelId = null) =>
new OpenAIResponsesChatClient(responseClient, defaultModelId);
Comment thread
stephentoub marked this conversation as resolved.

/// <summary>Gets an <see cref="IChatClient"/> for use with this <see cref="AssistantClient"/>.</summary>
/// <param name="assistantClient">The <see cref="AssistantClient"/> instance to be accessed as an <see cref="IChatClient"/>.</param>
Expand Down
46 changes: 24 additions & 22 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIImageGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,14 @@
using System.IO;
using System.Linq;
using System.Net.Mime;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Text.Json.Serialization.Metadata;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;
using OpenAI;
using OpenAI.Images;

#pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields

namespace Microsoft.Extensions.AI;

/// <summary>Represents an <see cref="IImageGenerator"/> for an OpenAI <see cref="OpenAIClient"/> or <see cref="ImageClient"/>.</summary>
Expand Down Expand Up @@ -110,20 +105,11 @@ void IDisposable.Dispose()
/// <summary>Converts a <see cref="GeneratedImageCollection"/> to a <see cref="ImageGenerationResponse"/>.</summary>
private static ImageGenerationResponse ToImageGenerationResponse(GeneratedImageCollection generatedImages)
{
string contentType = "image/png"; // Default content type for images

// OpenAI doesn't expose the content type, so we need to read from the internal JSON representation.
// https://github.com/openai/openai-dotnet/issues/561
var additionalRawData = typeof(GeneratedImageCollection)
.GetProperty("SerializedAdditionalRawData", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance)
?.GetValue(generatedImages) as IDictionary<string, BinaryData>;

if (additionalRawData?.TryGetValue("output_format", out var outputFormat) ?? false)
{
var stringJsonTypeInfo = (JsonTypeInfo<string>)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(string));
var outputFormatString = JsonSerializer.Deserialize(outputFormat, stringJsonTypeInfo);
contentType = $"image/{outputFormatString}";
}
#pragma warning disable OPENAI001
string contentType = generatedImages.OutputFileFormat?.ToString() is { } outputFormat ?
$"image/{outputFormat}" :
"image/png"; // Default content type for images
#pragma warning restore OPENAI001

List<AIContent> contents = [];

Expand Down Expand Up @@ -175,15 +161,15 @@ private OpenAI.Images.ImageGenerationOptions ToOpenAIImageGenerationOptions(Imag

if (result.OutputFileFormat is null)
{
if (options?.MediaType?.Equals("image/png", StringComparison.OrdinalIgnoreCase) == true)
if (options?.MediaType?.Equals("image/png", StringComparison.OrdinalIgnoreCase) is true)
{
result.OutputFileFormat = GeneratedImageFileFormat.Png;
}
else if (options?.MediaType?.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) == true)
else if (options?.MediaType?.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) is true)
{
result.OutputFileFormat = GeneratedImageFileFormat.Jpeg;
}
else if (options?.MediaType?.Equals("image/webp", StringComparison.OrdinalIgnoreCase) == true)
else if (options?.MediaType?.Equals("image/webp", StringComparison.OrdinalIgnoreCase) is true)
{
result.OutputFileFormat = GeneratedImageFileFormat.Webp;
}
Expand All @@ -208,6 +194,22 @@ private ImageEditOptions ToOpenAIImageEditOptions(ImageGenerationOptions? option
{
ImageEditOptions result = options?.RawRepresentationFactory?.Invoke(this) as ImageEditOptions ?? new();

if (result.OutputFileFormat is null)
{
if (options?.MediaType?.Equals("image/png", StringComparison.OrdinalIgnoreCase) is true)
{
result.OutputFileFormat = GeneratedImageFileFormat.Png;
}
else if (options?.MediaType?.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) is true)
{
result.OutputFileFormat = GeneratedImageFileFormat.Jpeg;
}
else if (options?.MediaType?.Equals("image/webp", StringComparison.OrdinalIgnoreCase) is true)
{
result.OutputFileFormat = GeneratedImageFileFormat.Webp;
}
}

result.ResponseFormat ??= options?.ResponseFormat switch
{
ImageGenerationResponseFormat.Uri => GeneratedImageFormat.Uri,
Expand Down
8 changes: 4 additions & 4 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeConversationClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,16 @@ namespace Microsoft.Extensions.AI;
[Experimental(DiagnosticIds.Experiments.AIOpenAIRealtime)]
internal sealed class OpenAIRealtimeConversationClient
{
public static ConversationFunctionTool ToOpenAIConversationFunctionTool(AIFunctionDeclaration aiFunction, ChatOptions? options = null)
public static RealtimeFunctionTool ToOpenAIRealtimeFunctionTool(AIFunctionDeclaration aiFunction, ChatOptions? options = null)
{
bool? strict =
OpenAIClientExtensions.HasStrict(aiFunction.AdditionalProperties) ??
OpenAIClientExtensions.HasStrict(options?.AdditionalProperties);

return new ConversationFunctionTool(aiFunction.Name)
return new RealtimeFunctionTool(aiFunction.Name)
{
Description = aiFunction.Description,
Parameters = OpenAIClientExtensions.ToOpenAIFunctionParameters(aiFunction, strict),
FunctionDescription = aiFunction.Description,
FunctionParameters = OpenAIClientExtensions.ToOpenAIFunctionParameters(aiFunction, strict),
};
}
}
41 changes: 16 additions & 25 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Net.Mime;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;
Expand Down Expand Up @@ -49,28 +50,27 @@ private static readonly Func<ResponsesClient, GetResponseOptions, RequestOptions
null, [typeof(GetResponseOptions), typeof(RequestOptions)], null)
?.CreateDelegate(typeof(Func<ResponsesClient, GetResponseOptions, RequestOptions, AsyncCollectionResult<StreamingResponseUpdate>>));

// Workaround for https://github.com/openai/openai-dotnet/pull/874.
// The OpenAI library doesn't yet expose InputImageUrl as a public property, so we access it via reflection.
// Replace this with the actual public property once it's available (e.g., part.InputImageUrl).
private static readonly PropertyInfo? _inputImageUrlProperty =
Type.GetType("OpenAI.Responses.InternalItemContentInputImage, OpenAI")?.GetProperty("ImageUrl");

/// <summary>Metadata about the client.</summary>
private readonly ChatClientMetadata _metadata;

/// <summary>The underlying <see cref="ResponsesClient" />.</summary>
private readonly ResponsesClient _responseClient;

/// <summary>The default model ID to use for the chat client.</summary>
private readonly string? _defaultModelId;

/// <summary>Initializes a new instance of the <see cref="OpenAIResponsesChatClient"/> class for the specified <see cref="ResponsesClient"/>.</summary>
/// <param name="responseClient">The underlying client.</param>
/// <param name="defaultModelId">The default model ID to use for the chat client.</param>
/// <exception cref="ArgumentNullException"><paramref name="responseClient"/> is <see langword="null"/>.</exception>
public OpenAIResponsesChatClient(ResponsesClient responseClient)
public OpenAIResponsesChatClient(ResponsesClient responseClient, string? defaultModelId)
Comment thread
stephentoub marked this conversation as resolved.
{
_ = Throw.IfNull(responseClient);

_responseClient = responseClient;
_defaultModelId = defaultModelId;

_metadata = new("openai", responseClient.Endpoint, responseClient.Model);
_metadata = new("openai", responseClient.Endpoint, defaultModelId);
}

/// <inheritdoc />
Expand Down Expand Up @@ -737,7 +737,7 @@ private CreateResponseOptions AsCreateResponseOptions(ChatOptions? options, out
{
return new()
{
Model = _responseClient.Model,
Model = _defaultModelId,
};
}

Expand All @@ -753,7 +753,7 @@ private CreateResponseOptions AsCreateResponseOptions(ChatOptions? options, out

result.BackgroundModeEnabled ??= options.AllowBackgroundResponses;
result.MaxOutputTokenCount ??= options.MaxOutputTokens;
result.Model ??= options.ModelId ?? _responseClient.Model;
result.Model ??= options.ModelId ?? _defaultModelId;
result.Temperature ??= options.Temperature;
result.TopP ??= options.TopP;
result.ReasoningOptions ??= ToOpenAIResponseReasoningOptions(options.Reasoning);
Expand Down Expand Up @@ -864,7 +864,7 @@ ChatResponseFormatJson jsonFormat when OpenAIClientExtensions.StrictSchemaTransf

ResponseReasoningEffortLevel? effortLevel = reasoning.Effort switch
{
ReasoningEffort.None => new ResponseReasoningEffortLevel("none"),
ReasoningEffort.None => ResponseReasoningEffortLevel.None,
ReasoningEffort.Low => ResponseReasoningEffortLevel.Low,
ReasoningEffort.Medium => ResponseReasoningEffortLevel.Medium,
ReasoningEffort.High => ResponseReasoningEffortLevel.High,
Expand Down Expand Up @@ -965,7 +965,7 @@ internal static IEnumerable<ResponseItem> ToOpenAIResponseItems(IEnumerable<Chat
break;

case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
(parts ??= []).Add(ResponseContentPart.CreateInputImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(item)));
(parts ??= []).Add(ResponseContentPart.CreateInputImagePart(new Uri(dataContent.Uri), GetImageDetail(item)));
break;

case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
Expand Down Expand Up @@ -1287,20 +1287,11 @@ private static List<AIContent> ToAIContents(IEnumerable<ResponseContentPart> con
{
content = new DataContent(part.InputFileBytes, part.InputFileBytesMediaType ?? "application/octet-stream") { Name = part.InputFilename };
}
else if (_inputImageUrlProperty?.GetValue(part) is string inputImageUrl)
else if (part.InputImageUri is { } inputImageUrl)
{
if (inputImageUrl.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
{
content = new DataContent(inputImageUrl);
}
else if (Uri.TryCreate(inputImageUrl, UriKind.Absolute, out Uri? imageUri))
{
content = new UriContent(imageUri, "image/*");
}
else
{
content = null;
}
content = inputImageUrl.Scheme.Equals("data", StringComparison.OrdinalIgnoreCase) ?
new DataContent(inputImageUrl) :
new UriContent(inputImageUrl, MediaTypeMap.GetMediaType(inputImageUrl.AbsoluteUri) ?? "image/*");
}
else
{
Expand Down
Loading
Loading