Port from Chat Completions API to Responses API

.devcontainer/Dockerfile

@@ -4,5 +4,5 @@ FROM mcr.microsoft.com/devcontainers/${IMAGE}
 ENV PYTHONUNBUFFERED 1
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends postgresql-client \
-     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+    && apt-get -y install --no-install-recommends postgresql-client zstd \
+     && apt-get clean -y && rm -rf /var/lib/apt/lists/*

.devcontainer/devcontainer.json

@@ -29,16 +29,13 @@
 			"extensions": [
 				"ms-python.python",
 				"ms-python.vscode-pylance",
-				"ms-python.vscode-python-envs",
 				"charliermarsh.ruff",
 				"mtxr.sqltools",
 				"mtxr.sqltools-driver-pg",
 				"esbenp.prettier-vscode",
 				"mechatroner.rainbow-csv",
 				"ms-vscode.vscode-node-azure-pack",
-				"esbenp.prettier-vscode",
 				"twixes.pypi-assistant",
-				"ms-python.vscode-python-envs",
 				"teamsdevapp.vscode-ai-foundry",
 				"ms-windows-ai-studio.windows-ai-studio"
 			],

.env.sample

@@ -5,16 +5,15 @@ POSTGRES_PASSWORD=postgres
 POSTGRES_DATABASE=postgres
 POSTGRES_SSL=disable
 
-# OPENAI_CHAT_HOST can be either azure, openai, ollama, or github:
+# OPENAI_CHAT_HOST can be either azure, openai, or ollama:
 OPENAI_CHAT_HOST=azure
-# OPENAI_EMBED_HOST can be either azure, openai, ollama, or github:
+# OPENAI_EMBED_HOST can be either azure, openai, or ollama:
 OPENAI_EMBED_HOST=azure
 # Needed for Azure:
 # You also need to `azd auth login` if running this locally
 AZURE_OPENAI_ENDPOINT=https://YOUR-AZURE-OPENAI-SERVICE-NAME.openai.azure.com
-AZURE_OPENAI_VERSION=2024-03-01-preview
-AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini
-AZURE_OPENAI_CHAT_MODEL=gpt-4o-mini
+AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-5.4
+AZURE_OPENAI_CHAT_MODEL=gpt-5.4
 AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-3-large
 AZURE_OPENAI_EMBED_MODEL=text-embedding-3-large
 AZURE_OPENAI_EMBED_DIMENSIONS=1024
@@ -35,9 +34,3 @@ OLLAMA_ENDPOINT=http://host.docker.internal:11434/v1
 OLLAMA_CHAT_MODEL=llama3.1
 OLLAMA_EMBED_MODEL=nomic-embed-text
 OLLAMA_EMBEDDING_COLUMN=embedding_nomic
-# Needed for GitHub Models:
-GITHUB_TOKEN=YOUR-GITHUB-TOKEN
-GITHUB_MODEL=openai/gpt-4o
-GITHUB_EMBED_MODEL=openai/text-embedding-3-large
-GITHUB_EMBED_DIMENSIONS=1024
-GITHUB_EMBEDDING_COLUMN=embedding_3l

.github/workflows/evaluate.yaml

@@ -34,7 +34,6 @@ jobs:
       OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }}
       OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }}
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
-      AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }}
       AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
       AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }}
       AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }}

.vscode/settings.json

@@ -36,5 +36,6 @@
         "htmlcov": true,
         ".mypy_cache": true,
         ".coverage": true
-    }
+    },
+    "python-envs.defaultEnvManager": "ms-python.python:system"
 }

AGENTS.md

@@ -8,3 +8,19 @@ When adding new azd environment variables, update:
 1. infra/main.parameters.json : Add the new parameter with a Bicep-friendly variable name and map to the new environment variable
 1. infra/main.bicep: Add the new Bicep parameter at the top, and add it to the `webAppEnv` object
 1. .github/workflows/azure-dev.yml: Add the new environment variable under `env` section. If it's a @secure variable in main.bicep, it should come from `secrets`, otherwise from `vars`.
+
+## Upgrading Python packages
+
+1. Update the version constraint in src/backend/pyproject.toml
+
+2. Re-compile src/backend/requirements.txt from the src folder:
+
+    ```shell
+    uv pip compile pyproject.toml -o requirements.txt --python-version 3.10
+    ```
+
+3. Reinstall with:
+
+    ```shell
+    python -m pip install -r src/backend/requirements.txt
+    ```

azure.yaml

@@ -42,7 +42,6 @@ pipeline:
     - OPENAI_CHAT_HOST
     - OPENAI_EMBED_HOST
     - AZURE_OPENAI_ENDPOINT
-    - AZURE_OPENAI_VERSION
     - AZURE_OPENAI_CHAT_DEPLOYMENT
     - AZURE_OPENAI_CHAT_MODEL
     - AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION

evals/generate_ground_truth.py

@@ -3,12 +3,10 @@
 import os
 from collections.abc import Generator
 from pathlib import Path
-from typing import Union
 
 from azure.identity import AzureDeveloperCliCredential, get_bearer_token_provider
 from dotenv_azd import load_azd_env
-from openai import AzureOpenAI, OpenAI
-from openai.types.chat import ChatCompletionToolParam
+from openai import OpenAI
 from sqlalchemy import create_engine, select
 from sqlalchemy.orm import Session
 
@@ -17,32 +15,30 @@
 logger = logging.getLogger("ragapp")
 
 
-def qa_pairs_tool(num_questions: int = 1) -> ChatCompletionToolParam:
+def qa_pairs_tool(num_questions: int = 1) -> dict:
     return {
         "type": "function",
-        "function": {
-            "name": "qa_pairs",
-            "description": "Send in question and answer pairs for a customer-facing chat app",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "qa_list": {
-                        "type": "array",
-                        "description": f"List of {num_questions} question and answer pairs",
-                        "items": {
-                            "type": "object",
-                            "properties": {
-                                "question": {"type": "string", "description": "The question text"},
-                                "answer": {"type": "string", "description": "The answer text"},
-                            },
-                            "required": ["question", "answer"],
+        "name": "qa_pairs",
+        "description": "Send in question and answer pairs for a customer-facing chat app",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "qa_list": {
+                    "type": "array",
+                    "description": f"List of {num_questions} question and answer pairs",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "question": {"type": "string", "description": "The question text"},
+                            "answer": {"type": "string", "description": "The answer text"},
                         },
-                        "minItems": num_questions,
-                        "maxItems": num_questions,
-                    }
-                },
-                "required": ["qa_list"],
+                        "required": ["question", "answer"],
+                    },
+                    "minItems": num_questions,
+                    "maxItems": num_questions,
+                }
             },
+            "required": ["qa_list"],
         },
     }
 
@@ -67,7 +63,6 @@ def source_retriever() -> Generator[str, None, None]:
         # for record in records:
         #    logger.info(f"Processing database record: {record.name}")
         #    yield f"## Product ID: [{record.id}]\n" + record.to_str_for_rag()
-        # await self.openai_chat_client.chat.completions.create(
 
 
 def source_to_text(source) -> str:
@@ -78,32 +73,29 @@ def answer_formatter(answer, source) -> str:
     return f"{answer} [{source['id']}]"
 
 
-def get_openai_client() -> tuple[Union[AzureOpenAI, OpenAI], str]:
+def get_openai_client() -> tuple[OpenAI, str]:
     """Return an OpenAI client based on the environment variables"""
-    openai_client: Union[AzureOpenAI, OpenAI]
+    openai_client: OpenAI
     OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST")
     if OPENAI_CHAT_HOST == "azure":
+        azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
         if api_key := os.getenv("AZURE_OPENAI_KEY"):
             logger.info("Using Azure OpenAI Service with API Key from AZURE_OPENAI_KEY")
-            openai_client = AzureOpenAI(
-                api_version=os.environ["AZURE_OPENAI_VERSION"],
-                azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
+            openai_client = OpenAI(
+                base_url=f"{azure_endpoint.rstrip('/')}/openai/v1/",
                 api_key=api_key,
             )
         else:
             logger.info("Using Azure OpenAI Service with Azure Developer CLI Credential")
             azure_credential = AzureDeveloperCliCredential(process_timeout=60, tenant_id=os.environ["AZURE_TENANT_ID"])
             token_provider = get_bearer_token_provider(azure_credential, "https://cognitiveservices.azure.com/.default")
-            openai_client = AzureOpenAI(
-                api_version=os.environ["AZURE_OPENAI_VERSION"],
-                azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
-                azure_ad_token_provider=token_provider,
+            openai_client = OpenAI(
+                base_url=f"{azure_endpoint.rstrip('/')}/openai/v1/",
+                api_key=token_provider,
             )
         model = os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"]
     elif OPENAI_CHAT_HOST == "ollama":
         raise NotImplementedError("Ollama is not supported. Switch to Azure or OpenAI.com")
-    elif OPENAI_CHAT_HOST == "github":
-        raise NotImplementedError("GitHub Models is not supported. Switch to Azure or OpenAI.com")
     else:
         logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY")
         openai_client = OpenAI(api_key=os.environ["OPENAICOM_KEY"])
@@ -123,18 +115,21 @@ def generate_ground_truth_data(num_questions_total: int, num_questions_per_sourc
         if len(qa) > num_questions_total:
             logger.info("Generated enough questions already, stopping")
             break
-        result = openai_client.chat.completions.create(
+        result = openai_client.responses.create(
             model=model,
-            messages=[
+            input=[
                 {"role": "system", "content": generate_prompt},
                 {"role": "user", "content": json.dumps(source)},
             ],
             tools=[qa_pairs_tool(num_questions=2)],
+            max_output_tokens=1000,
+            store=False,
         )
-        if not result.choices[0].message.tool_calls:
+        tool_calls = [item for item in result.output if item.type == "function_call"]
+        if not tool_calls:
             logger.warning("No tool calls found in response, skipping")
             continue
-        qa_pairs = json.loads(result.choices[0].message.tool_calls[0].function.arguments)["qa_list"]
+        qa_pairs = json.loads(tool_calls[0].arguments)["qa_list"]
         qa_pairs = [{"question": qa_pair["question"], "truth": qa_pair["answer"]} for qa_pair in qa_pairs]
         qa.extend(qa_pairs)
 

infra/main.bicep

@@ -131,7 +131,6 @@ param openAIEmbedHost string = 'azure'
 @secure()
 param openAIComKey string = ''
 
-param azureOpenAIAPIVersion string = '2024-03-01-preview'
 @secure()
 param azureOpenAIKey string = ''
 
@@ -385,10 +384,6 @@ var webAppEnv = union(azureOpenAIKeyEnv, openAIComKeyEnv, [
     name: 'AZURE_OPENAI_ENDPOINT'
     value: !empty(azureOpenAIEndpoint) ? azureOpenAIEndpoint : (deployAzureOpenAI ? openAI.outputs.endpoint : '')
   }
-  {
-    name: 'AZURE_OPENAI_VERSION'
-    value: openAIChatHost == 'azure' ? azureOpenAIAPIVersion : ''
-  }
 ])
 
 module web 'web.bicep' = {
@@ -613,7 +608,6 @@ output AZURE_OPENAI_RESOURCE_GROUP string = deployAzureOpenAI ? openAIResourceGr
 output AZURE_OPENAI_ENDPOINT string = !empty(azureOpenAIEndpoint)
   ? azureOpenAIEndpoint
   : (deployAzureOpenAI ? openAI.outputs.endpoint : '')
-output AZURE_OPENAI_VERSION string = azureOpenAIAPIVersion
 output AZURE_OPENAI_CHAT_DEPLOYMENT string = deployAzureOpenAI ? chatDeploymentName : ''
 output AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION string = deployAzureOpenAI ? chatDeploymentVersion : ''
 output AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY int = deployAzureOpenAI ? chatDeploymentCapacity : 0

infra/main.parameters.json

@@ -33,13 +33,13 @@
         "value": "${OPENAI_CHAT_HOST=azure}"
       },
       "chatModelName": {
-        "value": "${AZURE_OPENAI_CHAT_MODEL=gpt-4o-mini}"
+        "value": "${AZURE_OPENAI_CHAT_MODEL=gpt-5.4}"
       },
       "chatDeploymentName": {
-        "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini}"
+        "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-5.4}"
       },
       "chatDeploymentVersion":{
-        "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION=2024-07-18}"
+        "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION=2026-03-05}"
       },
       "chatDeploymentSku": {
         "value": "${AZURE_OPENAI_CHAT_DEPLOYMENT_SKU=GlobalStandard}"

src/backend/fastapi_app/__init__.py

@@ -2,12 +2,12 @@
 import os
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
-from typing import TypedDict, Union
+from typing import TypedDict
 
 import fastapi
 from azure.monitor.opentelemetry import configure_azure_monitor
 from dotenv import load_dotenv
-from openai import AsyncAzureOpenAI, AsyncOpenAI
+from openai import AsyncOpenAI
 from opentelemetry.instrumentation.openai import OpenAIInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -27,8 +27,8 @@
 class State(TypedDict):
     sessionmaker: async_sessionmaker[AsyncSession]
     context: FastAPIAppContext
-    chat_client: Union[AsyncOpenAI, AsyncAzureOpenAI]
-    embed_client: Union[AsyncOpenAI, AsyncAzureOpenAI]
+    chat_client: AsyncOpenAI
+    embed_client: AsyncOpenAI
 
 
 @asynccontextmanager

src/backend/fastapi_app/api_models.py

@@ -28,7 +28,6 @@ class ChatRequestOverrides(BaseModel):
     retrieval_mode: RetrievalMode = RetrievalMode.HYBRID
     use_advanced_flow: bool = True
     prompt_template: Optional[str] = None
-    seed: Optional[int] = None
 
 
 class ChatRequestContext(BaseModel):

src/backend/fastapi_app/dependencies.py

@@ -1,11 +1,11 @@
 import logging
 import os
 from collections.abc import AsyncGenerator
-from typing import Annotated, Optional, Union
+from typing import Annotated, Optional
 
 import azure.identity
 from fastapi import Depends, Request
-from openai import AsyncAzureOpenAI, AsyncOpenAI
+from openai import AsyncOpenAI
 from pydantic import BaseModel
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
 
@@ -17,7 +17,7 @@ class OpenAIClient(BaseModel):
     OpenAI client
     """
 
-    client: Union[AsyncOpenAI, AsyncAzureOpenAI]
+    client: AsyncOpenAI
     model_config = {"arbitrary_types_allowed": True}
 
 
@@ -51,26 +51,18 @@ async def common_parameters():
         openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text"
         openai_embed_dimensions = None
         embedding_column = os.getenv("OLLAMA_EMBEDDING_COLUMN") or "embedding_nomic"
-    elif OPENAI_EMBED_HOST == "github":
-        openai_embed_deployment = None
-        openai_embed_model = os.getenv("GITHUB_EMBED_MODEL") or "openai/text-embedding-3-large"
-        openai_embed_dimensions = int(os.getenv("GITHUB_EMBED_DIMENSIONS", 1024))
-        embedding_column = os.getenv("GITHUB_EMBEDDING_COLUMN") or "embedding_3l"
     else:
         openai_embed_deployment = None
         openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL") or "text-embedding-3-large"
         openai_embed_dimensions = int(os.getenv("OPENAICOM_EMBED_DIMENSIONS", 1024))
         embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN") or "embedding_3l"
     if OPENAI_CHAT_HOST == "azure":
-        openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT") or "gpt-4o-mini"
-        openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL") or "gpt-4o-mini"
+        openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT") or "gpt-5.4"
+        openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL") or "gpt-5.4"
     elif OPENAI_CHAT_HOST == "ollama":
         openai_chat_deployment = None
         openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL") or "phi3:3.8b"
         openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text"
-    elif OPENAI_CHAT_HOST == "github":
-        openai_chat_deployment = None
-        openai_chat_model = os.getenv("GITHUB_MODEL") or "openai/gpt-4o"
     else:
         openai_chat_deployment = None
         openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL") or "gpt-3.5-turbo"
@@ -84,10 +76,10 @@ async def common_parameters():
     )
 
 
-async def get_azure_credential() -> Union[
-    azure.identity.AzureDeveloperCliCredential, azure.identity.ManagedIdentityCredential
-]:
-    azure_credential: Union[azure.identity.AzureDeveloperCliCredential, azure.identity.ManagedIdentityCredential]
+async def get_azure_credential() -> (
+    azure.identity.AzureDeveloperCliCredential | azure.identity.ManagedIdentityCredential
+):
+    azure_credential: azure.identity.AzureDeveloperCliCredential | azure.identity.ManagedIdentityCredential
     try:
         if client_id := os.getenv("APP_IDENTITY_ID"):
             # Authenticate using a user-assigned managed identity on Azure