From 811c24a723920f6239f01a4e0acc14437be7e2ca Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 26 Mar 2026 10:54:04 -0700 Subject: [PATCH] fix(google_genai): attach grounding_metadata to streaming responses --- .../test_google_search_grounding[stream].yaml | 182 ++++++++++++++++++ .../test_google_search_grounding[sync].yaml | 113 +++++++++++ ..._google_search_grounding_async[async].yaml | 121 ++++++++++++ ..._search_grounding_async[async_stream].yaml | 168 ++++++++++++++++ .../google_genai/test_google_genai.py | 145 ++++++++++++++ .../integrations/google_genai/tracing.py | 5 + 6 files changed, 734 insertions(+) create mode 100644 py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[stream].yaml create mode 100644 py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[sync].yaml create mode 100644 py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async].yaml create mode 100644 py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async_stream].yaml diff --git a/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[stream].yaml b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[stream].yaml new file mode 100644 index 00000000..1186f04c --- /dev/null +++ b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[stream].yaml @@ -0,0 +1,182 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, + Japan?"}], "role": "user"}], "tools": [{"googleSearch": {}}], "generationConfig": + {"maxOutputTokens": 300}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '183' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + x-goog-api-client: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:streamGenerateContent?alt=sse + response: + body: + string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"The + population of Tokyo\"}],\"role\": \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": + {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \" can be defined in different ways, depending on + the area being considered.\"}],\"role\": \"model\"},\"groundingMetadata\": + {}}],\"usageMetadata\": {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \" Here are a few estimates for 2026:\\n\\n* **Urban\"}],\"role\": + \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": {\"promptTokenCount\": + 11,\"totalTokenCount\": 11,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": + 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\ndata: + {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" Agglomeration:** + Approximately 36,953,600. Another source states 36,954,\"}],\"role\": \"model\"},\"groundingMetadata\": + {}}],\"usageMetadata\": {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"000. This estimate includes Tokyo's population in + addition to adjacent suburban areas.\\n* **\"}],\"role\": \"model\"},\"groundingMetadata\": + {}}],\"usageMetadata\": {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"City Proper:** In 2023, the population of the city + proper was over 14 million. Another source states that the population in 2023 + was 14,085,\"}],\"role\": \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": + {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"890.\\n* **Metropolitan Area:** A UN report states + that the population of the Tokyo metropolitan area is 33.4 million.\\n\"}],\"role\": + \"model\"},\"finishReason\": \"STOP\",\"groundingMetadata\": {\"searchEntryPoint\": + {\"renderedContent\": \"\\u003cstyle\\u003e\\n.container {\\n align-items: + center;\\n border-radius: 8px;\\n display: flex;\\n font-family: Google + Sans, Roboto, sans-serif;\\n font-size: 14px;\\n line-height: 20px;\\n padding: + 8px 12px;\\n}\\n.chip {\\n display: inline-block;\\n border: solid 1px;\\n + \ border-radius: 16px;\\n min-width: 14px;\\n padding: 5px 16px;\\n text-align: + center;\\n user-select: none;\\n margin: 0 8px;\\n -webkit-tap-highlight-color: + transparent;\\n}\\n.carousel {\\n overflow: auto;\\n scrollbar-width: none;\\n + \ white-space: nowrap;\\n margin-right: -12px;\\n}\\n.headline {\\n display: + flex;\\n margin-right: 4px;\\n}\\n.gradient-container {\\n position: relative;\\n}\\n.gradient + {\\n position: absolute;\\n transform: translate(3px, -9px);\\n height: + 36px;\\n width: 9px;\\n}\\n@media (prefers-color-scheme: light) {\\n .container + {\\n background-color: #fafafa;\\n box-shadow: 0 0 0 1px #0000000f;\\n + \ }\\n .headline-label {\\n color: #1f1f1f;\\n }\\n .chip {\\n background-color: + #ffffff;\\n border-color: #d2d2d2;\\n color: #5e5e5e;\\n text-decoration: + none;\\n }\\n .chip:hover {\\n background-color: #f2f2f2;\\n }\\n .chip:focus + {\\n background-color: #f2f2f2;\\n }\\n .chip:active {\\n background-color: + #d8d8d8;\\n border-color: #b6b6b6;\\n }\\n .logo-dark {\\n display: + none;\\n }\\n .gradient {\\n background: linear-gradient(90deg, #fafafa + 15%, #fafafa00 100%);\\n }\\n}\\n@media (prefers-color-scheme: dark) {\\n + \ .container {\\n background-color: #1f1f1f;\\n box-shadow: 0 0 0 1px + #ffffff26;\\n }\\n .headline-label {\\n color: #fff;\\n }\\n .chip + {\\n background-color: #2c2c2c;\\n border-color: #3c4043;\\n color: + #fff;\\n text-decoration: none;\\n }\\n .chip:hover {\\n background-color: + #353536;\\n }\\n .chip:focus {\\n background-color: #353536;\\n }\\n + \ .chip:active {\\n background-color: #464849;\\n border-color: #53575b;\\n + \ }\\n .logo-light {\\n display: none;\\n }\\n .gradient {\\n background: + linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n }\\n}\\n\\u003c/style\\u003e\\n\\u003cdiv + class=\\\"container\\\"\\u003e\\n \\u003cdiv class=\\\"headline\\\"\\u003e\\n + \ \\u003csvg class=\\\"logo-light\\\" width=\\\"18\\\" height=\\\"18\\\" + viewBox=\\\"9 9 35 35\\\" fill=\\\"none\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n + \ \\u003cpath fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M42.8622 + 27.0064C42.8622 25.7839 42.7525 24.6084 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 + 32.3041 33.9596 34.1222 32.1258 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 + 32.2571 42.8622 27.0064V27.0064Z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n \\u003cpath + fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 43.8555C30.9659 + 43.8555 34.8687 42.3195 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 + 37.0061 26.3109 37.0061C21.8282 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 + 39.9841 19.5712 43.8555 26.3109 43.8555V43.8555Z\\\" fill=\\\"#34A853\\\"/\\u003e\\n + \ \\u003cpath fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M16.6559 + 29.8904C16.3111 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 + 24.3733 16.6559 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 + 26.6146C9.06992 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 + 29.8904V29.8904Z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" + clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 + 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 + 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 + 16.2386 26.3109 16.2386V16.2386Z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n \\u003c/svg\\u003e\\n + \ \\u003csvg class=\\\"logo-dark\\\" width=\\\"18\\\" height=\\\"18\\\" + viewBox=\\\"0 0 48 48\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n + \ \\u003ccircle cx=\\\"24\\\" cy=\\\"23\\\" fill=\\\"#FFF\\\" r=\\\"22\\\"/\\u003e\\n + \ \\u003cpath d=\\\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 + 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n + \ \\u003cpath d=\\\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 + 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 + 4.06-2.85h.8z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath d=\\\"M15.59 + 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 + 1.53-6.64l1-.22 3.81 2.98.22 1.08z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath + d=\\\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 + 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n + \ \\u003c/svg\\u003e\\n \\u003cdiv class=\\\"gradient-container\\\"\\u003e\\u003cdiv + class=\\\"gradient\\\"\\u003e\\u003c/div\\u003e\\u003c/div\\u003e\\n \\u003c/div\\u003e\\n + \ \\u003cdiv class=\\\"carousel\\\"\\u003e\\n \\u003ca class=\\\"chip\\\" + href=\\\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQEJnHGvpZY9FHXRAKLap3Y-n-znEofWXB-8WZ_rCNsOCdgVtNzgHdiNwV9_uJvR8zu49TZdqIiH2Kw4LTC1iKfuZErL7ytdzRX1Oc4vrjwvQUzYgDy8K4Xqa1XVNrlnV4lZpa9o3Rvfd5pNIXcR3vmQJ100cP2plWlNa2ad0FFg2C5YKSmB1TNyCnkNtSjFKW19WmqT3NRZNSpWDgJmumkRGqoP\\\"\\u003ecurrent + population of Tokyo Japan\\u003c/a\\u003e\\n \\u003ca class=\\\"chip\\\" + href=\\\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQH8yiLATSIW0I_Br2IFHh7ZBLJph6YXK_5dlg6zNx2Weis4S4HT8GMbVH10J8vubrJc4x_dNBStzJA6JuYNL-F5QOEw6yNEk8jw8URuARBvRGIsIXJiKT5ejiY3z_YDPj0wUxBOzjL_3_pCVDKSsG3Ss2VgSLUfXJKyIomtumelU1N2evFDXHyVekm9blABymCS-YX5FyyklbNwZpvr\\\"\\u003eTokyo + Japan population 2026\\u003c/a\\u003e\\n \\u003c/div\\u003e\\n\\u003c/div\\u003e\\n\"},\"groundingChunks\": + [{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGfY6De1vUdeoQ_7yxMcRxmoU1jXGTFpmsCecctoPecepIwqMdbvgd1ePeHcaGydZBKoEO9wlq_RQmPCUy_EY6EvYNBwE54f5zZWN3w9wSM5wm3ZRvCrU4SDxW7_ovEwEVIes8-fncBJKbjH0QDdQ==\",\"title\": + \"worldpopulationreview.com\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGM_lsMwoLVzOx-O0xoK85P26DFdWGglx9-2cIyFRxmrr14HVAcW-btryDkExzGyZsBVmdV5NidXH8slM6JC3bzivhJ0eqsldjMW65NikgfvWxp3HQJuSDiNi8hrLJaFHsf28WJOIDd1irMmiA-DpK7Ls4s4Q3-Toh4VstxFQhpT0g8\",\"title\": + \"macrotrends.net\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQFhiyRWzYP55hXrg6JQ6i5ilTKqVY_4KOoj7Z4B_gpXdr9fkTuWhv7aGiAkc2raQPwmTqhCx6YxxwzDX6JH1TbNnqI0dqTYTc5Rn1gdVZR2kJT8RgOF7gCznuMpbgk=\",\"title\": + \"wikipedia.org\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGF1ztZcIRZZewcstfjPGwlmKrDiMVs9iIOmseEzKZanyJ2MDmxH0a8GIQrxiuTV3grwI2wM9MWurduSdZRUD4quUtAtWnZCfngk6iaS6W8Mu_2GG4PHygNIShbpZNqWfw7XhOY55oosA==\",\"title\": + \"datacommons.org\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQG9biqQ4AqplO6fKM3HR3c5txhgzSCe1Dtyoctv4pNS1layNT5Wpm9x-4syGf6kEuJdI9jtWiTgSqffa_NstcLVEL1t4eYTXt5O91tiL8SXVQlKFiJDJq1w9-3VX8Hr4SBc8nAtwUo=\",\"title\": + \"nippon.com\"}}],\"groundingSupports\": [{\"segment\": {\"endIndex\": 96,\"text\": + \"The population of Tokyo can be defined in different ways, depending on the + area being considered\"},\"groundingChunkIndices\": [0],\"confidenceScores\": + [0.5695162]},{\"segment\": {\"startIndex\": 134,\"endIndex\": 187,\"text\": + \"* **Urban Agglomeration:** Approximately 36,953,600\"},\"groundingChunkIndices\": + [0],\"confidenceScores\": [0.7969141]},{\"segment\": {\"startIndex\": 189,\"endIndex\": + 221,\"text\": \"Another source states 36,954,000\"},\"groundingChunkIndices\": + [1],\"confidenceScores\": [0.44627395]},{\"segment\": {\"startIndex\": 223,\"endIndex\": + 303,\"text\": \"This estimate includes Tokyo's population in addition to adjacent + suburban areas\"},\"groundingChunkIndices\": [0],\"confidenceScores\": [0.8972164]},{\"segment\": + {\"startIndex\": 305,\"endIndex\": 388,\"text\": \"* **City Proper:** In + 2023, the population of the city proper was over 14 million\"},\"groundingChunkIndices\": + [2],\"confidenceScores\": [0.9636004]},{\"segment\": {\"startIndex\": 390,\"endIndex\": + 454,\"text\": \"Another source states that the population in 2023 was 14,085,890\"},\"groundingChunkIndices\": + [3],\"confidenceScores\": [0.6583797]},{\"segment\": {\"startIndex\": 456,\"endIndex\": + 568,\"text\": \"* **Metropolitan Area:** A UN report states that the population + of the Tokyo metropolitan area is 33.4 million\"},\"groundingChunkIndices\": + [4],\"confidenceScores\": [0.92047733]}],\"retrievalMetadata\": {},\"webSearchQueries\": + [\"current population of Tokyo Japan\",\"Tokyo Japan population 2026\"]}}],\"usageMetadata\": + {\"promptTokenCount\": 10,\"candidatesTokenCount\": 162,\"totalTokenCount\": + 172,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"candidatesTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 162}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"cXHFaaixONjR6MEP7r7FsQg\"}\r\n\r\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Disposition: + - attachment + Content-Type: + - text/event-stream + Date: + - Thu, 26 Mar 2026 17:48:35 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1976 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[sync].yaml b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[sync].yaml new file mode 100644 index 00000000..f0e9ae1c --- /dev/null +++ b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding[sync].yaml @@ -0,0 +1,113 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, + Japan?"}], "role": "user"}], "tools": [{"googleSearch": {}}], "generationConfig": + {"maxOutputTokens": 300}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '183' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + x-goog-api-client: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/81aa0/bTBb+3l8xyqtXaitsxp7xjarSBhMgFEoSIFyWV2hiTxwTxza2c63473vG + jk0SKG23VLsxijO3c3nOM2fGY769Q6jmsND1XZbxtLaD/g01CH3Lv0VbFGY8zKChrILKmCXZU9/i + 823lN3TJ+EwMqp0POIqjeBywzI9CFPXReTScR8hPEU8zfwRaXeSHyPX7fZ6AJjRl8xS5POZgVOgh + GJSBDJZwhsCY1HehmyvfhrfhR1D08aPtZ3PUSqKYJ+i9SmB84qYfdj5+RPU4TqJZriOYI0tW0cgP + AjBDLscWtpzwDIZHgZ+KUacTEKTQsi/IxCr5UA05AEMy6FEMrQuz3o9KARkLc0tfUE/0LUujWxhj + 4S7I1EFkbQW0x+r3P1tPUCdRwAWOo8jlQdn9sexQ6/uhnw46nKVRKLqdnZ+2alWrl0TjHEXwkEGA + 2XoYU84SZ9AIs2TeivyNIAvdEAIBtl1xoHY7xpg4aTYPeP6T34ayoAjzQ8Dk222IEAt8L5T8jI/S + HeTAOJ58EvW9KAFpUsJcfwwtZjzLq10/jQM230H9gBc1fZAn9RngD7UHUeQFHJ2xMN1CnagXZdEW + SqEkpTzx+08DUn/BdyBuS7EBGCQNuO8Nsh1Ae1kbM1fAkWtHiprXPgoXBn5cWF+Z44e5iF4QOcMV + +3dQCnF2kbKUuOGVoi/rR34oTX03G6zYVGnXhPayp5gpUg7aGlxj8A98DLgDDoRRyAuxLPF86Igr + /KQp7w39TMpYLA3A3UC4LDlREIGtWQJIwWwFsaWnDDgBYgtvI+B6P4imO4iNsyiXlzpAuaDHktL8 + Svd0AEGVQJzDRe00YfGKTVJSYC2twDrgzBUobkBbRXp9JK3GeQJOsFnaoFYcpb5IIzso4SKhTPjG + gM1urAfRGmeF/TkY/SgZLXEBCfw9iWdbSLLi2Ye8T8kYUkZnCYJV2vavEXd9mPJxwiFfpQXQUuoM + +AhQycH/UFixOS+AK8wZFlOyjM9ffSauT8v2aCalA+aKeOD8Apahv3DxKbj+mIsugZUC1itjiVAp + VOmLa6X7E71fNiL/VEbkhC7bXFVcn9YVaFxcy8qcvy53ooQVoFeEeVK+MxBMe80EVVybo/qRM05/ + eRRzBDNeGeaa4nrZ354urhWRQeRFksuSYSmwovGGn+scXFW8k2cjmFFll/cWdrm3VYYfKdrfVQHW + BwXjvz+Ukn/IOWHbr1BuhR3fpVzBCFX/acqV/PkpvqmOuF7GnzgUU/LpZelvRDaiwaX/KtleHvVD + slGdmtR62VmQaGi9TbLlSeRPsK2I/JJtReEFthVL/PbaGl/Uuf4EOQFL08+3tYppt7WyE0LPu5Ws + We1V9ksnXtXvyfHbWpFzoVIxoVBk5LI08fl0N5pB0UIWIhr8QW0ftmpQJSCC0mwUhLnuLIt3tren + 06k8JXKUeNsqpNFtULtuTWlPzLJBLkpKxgEHAXzCw8h1QaQT+PHzWhdKJ1SVTV1VkWrIGOvUrsqa + bJjEQlA2NFVDKpV1bFJR1qhpIJXI1LCsQ9gHEgVbXYJlRdPNQ6LJmmkZNtwVU1UQUWWCKdyJbGmW + jgiVFRXkQ72iaiYAIBNKzS6xZN3U6SExZENViE2xbJmwjSW6rBsGKs2CYapmKGjD7O7yfvOE5l9U + NbV9elvbfjuwls4iSmRT0zQbnLZ0rSoL50zdzK0ligXlwhlUOEeO1502xHgAyxROEsMCb0xZJ1gX + 48AbBZX6lmVbVWRTNdWqXTFlnOshAiwoA1ga2KNasoX1QwXMUyzaBbNgphJbIRBLPTfHMiEmiiUD + lmpl/oZ73eV9FVRC66ZG3hTUFaNNC1NbEVaAMwCGKeqhrGCDilAbmmk+lWGcQnW7KmvASExRNR68 + Ngh5AoXIhJhmF0AzTUMr0bEt2aBQjwBcFRsqPGhh3RLBAO9VajyVl/qqsiVDrtdQOR6Ci4GRS7GC + C4QqxrECsQZiI4g1MFY53nC3u7yvgry/u2tj7Y8wF7SrYI6dg1uWEFEEhEgxZEURpqpAH6CVQApb + yjHQzaS6joA/pqFYtqC5BsAqMhZkXE4CS8CtVyQqinbJsWVrSUG4q4BdCdYyJscbsbJLgoMUVRfC + lxOgNHzDre7yvgpmo04J2QSzXCQgl/44r4sdys+mdbEFoSb8/UYid/zEgWdGR8hTqYjrXPwiqwzZ + 34dSIqrV13iSxx6yg5EnXlV3RDaXIIXriMrUkkTeKX5BMFUdYQnIKEFYJQCaSrJqSeQQJhJWJpps + WQMIB3EkGSYbcB36QOoBWRKRsRCj6RMQH0AygtknW8YAgrr4pYxcJASxhOQLkFEHXgBNiu9ig6dS + QVdTdxQZJiEGA3QVLBIOWQaYRJSAygbkTNlQbEHsfA0xRPrQwVwzF2A6EvhDwF9BYgUcoBJQkmqS + KRM9kBUhCcMyA3Md4DI1cGXxS2mw8gTIjGVVYeADTK3iGy8v4WgAlmDIycIlCfKylONKJAiXVvyC + OQFDwAzI25oGDVCsGgIFoqSK4TnmpiiIebn4pXxS5AgQC6szBZJA+sTCeUEXYaWQaZkALNEl8SWA + VcWkpkQAC7pzXDdgRRWsKIe1dFQTYdVzLIo9OypVL35n4q5u3J4/gldz7bt913tsQ5fvlKut4kt1 + a7vM5WnFS9tH9tQLNuIipcDj0TJbpJAu4BkAnhaYX5xyyU4QjV3Zy4+T4DlptF0djkks9qUEHrES + 7mTb9Yub5nW7MYrGerjA02tjLx5q+8cH7bnPbhpfpoOHdKbXu66+51kX+sPcPrq4Zt3wy/3cagzZ + Bb7oTJ3M8VrqxL1gl/ZlGNmG3+4t7qZ32ZGaHnojb9og86Pp+Biegq6T+oU27u7jPcoXp+f+rtYa + NdLFwUGHBeNW0zwwDpKUts39K/XyQFpcH/OjvYZXT21tr9FajDSt1/T2rRMlsJyL08i4ur/Trevh + 2Ul7/y69ygwXD2PFI2xyQh/43Ze0NR9en558LuF0xkl+5PrCAe0WOmIxKx8G2GuBe17zdKz5uHKU + WQFuD8bh8Afnx1Pe2ziOzKvHiS+OId8kxvuuGqjDK79tzXbbocQ65sQ1KPc6Vjc6OT+fqVL21b+z + 7WGjbfScO0yvsp4xYSeDcPd00b0K234rnIxVaYiVfnh5U6/Xg6Oe7l31pK4+rXeuUs069lIywXV3 + t/Hw8EVjvT3Fx3b/4Kx7pw8b2tEpG+727vv1r6SftKJ4ftOjn2tbmz5nflacAU+jJHCfYpVwsWwK + P2trQx5XD5W3/rcQB7aHo5PGfNZiUoxv3OBipiaG3vOjWV+b4oeRxhLeusPE80nSsSaNs1F9eHx4 + fdBstrqH7UU9bF5ee5cHl0GQnBtXevPe9lp8NKa9zsOXgTM/JO3Dcaz7xjScaNfNLxGeuiets4lx + PLsZvAalP/RjcaoithP/v/CdR4vW7l19yIdNPI2O9oLBfbCI25yG+uB+3E4W7Lp5f9KcSZP4NGld + 6X4rG3mpMb7+em+7jf3rK7d/P2lqft3rdjrDQ40oh7rXaSgDSJnT67E97KQPNp+dGf2LK0/h5uJY + jy9jojv9OGQnD1Gv6dZPaeBbkxNpfqbOwqn09b6jjiaX6ajRxteuejJudqLGpDO8iAbe59fYO2JO + EmXinUIqhzx7BfQXX4RUQJ2N4zj64funlHujzfdWyyawoBm6fAZtpvHc3j/74mrD7XX1GykSrPQd + vumo+OC18j8bUkBbH7SFDj9zouRlATKlimlSYqxL+knuv4JumrEkK/G18DN8V9BX6Pfh/6/f7f3f + AGwQlZp/GF+FWq8CbP0I4J9/Afo2uCq/j6ulifM09c/iar2Kq2o8b17H9e3eEr8N7urv465jw6Ca + /n3cX35/DU77fMKC1VfQq7tCWEbP8rWxPebQc1N37cd71CeESnMKU8R3bkptnDKPP3sHXgPgR3EG + onhoA4wifMoyYa38T8Rau6kt27MoY8Fak1U2rYhN90CpH6w6tfL+fRS5DNgwz5ebxtX5yuIJ8tes + WnPsn5dtfCNlprau7N2SdcW/IHR5kvrF/xp4fOSHPjxWY6kPT2IDCWMllwpBT2NY8XjTFf164eE+ + u2H28d1xd2E0W3pizHG9UXv3+O4/RPR8Xn8iAAA= + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Thu, 26 Mar 2026 17:48:33 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=2809 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async].yaml b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async].yaml new file mode 100644 index 00000000..5e28fde4 --- /dev/null +++ b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async].yaml @@ -0,0 +1,121 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, + Japan?"}], "role": "user"}], "tools": [{"googleSearch": {}}], "generationConfig": + {"maxOutputTokens": 300}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '183' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + x-goog-api-client: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/81aa2/iSLP+vr+ixerV2R3Fjt2+ZzXSAQKBhBBuYRJOVqixG+zB2MY219X897fa + xo4hmcyONrNn4gjo7urqqqeeKrcvf/2CUMkknuVYJKZR6QL9H/Qg9FfyycZ8L6ZeDANZF3QGJIyf + ZdO/vwq/QSSmWzap1Iz/J0KmvwhcukWxjyw6dTyKBv5858NI4Acrl8SO7yFrRZmA7W9QbFM0dWar + kEaIhBS5xLGQv4p51KDQZF0ETemmOJ1GsbNgPqCpHyLLmU5B1IuZMImQP02XROlvLGD14sl78j6A + rR8+pEO3NA79wHed6OLDB1QGIVHN1SIHMDIT/SQIQn+b9Lo7JEq8ghaO6zIjAuqDo3ymF0toQ0Ir + 0TcAn6AdBdR0iJv2g6MkRgsyp2gVJF6bTrx7NtYmFiJeccGixyBm8DhbO1/0ChyOaXhQUYZWtvzL + kTO0sR3TBudMd2XRKLFhkeMAi1soWoWhvwLvvRkKQjqlZswCc4ZgPEPHSiJH1iwuxxaSZC6S1DND + kc5UQeATU+49h81qJ3JRIXjMgNiPAaGCHlA+oakO+UwRce5rlcHVAWtpyJz85MQ28tfgoijnMQFb + HQuYEIGXLPDS2QEAsN8HKoKRidd+FKdr+qsIrcIJAz7hDkxjAhs/dC1YuVQg+pf8959nz+kR+i5l + 3F/4FnUz8S+ZQAkSwInsHqj2PSbWH9x1SvnoLAMb6EggKclx6kWUhKZd8+Jw1/Gdk8Rka1PPAuJb + 1TxvS08rQZDMKN65NPlJnzyepTWBRAzRX08eQsR1Zh4HIVlEF8iEeTT8g/VP/BC0cSGxnBWM6ME2 + 6bacKHDJ7gJNIauTnino46YEMIfeK9+fuRT1iQcs6fkTP/bPUAQtLqKhM32eEDl7egGxOqh1wSDO + ps7Mji8gUofegFgMjmR1JOKk9wtzwXaC1PrcHMdLVExc35wX7L9AEbDZQuJB44lXonroXzget3Gs + 2C7YlK+usNUzSVbduAS0I7hW4B/46EKKXCAPyJWqJeHMAUEhx4/b0MncibmYBJwN7rrMZc70XR9s + jUNACiosqM08ZUkEalNvGb+nrr+5QGQV+4m+yATKuRMSZubna0N6x5QDdSZlvZuQBAWbuDDFmivA + alNiMRRPoM0jfTxTzufNGJxgM3dCrcCPHJbEF5CILJ3X9GTCqRiZQLRWcWp/AgYU9MUBF9BAf5OC + 7RnijGD7eyKTMUbKonMAwchs+98FtRyCfkuqVxilQHORadMFoJKA/3tqxWleAFeIOU9TMovPr1PC + jj8O4/6Wi6BOs3gIyQEsQ78K6V/K9S+J6gxYziWTLJYIZUrFKTsK4s/0ft2I5C83IiF0NmZhdvxx + vIBC2XHoTPhrUdMPSQp6TpjnxS/spJK+YQJmx+msqW9C/fzeWcRkzHhjmqWz43V/Jyo7Cipdf+Zz + FgnnmcKcxid+HnOwuPBFUo0gozKR3wzBorOzLPxIVP6TNwSIuiD85/dM8zc5x2z7HsoV2PFVyqWM + wOrfplzGn7/FN2yy43X8JVMWZOmP17W/E9kkBQ71e8n2+qxvkk1WZV02XncWNGrK5JRsSRH5EWxL + I39gW9p4hW3pKf786Byf9lnOGpkuiaKPT6WcaU+lTAihl2IZa4pSmVy0nuVyz44/ldKaC52iDo20 + ImettUM3FX8LTQMZSFLgH3qnsD2DLgYRtLYL10vWjuPg4vx8s9nwG4n3w9k5hjJ6DsseW5PZExDY + 8TFVXLhyKSiga+r5lgUqTdcJXvZa0LqVMa+rGCOs8YKgytW8rfCaLhkI2pqCFYRlXhV0mbUVWddg + /87LmmE0sMpLomAMJYEXFVVvSAqv6IZWhW9RxyKSMC8JMnxLvKEYKpJkXsSgH/pFrOgAAC/Jsj6U + DF7VVbkhabyGRakqC7yhw9ZVUnlV01BmFkzDiiaiE7OHh+/RM5q/ylhX6vJT6fz9wDo4i2SJ1xVF + qYLThqrkbeacruqJtZJoQDt1BqXOSa1jpzU2H8DSmZOSZoA3Oq9KgsrmgTciytY7tKtY5HWs43xc + 1HkhWUdiYEEbwFLAHmzwhqA2RDBPNOQhmAWZKlXh+kxT1MQcQ4eYiAYPWOLc/BP3hofvIqiSXNYV + 6V1BLRitG4JcFZkV4AyAobN+aIuCJrNQa4quP7dhniir1bytACMFGeXzwWtNkp5BkXhJ0vUhgKbr + mpKhUzV4TYZ+BOBiQcNwISmoBgsGeI9l7bl9WC9vGzzUegVl8yG4AjDyoJZxQZJFrSVCrIHYCGIN + jBVbJ+4OD99FkOuVSlVQfghzYXUM5lQTcLMWkkQGIRI1XhSZqRjoA7RiSAmG2AK66bKqsut7XRON + KqO5AsCKvMDIeEgCg8Gt5iRKm9WMY4fRjILwjQG7DKxDTFonsapmBActWGXKDwmQGX7i1vDwXQSz + VpYl6RTM7CQBtfTbdZ3tUP5uWWdbEFmH/39QyE0nNOGa0WT6sMziumO/pCJD6nVohawbv8WTJPZQ + HbSk8GLVZNWcgxKuIpmXDY7VnfQXBBOrSOCAjByElQOgZY7HBic1IJEEca3whmFDOCST4yHZgOsg + A6UHdHESLzA1iroG9S4UI8g+3tBsCOr+uypyWhDYKSQ5AWll4AXQJP1MN3hYZnTVVVPkIQkFMEDF + YBFzyNDAJEl0ZV6DmslrYpUROzmHaKx8qGCunijQTQ78kcBfRmIRHJA5oKSscDovqS4vMk0CnGYg + 1wEuXQFX9t9VBnNPgMwCj0UCPkBqpZ/C4WCOumCJADWZucRBXeYSXCUOwqWkvyAnYAqYAXVbUWAA + mvmAK0KUMJueYK6zBsvL/XfVk7RGgFo4O8tAEiifAnOe0YVZyXQaOgArqRz7YMBiltSyxICFtRNc + T2BFOawogTVzVGFhVRMs0j07ypbe/5PELW7cXl6C57n2VdljiXMQ+Uo73yq+1ne0yzzcrXht+0ie + pWAjzkoKXB4dqkUE5QKuAeBqgTjpXS7edP2Vxc+S20lwnbQ4z2+OcSRwuBAusUJqxufl+1HzsXvV + 8Tv36/HNoz3aX886nVq9OrzvmlTC3tVAUaJ5qD4KTXG9IdupeEXU5rYuzxukP6naZXUzuPTJY10h + RGtMK8ZDcydGQlidTyay7cq1eStqN2r1G38UuqvpuibVr25bA8dvc8OHsr23FbE87txGDyDrhLPG + 1Gx192Fvv8ejT75+N/If3Y5g6sM+bSpe7TJcCJ7thWbNxaPa9m782G13ervy4Kpxy3Vas0F78/Fj + Bl96r7JwM5Tduz5EgvzLCNcsPFoMvEltftvZ3leXnNNe3I2iWt9twXb24WbY8DilHUoDodaVpref + BrPq/f2D3AoWuDU2Dblz05P7m165KtzMo3ZlHCmhcScEvctAmz5EXVvztjcVuV9ZjD7f1bTuZePO + 2bSE61r7vjvt7ALl86DWrHTuN6tKhwYa3j08TkKx06teTkfuuDtvDcujzlid3DVmZXzt39+Ll4tb + r34XVaxl/zHEm31vMBuNprtOfH9bcW/WwUwms0ocjPer2TPi5ipMnhwc38hO43BNAuK9RP9lZrzs + eb5v/KVwrzjHu2qvvPk3Hqps6OTkfm/SvQoddp/3XUJcvxaXZlewlU/lJjWutv2A02jbdxfby6Zy + 23o0BkY9EEXFWu+W7c1sbxL9mlPV217LGHjd0FvMdupm1NGHa7mnzYPdXBo4LdXQl63KcqX7lfsH + 01+uppPKZI31FudO7rx2WO3cr7YNexFety5JU/Mtucvt18tw82k4Wu7GzY+ls1OfYydOb7Int+Wf + QxVSti9hfpaOpnwp3rU/+3+FuKGVsdPe7T81PkcC99DuLL26MbCN9ry7NKvVydoKPz8OP981awGh + vbZyv2iPbxbkcV+7m2ix/0nCgh/XKt12ozze3G0XgTvu33aH5cZqNWrgyaJ/3e3JXVLdLT+b5dZd + LW6MlMtHa0K2HpnMhm17G10HPV2NRz29Np6XP74JrzN3AnYri+3hflpI603cbhjqtjLZxpv50lpS + 1RzHlfYoMrvL29i9qT40OqtlPzBi/HBVngWdTctoOvXW552s0brSr0aOFQ20vdCv9GRtyVn63W3s + 2LTtdP14uTVbg3Gv2bqzx6EwWzeU6m780J5as9ry/tLqvgN8rz5Hyl3ur4LA/+Yj14jOFqePag9D + 1LOankW3MKZpL639Ec9qT5w9XvSk7DXZs1V66h77E47af55ogX3OlD3bM2nf9MPXFcDVnK5h8VjP + 32TuG4hGMQnjDFNReQlqAXIs469i/kMeQP802BvY0FVd/6HgY1l+C3xJ0b8B/r/5lP6niYymayLG + 6g+NjKQKb0VGwco3IvOTv8rw8wRTVXWsqfIPjaaC3yxyiiF9NZr//I2P94Fa/OdQG7ImYV35sUgb + LxOjeAYXXxa847z511+LeZ/w4HcIj6ppgia/Uddef2MH6oRD18QtvnRTvEyDPWw/2Zh2VxQkTxcv + ffOa8WiPWHr1ov4ZxMzg1Fj2mRhbWkVkRl+8F1SC884iiEEn9aqANCOCeKi8hXf7jseNw5aklKTe + 0RjYcvZCcXQJyzpu0fHCW0kL3yIukC5J9trDoOAtLHBk15Frf75u5XstZuDj1X45cDN9M2tIw8hJ + X8Ga0YXjORzmBW7qksjmBEFM1AIzogDqFW1aTM56aNSJ1RJrRhh8vjU7wXTQ0rt66Zcvv/wXEQGl + UEopAAA= + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Thu, 26 Mar 2026 17:48:40 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=3192 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async_stream].yaml b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async_stream].yaml new file mode 100644 index 00000000..9c8da45e --- /dev/null +++ b/py/src/braintrust/integrations/google_genai/cassettes/test_google_search_grounding_async[async_stream].yaml @@ -0,0 +1,168 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "What is the current population of Tokyo, + Japan?"}], "role": "user"}], "tools": [{"googleSearch": {}}], "generationConfig": + {"maxOutputTokens": 300}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '183' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + x-goog-api-client: + - google-genai-sdk/1.68.0 gl-python/3.13.3 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-001:streamGenerateContent?alt=sse + response: + body: + string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"The + population of Tokyo\"}],\"role\": \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": + {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"eHHFaZSOLtjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \" depends on the area being considered.\\n\\n* **City\"}],\"role\": + \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": {\"promptTokenCount\": + 11,\"totalTokenCount\": 11,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": + 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": \"eHHFaZSOLtjR6MEP7r7FsQg\"}\r\n\r\ndata: + {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \" Proper (23 wards):** + Approximately 14,195,73\"}],\"role\": \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": + {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"eHHFaZSOLtjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"0 as of 2025. Another source states that the population + in 2023 was 14,0\"}],\"role\": \"model\"},\"groundingMetadata\": {}}],\"usageMetadata\": + {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"eHHFaZSOLtjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"85,890.\\n* **Greater Tokyo Area (metropolitan + area):** Approximately 36,954,\"}],\"role\": \"model\"},\"groundingMetadata\": + {}}],\"usageMetadata\": {\"promptTokenCount\": 11,\"totalTokenCount\": 11,\"promptTokensDetails\": + [{\"modality\": \"TEXT\",\"tokenCount\": 11}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": + \"eHHFaZSOLtjR6MEP7r7FsQg\"}\r\n\r\ndata: {\"candidates\": [{\"content\": + {\"parts\": [{\"text\": \"000 in 2026. This includes Tokyo and parts of six + neighboring prefectures.\\n\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"groundingMetadata\": + {\"searchEntryPoint\": {\"renderedContent\": \"\\u003cstyle\\u003e\\n.container + {\\n align-items: center;\\n border-radius: 8px;\\n display: flex;\\n font-family: + Google Sans, Roboto, sans-serif;\\n font-size: 14px;\\n line-height: 20px;\\n + \ padding: 8px 12px;\\n}\\n.chip {\\n display: inline-block;\\n border: + solid 1px;\\n border-radius: 16px;\\n min-width: 14px;\\n padding: 5px + 16px;\\n text-align: center;\\n user-select: none;\\n margin: 0 8px;\\n + \ -webkit-tap-highlight-color: transparent;\\n}\\n.carousel {\\n overflow: + auto;\\n scrollbar-width: none;\\n white-space: nowrap;\\n margin-right: + -12px;\\n}\\n.headline {\\n display: flex;\\n margin-right: 4px;\\n}\\n.gradient-container + {\\n position: relative;\\n}\\n.gradient {\\n position: absolute;\\n transform: + translate(3px, -9px);\\n height: 36px;\\n width: 9px;\\n}\\n@media (prefers-color-scheme: + light) {\\n .container {\\n background-color: #fafafa;\\n box-shadow: + 0 0 0 1px #0000000f;\\n }\\n .headline-label {\\n color: #1f1f1f;\\n + \ }\\n .chip {\\n background-color: #ffffff;\\n border-color: #d2d2d2;\\n + \ color: #5e5e5e;\\n text-decoration: none;\\n }\\n .chip:hover {\\n + \ background-color: #f2f2f2;\\n }\\n .chip:focus {\\n background-color: + #f2f2f2;\\n }\\n .chip:active {\\n background-color: #d8d8d8;\\n border-color: + #b6b6b6;\\n }\\n .logo-dark {\\n display: none;\\n }\\n .gradient {\\n + \ background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);\\n }\\n}\\n@media + (prefers-color-scheme: dark) {\\n .container {\\n background-color: #1f1f1f;\\n + \ box-shadow: 0 0 0 1px #ffffff26;\\n }\\n .headline-label {\\n color: + #fff;\\n }\\n .chip {\\n background-color: #2c2c2c;\\n border-color: + #3c4043;\\n color: #fff;\\n text-decoration: none;\\n }\\n .chip:hover + {\\n background-color: #353536;\\n }\\n .chip:focus {\\n background-color: + #353536;\\n }\\n .chip:active {\\n background-color: #464849;\\n border-color: + #53575b;\\n }\\n .logo-light {\\n display: none;\\n }\\n .gradient + {\\n background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\\n + \ }\\n}\\n\\u003c/style\\u003e\\n\\u003cdiv class=\\\"container\\\"\\u003e\\n + \ \\u003cdiv class=\\\"headline\\\"\\u003e\\n \\u003csvg class=\\\"logo-light\\\" + width=\\\"18\\\" height=\\\"18\\\" viewBox=\\\"9 9 35 35\\\" fill=\\\"none\\\" + xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" + clip-rule=\\\"evenodd\\\" d=\\\"M42.8622 27.0064C42.8622 25.7839 42.7525 24.6084 + 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 32.3041 33.9596 34.1222 32.1258 + 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 32.2571 42.8622 27.0064V27.0064Z\\\" + fill=\\\"#4285F4\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" + clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 43.8555C30.9659 43.8555 34.8687 42.3195 + 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 37.0061 26.3109 37.0061C21.8282 + 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 39.9841 19.5712 + 43.8555 26.3109 43.8555V43.8555Z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath + fill-rule=\\\"evenodd\\\" clip-rule=\\\"evenodd\\\" d=\\\"M16.6559 29.8904C16.3111 + 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 24.3733 16.6559 + 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 26.6146C9.06992 + 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 29.8904V29.8904Z\\\" + fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath fill-rule=\\\"evenodd\\\" + clip-rule=\\\"evenodd\\\" d=\\\"M26.3109 16.2386C28.85 16.2386 31.107 17.1164 + 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 + 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 + 16.2386 26.3109 16.2386V16.2386Z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n \\u003c/svg\\u003e\\n + \ \\u003csvg class=\\\"logo-dark\\\" width=\\\"18\\\" height=\\\"18\\\" + viewBox=\\\"0 0 48 48\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"\\u003e\\n + \ \\u003ccircle cx=\\\"24\\\" cy=\\\"23\\\" fill=\\\"#FFF\\\" r=\\\"22\\\"/\\u003e\\n + \ \\u003cpath d=\\\"M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 + 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z\\\" fill=\\\"#4285F4\\\"/\\u003e\\n + \ \\u003cpath d=\\\"M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 + 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 + 4.06-2.85h.8z\\\" fill=\\\"#34A853\\\"/\\u003e\\n \\u003cpath d=\\\"M15.59 + 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 + 1.53-6.64l1-.22 3.81 2.98.22 1.08z\\\" fill=\\\"#FBBC05\\\"/\\u003e\\n \\u003cpath + d=\\\"M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 + 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z\\\" fill=\\\"#EA4335\\\"/\\u003e\\n + \ \\u003c/svg\\u003e\\n \\u003cdiv class=\\\"gradient-container\\\"\\u003e\\u003cdiv + class=\\\"gradient\\\"\\u003e\\u003c/div\\u003e\\u003c/div\\u003e\\n \\u003c/div\\u003e\\n + \ \\u003cdiv class=\\\"carousel\\\"\\u003e\\n \\u003ca class=\\\"chip\\\" + href=\\\"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQG5AD7s4bUCvHLsnTSiynpks38yWGTOT5uHjcmOhjMw18VY-uuqBg8gf57l5fBMLvQLWMnydosGuzttAJrT1Ls7STvz1Vr4H16rGWvPkvnWAXxWiPF11m5wgiOZX096rwxKCF-qWIBwCLmGNo0eVQCpaVBrGXsRQ_JP8Ckq5s6XhASgFusqOeEjRmEEsn77m6TF7yN0QWl023TKQ_bVlQt3sBi1tA==\\\"\\u003ecurrent + population of Tokyo Japan\\u003c/a\\u003e\\n \\u003c/div\\u003e\\n\\u003c/div\\u003e\\n\"},\"groundingChunks\": + [{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQFL0JjVUmLk-ubelRWq1YauvpAkrCBHAgLGcVfwSxhJS6ilnfiU_Hy4A-3h3orvYOLjfIz_ctctXIDOl54wY0KVCqz5mqhRDQk0umjIu9bpXjaA0ELtdA5Xo85IKnT3kS8GBQuHjGYxm9aKcL4NG3A=\",\"title\": + \"worldpopulationreview.com\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQErTaDA36r1xGXViPI9RvbM5fNgHilauxNSNZGpizBntU8pOOjdIgAHL1ssU5zKfjDlyI0Uc5VWdXAaZu4P6FPIoLM8PzrX9y5vhT3Us0vp5TZ8-MoqGd_9GrfSHPrsD_Vq-GOk6HxCcXxJCM-3jA==\",\"title\": + \"wikipedia.org\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQGqKSBUPobKEujgizK183STTF8EIkZV71PwJ3LTN7Icyt1Eks9TgKArOMNiIj9SQ-uHxr1MFkRNS90ujEDRSRh1E-dZBQvK-XHMP-m0PTnpmL29DrDiPmoz9chULwY0sRCkz8G76Dv0OCg=\",\"title\": + \"datacommons.org\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQFG8zN-AEJUo_8jl8HudYXvIIdJVy_0dEnPJrGIOngbrkbvq_KsmKOAxPDWavIEssFioM6wilyFoRkPIa8pwiroYVUUW2bgDn4rJgxzi8PTowj1wXeVVNFdVKD_Qo7cTL1iY9rJD0cubpb1y_2Sdv9VL5pRfAS8dsTJ6KRLMJcIV5JAww==\",\"title\": + \"macrotrends.net\"}},{\"web\": {\"uri\": \"https://vertexaisearch.cloud.google.com/grounding-api-redirect/AUZIYQG3i2FzsE9ozw1Q5TkxckCEwGvkySqrbBHecwS-XouZ1KMPoJS0qFpsB0ZPk3yVjra3FiteuWcI-AH1-3aEgIGlW4_DOrWLjF3qMQbtfm2RjxG1ST_hA0yb8ab9JeVM\",\"title\": + \"wikipedia.org\"}}],\"groundingSupports\": [{\"segment\": {\"endIndex\": + 60,\"text\": \"The population of Tokyo depends on the area being considered\"},\"groundingChunkIndices\": + [0],\"confidenceScores\": [0.19867422]},{\"segment\": {\"startIndex\": 63,\"endIndex\": + 130,\"text\": \"* **City Proper (23 wards):** Approximately 14,195,730 as + of 2025\"},\"groundingChunkIndices\": [1],\"confidenceScores\": [0.25712863]},{\"segment\": + {\"startIndex\": 132,\"endIndex\": 196,\"text\": \"Another source states that + the population in 2023 was 14,085,890\"},\"groundingChunkIndices\": [2],\"confidenceScores\": + [0.30303612]},{\"segment\": {\"startIndex\": 198,\"endIndex\": 278,\"text\": + \"* **Greater Tokyo Area (metropolitan area):** Approximately 36,954,000 + in 2026\"},\"groundingChunkIndices\": [3],\"confidenceScores\": [0.45528665]},{\"segment\": + {\"startIndex\": 280,\"endIndex\": 340,\"text\": \"This includes Tokyo and + parts of six neighboring prefectures\"},\"groundingChunkIndices\": [4],\"confidenceScores\": + [0.9465313]}],\"retrievalMetadata\": {},\"webSearchQueries\": [\"current population + of Tokyo Japan\"]}}],\"usageMetadata\": {\"promptTokenCount\": 10,\"candidatesTokenCount\": + 112,\"totalTokenCount\": 122,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": + 10}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": + 112}]},\"modelVersion\": \"gemini-2.0-flash-001\",\"responseId\": \"eHHFaZSOLtjR6MEP7r7FsQg\"}\r\n\r\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Disposition: + - attachment + Content-Type: + - text/event-stream + Date: + - Thu, 26 Mar 2026 17:48:42 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1772 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/google_genai/test_google_genai.py b/py/src/braintrust/integrations/google_genai/test_google_genai.py index 9834fe30..0fc7308c 100644 --- a/py/src/braintrust/integrations/google_genai/test_google_genai.py +++ b/py/src/braintrust/integrations/google_genai/test_google_genai.py @@ -715,6 +715,151 @@ class TestModel(BaseModel): assert copied["context_file"] is attachment +GROUNDING_MODEL = "gemini-2.0-flash-001" + + +def _assert_grounding_metadata(span_output): + """Assert that grounding metadata is present and well-structured in span output.""" + # The grounding_metadata should be present on the first candidate + candidates = span_output.get("candidates", []) + assert candidates, "Expected candidates in span output" + + first_candidate = candidates[0] + grounding = first_candidate.get("grounding_metadata") + assert grounding is not None, ( + f"Expected grounding_metadata on first candidate, got keys: {list(first_candidate.keys())}" + ) + + # web_search_queries should be a non-empty list of strings + web_search_queries = grounding.get("web_search_queries") + assert web_search_queries, "Expected web_search_queries in grounding_metadata" + assert isinstance(web_search_queries, list) + assert all(isinstance(q, str) for q in web_search_queries) + + # grounding_chunks should contain search result snippets + grounding_chunks = grounding.get("grounding_chunks") + assert grounding_chunks, "Expected grounding_chunks in grounding_metadata" + assert isinstance(grounding_chunks, list) + + # grounding_supports should link response segments to chunks + grounding_supports = grounding.get("grounding_supports") + assert grounding_supports, "Expected grounding_supports in grounding_metadata" + assert isinstance(grounding_supports, list) + + +# Test: Google Search Grounding (Sync) +@pytest.mark.vcr +@pytest.mark.parametrize( + "mode", + ["sync", "stream"], +) +def test_google_search_grounding(memory_logger, mode): + """Test that Google Search grounding metadata is captured in span output.""" + assert not memory_logger.pop() + + client = Client() + start = time.time() + + if mode == "sync": + response = client.models.generate_content( + model=GROUNDING_MODEL, + contents="What is the current population of Tokyo, Japan?", + config=types.GenerateContentConfig( + tools=[types.Tool(google_search=types.GoogleSearch())], + max_output_tokens=300, + ), + ) + text = response.text + elif mode == "stream": + stream = client.models.generate_content_stream( + model=GROUNDING_MODEL, + contents="What is the current population of Tokyo, Japan?", + config=types.GenerateContentConfig( + tools=[types.Tool(google_search=types.GoogleSearch())], + max_output_tokens=300, + ), + ) + text = "" + for chunk in stream: + if chunk.text: + text += chunk.text + + end = time.time() + + # Verify response contains expected content + assert text + assert len(text) > 0 + + # Verify logging + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["metadata"]["model"] == GROUNDING_MODEL + assert "population" in str(span["input"]).lower() or "Tokyo" in str(span["input"]) + assert span["output"] + _assert_metrics_are_valid(span["metrics"], start, end) + + # Verify grounding metadata is captured + _assert_grounding_metadata(span["output"]) + + +# Test: Google Search Grounding (Async) +@pytest.mark.vcr +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mode", + ["async", "async_stream"], +) +async def test_google_search_grounding_async(memory_logger, mode): + """Test that Google Search grounding metadata is captured in async span output.""" + assert not memory_logger.pop() + + client = Client() + start = time.time() + + if mode == "async": + response = await client.aio.models.generate_content( + model=GROUNDING_MODEL, + contents="What is the current population of Tokyo, Japan?", + config=types.GenerateContentConfig( + tools=[types.Tool(google_search=types.GoogleSearch())], + max_output_tokens=300, + ), + ) + text = response.text + elif mode == "async_stream": + stream = await client.aio.models.generate_content_stream( + model=GROUNDING_MODEL, + contents="What is the current population of Tokyo, Japan?", + config=types.GenerateContentConfig( + tools=[types.Tool(google_search=types.GoogleSearch())], + max_output_tokens=300, + ), + ) + text = "" + async for chunk in stream: + if chunk.text: + text += chunk.text + + end = time.time() + + # Verify response contains expected content + assert text + assert len(text) > 0 + + # Verify logging + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["metadata"]["model"] == GROUNDING_MODEL + assert "population" in str(span["input"]).lower() or "Tokyo" in str(span["input"]) + assert span["output"] + _assert_metrics_are_valid(span["metrics"], start, end) + + # Verify grounding metadata is captured + _assert_grounding_metadata(span["output"]) + + class TestAutoInstrumentGoogleGenAI: """Tests for auto_instrument() with Google GenAI.""" diff --git a/py/src/braintrust/integrations/google_genai/tracing.py b/py/src/braintrust/integrations/google_genai/tracing.py index ed7f572e..89343bfd 100644 --- a/py/src/braintrust/integrations/google_genai/tracing.py +++ b/py/src/braintrust/integrations/google_genai/tracing.py @@ -314,6 +314,11 @@ def _aggregate_generate_content_chunks( candidate_dict["finish_reason"] = candidate.finish_reason if hasattr(candidate, "safety_ratings"): candidate_dict["safety_ratings"] = candidate.safety_ratings + if hasattr(candidate, "grounding_metadata") and candidate.grounding_metadata: + gm = candidate.grounding_metadata + candidate_dict["grounding_metadata"] = ( + gm.model_dump(exclude_none=True) if hasattr(gm, "model_dump") else gm + ) candidates.append(candidate_dict)