From e3c54df74a3376f511cb9a5adb446624f6e8cfc2 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:07:12 -0800 Subject: [PATCH 01/35] - --- examples/eval/README.md | 3 ++ examples/eval/pubspec.yaml | 41 +++++++++++++++++++ examples/eval/test/smoke_test.dart | 10 +++++ .../eval/test/test_infra/io_get_api_key.dart | 22 ++++++++++ 4 files changed, 76 insertions(+) create mode 100644 examples/eval/README.md create mode 100644 examples/eval/pubspec.yaml create mode 100644 examples/eval/test/smoke_test.dart create mode 100644 examples/eval/test/test_infra/io_get_api_key.dart diff --git a/examples/eval/README.md b/examples/eval/README.md new file mode 100644 index 000000000..50ff1fa99 --- /dev/null +++ b/examples/eval/README.md @@ -0,0 +1,3 @@ +# Eval + +Project for tests that evaluate genui against real AI models. diff --git a/examples/eval/pubspec.yaml b/examples/eval/pubspec.yaml new file mode 100644 index 000000000..e0874c82b --- /dev/null +++ b/examples/eval/pubspec.yaml @@ -0,0 +1,41 @@ +# Copyright 2025 The Flutter Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +name: eval +publish_to: "none" + +environment: + sdk: ">=3.10.0 <4.0.0" + flutter: ">=3.35.7 <4.0.0" + +resolution: workspace + +dependencies: + firebase_app_check: ^0.4.1+2 + firebase_core: ^4.2.1 + flutter: + sdk: flutter + genui: ^0.7.0 + google_cloud_ai_generativelanguage_v1beta: ^0.4.0 + google_cloud_protobuf: ^0.4.0 + google_cloud_rpc: ^0.4.0 + google_cloud_type: ^0.4.0 + + gpt_markdown: ^1.1.4 + intl: ^0.20.2 + json_schema_builder: ^0.1.3 + logging: ^1.3.0 + +dev_dependencies: + flutter_test: + sdk: flutter + integration_test: + sdk: flutter + network_image_mock: ^2.1.1 + +flutter: + uses-material-design: true + assets: + - assets/travel_images/ + - assets/booking_service/ diff --git a/examples/eval/test/smoke_test.dart b/examples/eval/test/smoke_test.dart new file mode 100644 index 000000000..c90000027 --- /dev/null +++ b/examples/eval/test/smoke_test.dart @@ -0,0 +1,10 @@ +import 'package:flutter_test/flutter_test.dart'; + +import 'test_infra/io_get_api_key.dart'; + +void main() { + test('smoke test', () { + final String key = apiKey(); + expect(key, isNotEmpty); + }); +} diff --git a/examples/eval/test/test_infra/io_get_api_key.dart b/examples/eval/test/test_infra/io_get_api_key.dart new file mode 100644 index 000000000..773b7f265 --- /dev/null +++ b/examples/eval/test/test_infra/io_get_api_key.dart @@ -0,0 +1,22 @@ +// Copyright 2025 The Flutter Authors. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +/// API key for Google Generative AI (only needed if using google backend). +/// Get an API key from https://aistudio.google.com/app/apikey +/// Specify this when running the app with "-D GEMINI_API_KEY=$GEMINI_API_KEY" +const String geminiApiKey = String.fromEnvironment('GEMINI_API_KEY'); + +String apiKey() { + String apiKey = geminiApiKey.isEmpty + ? Platform.environment['GEMINI_API_KEY'] ?? '' + : geminiApiKey; + if (apiKey.isEmpty) { + throw Exception( + '''Gemini API key is required when using google backend. Run the app with a GEMINI_API_KEY as a Dart environment variable, for example by running with -D GEMINI_API_KEY=\$GEMINI_API_KEY or set the GEMINI_API_KEY environment variable in your shell environment.''', + ); + } + return apiKey; +} From 65c6189321d570264b5721c4b6ba9d0d8e7bef8f Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:09:34 -0800 Subject: [PATCH 02/35] Update smoke_test.dart --- examples/eval/test/smoke_test.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/eval/test/smoke_test.dart b/examples/eval/test/smoke_test.dart index c90000027..06758cc83 100644 --- a/examples/eval/test/smoke_test.dart +++ b/examples/eval/test/smoke_test.dart @@ -1,3 +1,7 @@ +// Copyright 2025 The Flutter Authors. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'package:flutter_test/flutter_test.dart'; import 'test_infra/io_get_api_key.dart'; From 9f7343bf972b8e41dfc00766d08f0cae7a7d436a Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:13:06 -0800 Subject: [PATCH 03/35] - --- .github/workflows/flutter_packages.yaml | 2 +- examples/eval/pubspec.yaml | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/flutter_packages.yaml b/.github/workflows/flutter_packages.yaml index 6e75b69af..54b16ed72 100644 --- a/.github/workflows/flutter_packages.yaml +++ b/.github/workflows/flutter_packages.yaml @@ -50,7 +50,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | # Find all directories containing pubspec.yaml in packages, tools, and examples. - ALL_DIRS=$(find packages examples tool -name pubspec.yaml -not -path "*/.dart_tool/*" -exec dirname {} \;) + ALL_DIRS=$(find packages examples tool -name pubspec.yaml -not -path "*/.dart_tool/*" -not -path "examples/eval/*" -exec dirname {} \;) # Get the list of changed files. The method depends on the event type. if [[ "${{ github.event_name }}" == "pull_request" ]]; then diff --git a/examples/eval/pubspec.yaml b/examples/eval/pubspec.yaml index e0874c82b..ba4a423bb 100644 --- a/examples/eval/pubspec.yaml +++ b/examples/eval/pubspec.yaml @@ -12,30 +12,13 @@ environment: resolution: workspace dependencies: - firebase_app_check: ^0.4.1+2 - firebase_core: ^4.2.1 flutter: sdk: flutter genui: ^0.7.0 - google_cloud_ai_generativelanguage_v1beta: ^0.4.0 - google_cloud_protobuf: ^0.4.0 - google_cloud_rpc: ^0.4.0 - google_cloud_type: ^0.4.0 - - gpt_markdown: ^1.1.4 - intl: ^0.20.2 - json_schema_builder: ^0.1.3 - logging: ^1.3.0 dev_dependencies: flutter_test: sdk: flutter - integration_test: - sdk: flutter - network_image_mock: ^2.1.1 flutter: uses-material-design: true - assets: - - assets/travel_images/ - - assets/booking_service/ From a9f861f898c97dc4759004901941562269d4f5d1 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:18:08 -0800 Subject: [PATCH 04/35] Create eval.yaml --- .github/workflows/eval.yaml | 63 +++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/eval.yaml diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml new file mode 100644 index 000000000..33ae357e7 --- /dev/null +++ b/.github/workflows/eval.yaml @@ -0,0 +1,63 @@ +# Copyright 2025 The Flutter Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +name: Eval CI + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + # Tests may fail due to new dependency releases. + # Regular execution provides early detection of such regressions. + - cron: '0 * * * *' # hourly + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + # Prevents the job from running on forks + if: github.repository == 'flutter/genui' + name: examples/eval (${{ matrix.flutter_version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + flutter_version: [beta, stable] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 + with: + distribution: "zulu" + java-version: "17" + cache: "gradle" + - uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 + with: + channel: ${{ matrix.flutter_version }} + cache: true + - name: Print Flutter version + run: flutter --version + - name: Cache Pub dependencies + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 + with: + path: ${{ env.PUB_CACHE }} + key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.lock') }} + restore-keys: ${{ runner.os }}-pub- + - name: Install dependencies + working-directory: examples/eval + run: dart pub get + - name: Run tests + working-directory: examples/eval + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: flutter test --test-randomize-ordering-seed=random From 3b678ac4d3172e4c8b012a7d45e6c0dc274b2fb8 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:20:47 -0800 Subject: [PATCH 05/35] - --- examples/eval/test/smoke_test.dart | 1 + pubspec.yaml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/eval/test/smoke_test.dart b/examples/eval/test/smoke_test.dart index 06758cc83..95bd70dd4 100644 --- a/examples/eval/test/smoke_test.dart +++ b/examples/eval/test/smoke_test.dart @@ -10,5 +10,6 @@ void main() { test('smoke test', () { final String key = apiKey(); expect(key, isNotEmpty); + print('API Key: ${key.substring(0, 2)}...${key.substring(key.length - 2)}'); }); } diff --git a/pubspec.yaml b/pubspec.yaml index 4d780ea9f..543f0049d 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,13 +12,12 @@ environment: workspace: - examples/catalog_gallery + - examples/eval - examples/simple_chat - - examples/travel_app - examples/verdure/client - packages/genui - packages/genui_a2a - - packages/json_schema_builder - tool/fix_copyright - tool/release From fe8a7ff031eb067184a589eb56dd4037f5ce2805 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:21:07 -0800 Subject: [PATCH 06/35] Update pubspec.yaml --- pubspec.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pubspec.yaml b/pubspec.yaml index 543f0049d..9fc0c1d24 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -16,9 +16,11 @@ workspace: - examples/simple_chat - examples/travel_app - examples/verdure/client + - packages/genui - packages/genui_a2a - packages/json_schema_builder + - tool/fix_copyright - tool/release - tool/test_and_fix From 33c1b58135c4c54715fe21cb7ec769d8d7621b35 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:25:12 -0800 Subject: [PATCH 07/35] Update eval.yaml --- .github/workflows/eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 33ae357e7..34d4be633 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -60,4 +60,4 @@ jobs: working-directory: examples/eval env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: flutter test --test-randomize-ordering-seed=random + run: flutter test -D GEMINI_API_KEY=$GEMINI_API_KEY From 12d791b33815686a1cefa6eafe5063cf4a3d6d4b Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:29:37 -0800 Subject: [PATCH 08/35] Update eval.yaml --- .github/workflows/eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 34d4be633..baf13d4a2 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -60,4 +60,4 @@ jobs: working-directory: examples/eval env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: flutter test -D GEMINI_API_KEY=$GEMINI_API_KEY + run: flutter test --dart-define=GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }} From 2fa640f21f52018bd55518983d4cd7aa902be03e Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:31:50 -0800 Subject: [PATCH 09/35] Update eval.yaml --- .github/workflows/eval.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index baf13d4a2..19a7630f2 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -56,6 +56,11 @@ jobs: - name: Install dependencies working-directory: examples/eval run: dart pub get + - name: check api key + working-directory: examples/eval + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - name: Run tests working-directory: examples/eval env: From 68caeed2e2bea5e4f389ed7635c714699e95f62e Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:38:23 -0800 Subject: [PATCH 10/35] Update eval.yaml --- .github/workflows/eval.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 19a7630f2..0e70edea3 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -25,8 +25,8 @@ concurrency: jobs: test: - # Prevents the job from running on forks - if: github.repository == 'flutter/genui' + # Prevents the job from running on forks or fork PRs (which don't have access to secrets). + if: github.repository == 'flutter/genui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) name: examples/eval (${{ matrix.flutter_version }}) runs-on: ${{ matrix.os }} strategy: @@ -35,6 +35,8 @@ jobs: flutter_version: [beta, stable] os: [ubuntu-latest] steps: + - name: check api key + run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 with: @@ -56,11 +58,6 @@ jobs: - name: Install dependencies working-directory: examples/eval run: dart pub get - - name: check api key - working-directory: examples/eval - env: - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - name: Run tests working-directory: examples/eval env: From e0c5cab8a241f220a8e6cfb7233b22e0ec18e804 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:41:33 -0800 Subject: [PATCH 11/35] Update eval.yaml --- .github/workflows/eval.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 0e70edea3..3d865f56e 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -35,8 +35,6 @@ jobs: flutter_version: [beta, stable] os: [ubuntu-latest] steps: - - name: check api key - run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 with: @@ -47,6 +45,8 @@ jobs: with: channel: ${{ matrix.flutter_version }} cache: true + - name: check api key + run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies From c2d1a218d123d6173ebe0be14207d57c9690e88e Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Sun, 1 Mar 2026 20:48:56 -0800 Subject: [PATCH 12/35] Update eval.yaml --- .github/workflows/eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 3d865f56e..1ba9cb3d5 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -26,7 +26,7 @@ concurrency: jobs: test: # Prevents the job from running on forks or fork PRs (which don't have access to secrets). - if: github.repository == 'flutter/genui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + # if: github.repository == 'flutter/genui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) name: examples/eval (${{ matrix.flutter_version }}) runs-on: ${{ matrix.os }} strategy: From b2ea20e5c6ba9c082ac7afec94deb2b9053421b2 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 14:50:08 -0700 Subject: [PATCH 13/35] - --- .github/workflows/eval.yaml | 1 + examples/eval/test/smoke_test.dart | 3 ++- examples/eval/test/test_infra/io_get_api_key.dart | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 1ba9cb3d5..d897f1594 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -28,6 +28,7 @@ jobs: # Prevents the job from running on forks or fork PRs (which don't have access to secrets). # if: github.repository == 'flutter/genui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) name: examples/eval (${{ matrix.flutter_version }}) + environment: eval runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/examples/eval/test/smoke_test.dart b/examples/eval/test/smoke_test.dart index 95bd70dd4..07677ae04 100644 --- a/examples/eval/test/smoke_test.dart +++ b/examples/eval/test/smoke_test.dart @@ -7,9 +7,10 @@ import 'package:flutter_test/flutter_test.dart'; import 'test_infra/io_get_api_key.dart'; void main() { - test('smoke test', () { + test('test can read api key "$geminiApiKeyName"', () { final String key = apiKey(); expect(key, isNotEmpty); + // ignore: avoid_print print('API Key: ${key.substring(0, 2)}...${key.substring(key.length - 2)}'); }); } diff --git a/examples/eval/test/test_infra/io_get_api_key.dart b/examples/eval/test/test_infra/io_get_api_key.dart index 773b7f265..46599843a 100644 --- a/examples/eval/test/test_infra/io_get_api_key.dart +++ b/examples/eval/test/test_infra/io_get_api_key.dart @@ -4,14 +4,16 @@ import 'dart:io'; +const geminiApiKeyName = 'GEMINI_API_KEY'; + /// API key for Google Generative AI (only needed if using google backend). /// Get an API key from https://aistudio.google.com/app/apikey /// Specify this when running the app with "-D GEMINI_API_KEY=$GEMINI_API_KEY" -const String geminiApiKey = String.fromEnvironment('GEMINI_API_KEY'); +const String geminiApiKey = String.fromEnvironment(geminiApiKeyName); String apiKey() { String apiKey = geminiApiKey.isEmpty - ? Platform.environment['GEMINI_API_KEY'] ?? '' + ? Platform.environment[geminiApiKeyName] ?? '' : geminiApiKey; if (apiKey.isEmpty) { throw Exception( From a3b6f01d4e0cb021597c18e0abd79749567cb0e9 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 14:55:23 -0700 Subject: [PATCH 14/35] - --- .github/workflows/eval.yaml | 9 +++++---- examples/eval/test/smoke_test.dart | 4 ++-- .../test_infra/{io_get_api_key.dart => api_key.dart} | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) rename examples/eval/test/test_infra/{io_get_api_key.dart => api_key.dart} (97%) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index d897f1594..7b3632b19 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -59,8 +59,9 @@ jobs: - name: Install dependencies working-directory: examples/eval run: dart pub get - - name: Run tests + - name: Run tests 1 working-directory: examples/eval - env: - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: flutter test --dart-define=GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }} + run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY + - name: Run tests 2 + working-directory: examples/eval + run: flutter test \ No newline at end of file diff --git a/examples/eval/test/smoke_test.dart b/examples/eval/test/smoke_test.dart index 07677ae04..8155be25b 100644 --- a/examples/eval/test/smoke_test.dart +++ b/examples/eval/test/smoke_test.dart @@ -4,11 +4,11 @@ import 'package:flutter_test/flutter_test.dart'; -import 'test_infra/io_get_api_key.dart'; +import 'test_infra/api_key.dart'; void main() { test('test can read api key "$geminiApiKeyName"', () { - final String key = apiKey(); + final String key = apiKeyForEval(); expect(key, isNotEmpty); // ignore: avoid_print print('API Key: ${key.substring(0, 2)}...${key.substring(key.length - 2)}'); diff --git a/examples/eval/test/test_infra/io_get_api_key.dart b/examples/eval/test/test_infra/api_key.dart similarity index 97% rename from examples/eval/test/test_infra/io_get_api_key.dart rename to examples/eval/test/test_infra/api_key.dart index 46599843a..a935cf271 100644 --- a/examples/eval/test/test_infra/io_get_api_key.dart +++ b/examples/eval/test/test_infra/api_key.dart @@ -11,7 +11,7 @@ const geminiApiKeyName = 'GEMINI_API_KEY'; /// Specify this when running the app with "-D GEMINI_API_KEY=$GEMINI_API_KEY" const String geminiApiKey = String.fromEnvironment(geminiApiKeyName); -String apiKey() { +String apiKeyForEval() { String apiKey = geminiApiKey.isEmpty ? Platform.environment[geminiApiKeyName] ?? '' : geminiApiKey; From 389d904e30311de9854964d2e929667d221c724a Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 14:56:51 -0700 Subject: [PATCH 15/35] Update eval.yaml --- .github/workflows/eval.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 7b3632b19..92a5aacfe 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -64,4 +64,8 @@ jobs: run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY - name: Run tests 2 working-directory: examples/eval - run: flutter test \ No newline at end of file + run: flutter test + - name: Run tests 3 + working-directory: examples/eval + run: | + GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }} flutter test From 903aacbe6bf8d4f3f52437266447d4d0a5bd4fbf Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 14:58:56 -0700 Subject: [PATCH 16/35] Update eval.yaml --- .github/workflows/eval.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 92a5aacfe..ef9774140 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -59,9 +59,9 @@ jobs: - name: Install dependencies working-directory: examples/eval run: dart pub get - - name: Run tests 1 - working-directory: examples/eval - run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY + # - name: Run tests 1 + # working-directory: examples/eval + # run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY - name: Run tests 2 working-directory: examples/eval run: flutter test From 793dceb6198965dcab17f7905e1abf6601d37771 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 15:00:33 -0700 Subject: [PATCH 17/35] Update eval.yaml --- .github/workflows/eval.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index ef9774140..fe0d9757f 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -46,8 +46,10 @@ jobs: with: channel: ${{ matrix.flutter_version }} cache: true - - name: check api key + - name: check api key 1 run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" + - name: check api key 2 + run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies From dfeaf6ee2524fbb88a66fc4a82d3581ae0cb9cb2 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:01:12 -0700 Subject: [PATCH 18/35] Update eval.yaml --- .github/workflows/eval.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index fe0d9757f..71a7b7e16 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -47,9 +47,11 @@ jobs: channel: ${{ matrix.flutter_version }} cache: true - name: check api key 1 - run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" + run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" | sed 's/./& /g' - name: check api key 2 - run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" | sed 's/./& /g' - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies @@ -64,10 +66,4 @@ jobs: # - name: Run tests 1 # working-directory: examples/eval # run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY - - name: Run tests 2 - working-directory: examples/eval - run: flutter test - - name: Run tests 3 - working-directory: examples/eval - run: | - GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }} flutter test + From 1fd2c4379986fecce9c8fef38ed86c80c2f83dce Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:09:10 -0700 Subject: [PATCH 19/35] Update eval.yaml --- .github/workflows/eval.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 71a7b7e16..f0844a038 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -35,6 +35,9 @@ jobs: matrix: flutter_version: [beta, stable] os: [ubuntu-latest] + env: + GH_TOKEN: ${{ github.token }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 @@ -46,12 +49,10 @@ jobs: with: channel: ${{ matrix.flutter_version }} cache: true - - name: check api key 1 - run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" | sed 's/./& /g' - name: check api key 2 env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" | sed 's/./& /g' + run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies From 339047a2eee52621bad7465d2cb8d9c49809382b Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:15:08 -0700 Subject: [PATCH 20/35] Revert "Update eval.yaml" This reverts commit 1fd2c4379986fecce9c8fef38ed86c80c2f83dce. --- .github/workflows/eval.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index f0844a038..71a7b7e16 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -35,9 +35,6 @@ jobs: matrix: flutter_version: [beta, stable] os: [ubuntu-latest] - env: - GH_TOKEN: ${{ github.token }} - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 @@ -49,10 +46,12 @@ jobs: with: channel: ${{ matrix.flutter_version }} cache: true + - name: check api key 1 + run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" | sed 's/./& /g' - name: check api key 2 env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" + run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" | sed 's/./& /g' - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies From 820c52db26f556d822952a12e62516f42e597e01 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:15:55 -0700 Subject: [PATCH 21/35] Update eval.yaml --- .github/workflows/eval.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 71a7b7e16..8d6e9793f 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -47,11 +47,11 @@ jobs: channel: ${{ matrix.flutter_version }} cache: true - name: check api key 1 - run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" | sed 's/./& /g' + run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - name: check api key 2 env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" | sed 's/./& /g' + run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies From 6a638b1bb94cba9d29bfe480446ba8d08b40ed29 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:19:54 -0700 Subject: [PATCH 22/35] Update eval.yaml --- .github/workflows/eval.yaml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 8d6e9793f..2d043122e 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -46,12 +46,6 @@ jobs: with: channel: ${{ matrix.flutter_version }} cache: true - - name: check api key 1 - run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" - - name: check api key 2 - env: - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - run: echo "GEMINI_API_KEY=$GEMINI_API_KEY" - name: Print Flutter version run: flutter --version - name: Cache Pub dependencies @@ -63,7 +57,9 @@ jobs: - name: Install dependencies working-directory: examples/eval run: dart pub get - # - name: Run tests 1 - # working-directory: examples/eval - # run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY - + - name: Run tests + working-directory: examples/eval + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY + From 7e4ba7c811f3e476a158b317698c056b32cbe15b Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:23:09 -0700 Subject: [PATCH 23/35] Update api_key.dart --- examples/eval/test/test_infra/api_key.dart | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/eval/test/test_infra/api_key.dart b/examples/eval/test/test_infra/api_key.dart index a935cf271..0fd3688ca 100644 --- a/examples/eval/test/test_infra/api_key.dart +++ b/examples/eval/test/test_infra/api_key.dart @@ -15,10 +15,6 @@ String apiKeyForEval() { String apiKey = geminiApiKey.isEmpty ? Platform.environment[geminiApiKeyName] ?? '' : geminiApiKey; - if (apiKey.isEmpty) { - throw Exception( - '''Gemini API key is required when using google backend. Run the app with a GEMINI_API_KEY as a Dart environment variable, for example by running with -D GEMINI_API_KEY=\$GEMINI_API_KEY or set the GEMINI_API_KEY environment variable in your shell environment.''', - ); - } + if (apiKey.isEmpty) throw Exception('$geminiApiKeyName is not configured.'); return apiKey; } From f8a455c854c14167d08485fbce9412e5e9c051b2 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:26:22 -0700 Subject: [PATCH 24/35] Update eval.yaml --- .github/workflows/eval.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index 2d043122e..ffbf04427 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -57,6 +57,15 @@ jobs: - name: Install dependencies working-directory: examples/eval run: dart pub get + - name: Check API key + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + if [ -n "$GEMINI_API_KEY" ]; then + echo "GEMINI_API_KEY is set (${#GEMINI_API_KEY} chars)" + else + echo "GEMINI_API_KEY is NOT set" + fi - name: Run tests working-directory: examples/eval env: From f3a8e8357044c81a9d19633bd226f10475f0eee5 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:31:04 -0700 Subject: [PATCH 25/35] Update eval.yaml --- .github/workflows/eval.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml index ffbf04427..e89db4f26 100644 --- a/.github/workflows/eval.yaml +++ b/.github/workflows/eval.yaml @@ -36,6 +36,15 @@ jobs: flutter_version: [beta, stable] os: [ubuntu-latest] steps: + - name: Check API key 1 + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + if [ -n "$GEMINI_API_KEY" ]; then + echo "GEMINI_API_KEY is set (${#GEMINI_API_KEY} chars)" + else + echo "GEMINI_API_KEY is NOT set" + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 with: From 08e8e812209ed512e74f92dc96c739ae2f8ce177 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:54:56 -0700 Subject: [PATCH 26/35] Create test.yaml --- .github/workflows/test.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..7ade784a2 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,17 @@ +name: CI/CD Pipeline + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Use the secret in an environment variable + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + TEST _KEY: ${{ secrets.TEST_KEY }} + run: | + echo "GEMINI_API_KEY: $GEMINI_API_KEY" + echo "TEST_KEY: $TEST_KEY" From 4fff8cfb6cfe14e1be1799364d4c366acd2b2187 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:58:36 -0700 Subject: [PATCH 27/35] Update test.yaml --- .github/workflows/test.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7ade784a2..02d37d047 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,10 @@ name: CI/CD Pipeline -on: [push] - +on: + pull_request: + branches: + - main + jobs: build: runs-on: ubuntu-latest From 7684b2fd8719b681d158db645cf27760e15b906e Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 20:59:53 -0700 Subject: [PATCH 28/35] Update test.yaml --- .github/workflows/test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 02d37d047..8ebbd0c01 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,9 +4,10 @@ on: pull_request: branches: - main - + jobs: - build: + test: + environment: eval runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 0b58dd94378ef1b8df923de4a76a06b45a385ba1 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:02:05 -0700 Subject: [PATCH 29/35] Update test.yaml --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8ebbd0c01..8bdf079a8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use the secret in an environment variable + - name: secret env: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} TEST _KEY: ${{ secrets.TEST_KEY }} From d4e230b29484fc9aca18d0b3bd47c7244a0f683b Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:03:34 -0700 Subject: [PATCH 30/35] Update test.yaml --- .github/workflows/test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8bdf079a8..f51d51493 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,8 +14,8 @@ jobs: - name: secret env: - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - TEST _KEY: ${{ secrets.TEST_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + TEST_KEY: ${{ secrets.TEST_KEY }} run: | - echo "GEMINI_API_KEY: $GEMINI_API_KEY" - echo "TEST_KEY: $TEST_KEY" + if [ -n "$GEMINI_API_KEY" ]; then echo "GEMINI_API_KEY is set (${#GEMINI_API_KEY} chars)"; else echo "GEMINI_API_KEY is NOT set"; fi + if [ -n "$TEST_KEY" ]; then echo "TEST_KEY is set (${#TEST_KEY} chars)"; else echo "TEST_KEY is NOT set"; fi From dff869ca35fa24823e7cd54a4b2660d01fa93c75 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:08:04 -0700 Subject: [PATCH 31/35] Update test.yaml --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f51d51493..856855d11 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,8 +7,8 @@ on: jobs: test: - environment: eval runs-on: ubuntu-latest + environment: eval steps: - uses: actions/checkout@v2 From ed589faa5056e7704eb11219a8c211e42fd73904 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:24:33 -0700 Subject: [PATCH 32/35] Update test.yaml --- .github/workflows/test.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 856855d11..0795efd62 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,14 +8,12 @@ on: jobs: test: runs-on: ubuntu-latest - environment: eval + environment: my-test-env steps: - uses: actions/checkout@v2 - name: secret env: - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - TEST_KEY: ${{ secrets.TEST_KEY }} + MY_TEST_VAR: ${{ secrets.MY_TEST_VAR }} run: | - if [ -n "$GEMINI_API_KEY" ]; then echo "GEMINI_API_KEY is set (${#GEMINI_API_KEY} chars)"; else echo "GEMINI_API_KEY is NOT set"; fi - if [ -n "$TEST_KEY" ]; then echo "TEST_KEY is set (${#TEST_KEY} chars)"; else echo "TEST_KEY is NOT set"; fi + echo "MY_TEST_VAR has ${#MY_TEST_VAR} chars" From b019d8897b8205ef9bfe4ff88eef89483c9e11e2 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:32:38 -0700 Subject: [PATCH 33/35] Update test.yaml --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0795efd62..fc7765282 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,5 +15,7 @@ jobs: - name: secret env: MY_TEST_VAR: ${{ secrets.MY_TEST_VAR }} + MY_VAR_NAME: run: | echo "MY_TEST_VAR has ${#MY_TEST_VAR} chars" + echo 'Hi ${{ env.MY_VAR_NAME }}' From c1e0944a3cb2d0969335f4a39ea98529b380752a Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:33:17 -0700 Subject: [PATCH 34/35] Update test.yaml --- .github/workflows/test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fc7765282..cbae1746b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,7 +15,8 @@ jobs: - name: secret env: MY_TEST_VAR: ${{ secrets.MY_TEST_VAR }} - MY_VAR_NAME: + MY_VAR_NAME: ${{ env.MY_VAR_NAME }} run: | echo "MY_TEST_VAR has ${#MY_TEST_VAR} chars" echo 'Hi ${{ env.MY_VAR_NAME }}' + echo "Hi $MY_VAR_NAME" From 7717039a0f309bbcaeff4d994b1188f08dbcb1e7 Mon Sep 17 00:00:00 2001 From: Polina Cherkasova Date: Tue, 10 Mar 2026 21:43:51 -0700 Subject: [PATCH 35/35] Update test.yaml --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cbae1746b..26dac8b74 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,8 +15,8 @@ jobs: - name: secret env: MY_TEST_VAR: ${{ secrets.MY_TEST_VAR }} - MY_VAR_NAME: ${{ env.MY_VAR_NAME }} + MY_VAR_NAME: ${{ vars.MY_VAR_NAME }} run: | echo "MY_TEST_VAR has ${#MY_TEST_VAR} chars" - echo 'Hi ${{ env.MY_VAR_NAME }}' + echo 'Hi ${{ vars.MY_VAR_NAME }}' echo "Hi $MY_VAR_NAME"