diff --git a/.github/workflows/eval.yaml b/.github/workflows/eval.yaml new file mode 100644 index 000000000..e89db4f26 --- /dev/null +++ b/.github/workflows/eval.yaml @@ -0,0 +1,83 @@ +# Copyright 2025 The Flutter Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +name: Eval CI + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + # Tests may fail due to new dependency releases. + # Regular execution provides early detection of such regressions. + - cron: '0 * * * *' # hourly + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + # Prevents the job from running on forks or fork PRs (which don't have access to secrets). + # if: github.repository == 'flutter/genui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + name: examples/eval (${{ matrix.flutter_version }}) + environment: eval + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + flutter_version: [beta, stable] + os: [ubuntu-latest] + steps: + - name: Check API key 1 + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + if [ -n "$GEMINI_API_KEY" ]; then + echo "GEMINI_API_KEY is set (${#GEMINI_API_KEY} chars)" + else + echo "GEMINI_API_KEY is NOT set" + fi + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 + with: + distribution: "zulu" + java-version: "17" + cache: "gradle" + - uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046 + with: + channel: ${{ matrix.flutter_version }} + cache: true + - name: Print Flutter version + run: flutter --version + - name: Cache Pub dependencies + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 + with: + path: ${{ env.PUB_CACHE }} + key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.lock') }} + restore-keys: ${{ runner.os }}-pub- + - name: Install dependencies + working-directory: examples/eval + run: dart pub get + - name: Check API key + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + if [ -n "$GEMINI_API_KEY" ]; then + echo "GEMINI_API_KEY is set (${#GEMINI_API_KEY} chars)" + else + echo "GEMINI_API_KEY is NOT set" + fi + - name: Run tests + working-directory: examples/eval + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: flutter test --dart-define=GEMINI_API_KEY=$GEMINI_API_KEY + diff --git a/.github/workflows/flutter_packages.yaml b/.github/workflows/flutter_packages.yaml index 6e75b69af..54b16ed72 100644 --- a/.github/workflows/flutter_packages.yaml +++ b/.github/workflows/flutter_packages.yaml @@ -50,7 +50,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | # Find all directories containing pubspec.yaml in packages, tools, and examples. - ALL_DIRS=$(find packages examples tool -name pubspec.yaml -not -path "*/.dart_tool/*" -exec dirname {} \;) + ALL_DIRS=$(find packages examples tool -name pubspec.yaml -not -path "*/.dart_tool/*" -not -path "examples/eval/*" -exec dirname {} \;) # Get the list of changed files. The method depends on the event type. if [[ "${{ github.event_name }}" == "pull_request" ]]; then diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..26dac8b74 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,22 @@ +name: CI/CD Pipeline + +on: + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + environment: my-test-env + steps: + - uses: actions/checkout@v2 + + - name: secret + env: + MY_TEST_VAR: ${{ secrets.MY_TEST_VAR }} + MY_VAR_NAME: ${{ vars.MY_VAR_NAME }} + run: | + echo "MY_TEST_VAR has ${#MY_TEST_VAR} chars" + echo 'Hi ${{ vars.MY_VAR_NAME }}' + echo "Hi $MY_VAR_NAME" diff --git a/examples/eval/README.md b/examples/eval/README.md new file mode 100644 index 000000000..50ff1fa99 --- /dev/null +++ b/examples/eval/README.md @@ -0,0 +1,3 @@ +# Eval + +Project for tests that evaluate genui against real AI models. diff --git a/examples/eval/pubspec.yaml b/examples/eval/pubspec.yaml new file mode 100644 index 000000000..ba4a423bb --- /dev/null +++ b/examples/eval/pubspec.yaml @@ -0,0 +1,24 @@ +# Copyright 2025 The Flutter Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +name: eval +publish_to: "none" + +environment: + sdk: ">=3.10.0 <4.0.0" + flutter: ">=3.35.7 <4.0.0" + +resolution: workspace + +dependencies: + flutter: + sdk: flutter + genui: ^0.7.0 + +dev_dependencies: + flutter_test: + sdk: flutter + +flutter: + uses-material-design: true diff --git a/examples/eval/test/smoke_test.dart b/examples/eval/test/smoke_test.dart new file mode 100644 index 000000000..8155be25b --- /dev/null +++ b/examples/eval/test/smoke_test.dart @@ -0,0 +1,16 @@ +// Copyright 2025 The Flutter Authors. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; + +import 'test_infra/api_key.dart'; + +void main() { + test('test can read api key "$geminiApiKeyName"', () { + final String key = apiKeyForEval(); + expect(key, isNotEmpty); + // ignore: avoid_print + print('API Key: ${key.substring(0, 2)}...${key.substring(key.length - 2)}'); + }); +} diff --git a/examples/eval/test/test_infra/api_key.dart b/examples/eval/test/test_infra/api_key.dart new file mode 100644 index 000000000..0fd3688ca --- /dev/null +++ b/examples/eval/test/test_infra/api_key.dart @@ -0,0 +1,20 @@ +// Copyright 2025 The Flutter Authors. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +const geminiApiKeyName = 'GEMINI_API_KEY'; + +/// API key for Google Generative AI (only needed if using google backend). +/// Get an API key from https://aistudio.google.com/app/apikey +/// Specify this when running the app with "-D GEMINI_API_KEY=$GEMINI_API_KEY" +const String geminiApiKey = String.fromEnvironment(geminiApiKeyName); + +String apiKeyForEval() { + String apiKey = geminiApiKey.isEmpty + ? Platform.environment[geminiApiKeyName] ?? '' + : geminiApiKey; + if (apiKey.isEmpty) throw Exception('$geminiApiKeyName is not configured.'); + return apiKey; +} diff --git a/pubspec.yaml b/pubspec.yaml index 4d780ea9f..9fc0c1d24 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,14 +12,15 @@ environment: workspace: - examples/catalog_gallery + - examples/eval - examples/simple_chat - - examples/travel_app - examples/verdure/client + - packages/genui - packages/genui_a2a - - packages/json_schema_builder + - tool/fix_copyright - tool/release - tool/test_and_fix