Eval

.github/workflows/eval.yaml

@@ -0,0 +1,65 @@
+# Copyright 2025 The Flutter Authors.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
+name: Eval CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    # Tests may fail due to new dependency releases.
+    # Regular execution provides early detection of such regressions.
+    - cron: '0 * * * *' # hourly
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    # Prevents the job from running on forks or fork PRs (which don't have access to secrets).
+    # if: github.repository == 'flutter/genui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+    name: examples/eval (${{ matrix.flutter_version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        flutter_version: [beta, stable]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654
+        with:
+          distribution: "zulu"
+          java-version: "17"
+          cache: "gradle"
+      - uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046
+        with:
+          channel: ${{ matrix.flutter_version }}
+          cache: true
+      - name: check api key
+        run: echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}"
+      - name: Print Flutter version
+        run: flutter --version
+      - name: Cache Pub dependencies
+        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306
+        with:
+          path: ${{ env.PUB_CACHE }}
+          key: ${{ runner.os }}-pub-${{ hashFiles('**/pubspec.lock') }}
+          restore-keys: ${{ runner.os }}-pub-
+      - name: Install dependencies
+        working-directory: examples/eval
+        run: dart pub get
+      - name: Run tests
+        working-directory: examples/eval
+        env:
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+        run: flutter test --dart-define=GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}

.github/workflows/flutter_packages.yaml

@@ -50,7 +50,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: |
           # Find all directories containing pubspec.yaml in packages, tools, and examples.
-          ALL_DIRS=$(find packages examples tool -name pubspec.yaml -not -path "*/.dart_tool/*" -exec dirname {} \;)
+          ALL_DIRS=$(find packages examples tool -name pubspec.yaml -not -path "*/.dart_tool/*" -not -path "examples/eval/*" -exec dirname {} \;)
 
           # Get the list of changed files. The method depends on the event type.
           if [[ "${{ github.event_name }}" == "pull_request" ]]; then

examples/eval/README.md

@@ -0,0 +1,3 @@
+# Eval
+
+Project for tests that evaluate genui against real AI models.

examples/eval/pubspec.yaml

@@ -0,0 +1,24 @@
+# Copyright 2025 The Flutter Authors.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+name: eval
+publish_to: "none"
+
+environment:
+  sdk: ">=3.10.0 <4.0.0"
+  flutter: ">=3.35.7 <4.0.0"
+
+resolution: workspace
+
+dependencies:
+  flutter:
+    sdk: flutter
+  genui: ^0.7.0
+
+dev_dependencies:
+  flutter_test:
+    sdk: flutter
+
+flutter:
+  uses-material-design: true

examples/eval/test/smoke_test.dart

@@ -0,0 +1,15 @@
+// Copyright 2025 The Flutter Authors.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+
+import 'test_infra/io_get_api_key.dart';
+
+void main() {
+  test('smoke test', () {
+    final String key = apiKey();
+    expect(key, isNotEmpty);
+    print('API Key: ${key.substring(0, 2)}...${key.substring(key.length - 2)}');
+  });
+}

examples/eval/test/test_infra/io_get_api_key.dart

@@ -0,0 +1,22 @@
+// Copyright 2025 The Flutter Authors.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+/// API key for Google Generative AI (only needed if using google backend).
+/// Get an API key from https://aistudio.google.com/app/apikey
+/// Specify this when running the app with "-D GEMINI_API_KEY=$GEMINI_API_KEY"
+const String geminiApiKey = String.fromEnvironment('GEMINI_API_KEY');
+
+String apiKey() {
+  String apiKey = geminiApiKey.isEmpty
+      ? Platform.environment['GEMINI_API_KEY'] ?? ''
+      : geminiApiKey;
+  if (apiKey.isEmpty) {
+    throw Exception(
+      '''Gemini API key is required when using google backend. Run the app with a GEMINI_API_KEY as a Dart environment variable, for example by running with -D GEMINI_API_KEY=\$GEMINI_API_KEY or set the GEMINI_API_KEY environment variable in your shell environment.''',
+    );
+  }
+  return apiKey;
+}

pubspec.yaml

@@ -12,14 +12,15 @@ environment:
 
 workspace:
   - examples/catalog_gallery
+  - examples/eval
   - examples/simple_chat
-
   - examples/travel_app
   - examples/verdure/client
+
   - packages/genui
   - packages/genui_a2a
-
   - packages/json_schema_builder
+
   - tool/fix_copyright
   - tool/release
   - tool/test_and_fix