Add CLI and eval_config packages

.github/workflows/cli_tests.yml

@@ -0,0 +1,41 @@
+name: CLI Tests
+
+on:
+  pull_request:
+    paths:
+      - 'packages/eval_cli/**'
+      - 'packages/eval_config/**'
+      - '.github/workflows/cli_tests.yml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/eval_cli/**'
+      - 'packages/eval_config/**'
+      - '.github/workflows/cli_tests.yml'
+
+jobs:
+  cli-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Flutter
+        run: |
+          git clone https://github.com/flutter/flutter.git --depth 1 -b stable $HOME/flutter
+          echo "$HOME/flutter/bin" >> $GITHUB_PATH
+          echo "$HOME/.pub-cache/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: flutter pub get
+
+      - name: Analyze
+        working-directory: packages/eval_cli
+        run: dart analyze --fatal-infos
+
+      - name: Run tests
+        working-directory: packages/eval_cli
+        run: flutter test

.github/workflows/config_tests.yml

@@ -0,0 +1,39 @@
+name: Config Tests
+
+on:
+  pull_request:
+    paths:
+      - 'packages/eval_config/**'
+      - '.github/workflows/config_tests.yml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/eval_config/**'
+      - '.github/workflows/config_tests.yml'
+
+jobs:
+  config-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Install Flutter
+        run: |
+          git clone https://github.com/flutter/flutter.git --depth 1 -b stable $HOME/flutter
+          echo "$HOME/flutter/bin" >> $GITHUB_PATH
+          echo "$HOME/.pub-cache/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: flutter pub get
+
+      - name: Analyze
+        working-directory: packages/eval_config
+        run: dart analyze --fatal-infos
+
+      - name: Run tests
+        working-directory: packages/eval_config
+        run: dart test

.gitignore

@@ -0,0 +1,234 @@
+# Added by human
+data/
+logs/
+_uploaded_logs/
+*.log
+htmlcov/
+*/htmlcov/
+**/htmlcov/
+*/.coverage/
+**/.coverage/
+.coverage
+coverage
+.dart_tool/
+.devals-tool/
+/docs/_build
+/docs/dart_docs
+logs/
+
+
+##
+# Generated by Flutter, Python, Firebase... who knows what else.
+##
+
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+# C extensions
+*.so
+
+# Firebase service account keys (NEVER COMMIT THESE!)
+*firebase*key*.json
+firebase-adminsdk-*.json
+serviceAccountKey.json
+
+# Distribution / packaging
+.Python
+build/
+dist/
+downloads/
+eggs/
+.eggs/
+pkgs/runner/lib/
+pkgs/runner/lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.# Sandbox - track structure, ignore generated files
+sandbox/**/.dart_tool/
+sandbox/**/build/
+sandbox/**/.flutter-plugins
+sandbox/**/.flutter-plugins-dependencies
+sandbox/**/pubspec.lock
+sandbox/**/.packages
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   not pinning sub-dependencies is a problem, you can uncomment the next line.
+# Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   For more info, see: https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file
+# poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+# pdm.lock
+
+# PEP 582; used by e.g. pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+# Virtual environment should be in pkgs/runner/ subdirectory
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pkgs/runner/.venv
+pkgs/runner/env/
+pkgs/runner/venv/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+.DS_Store
+/**/.DS_Store
+
+# intellij
+.idea
+
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.build/
+.buildlog/
+.history
+.svn/
+.swiftpm/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+**/ios/Flutter/.last_build_id
+.dart_tool/
+.flutter-plugins-dependencies
+**/.flutter-plugins-dependencies
+.pub-cache/
+**/.pub-cache/
+.pub/
+**/.pub/
+/build/
+/coverage/
+
+# Symbolication related
+app.*.symbols
+
+# Obfuscation related
+app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release

packages/eval_cli/.dart_tool/pub/workspace_ref.json

@@ -0,0 +1,3 @@
+{
+  "workspaceRoot": "../../../.."
+}

packages/eval_cli/README.md

@@ -0,0 +1,5 @@
+# Deval CLI
+
+An opinionated CLI that manages dash-evals evaluation tasks and jobs. Requires the [Dart SDK](https://dart.dev/get-dart).
+
+📖 **[Full documentation](../../docs/cli.md)** — setup, commands, and usage.

packages/eval_cli/analysis_options.yaml

@@ -0,0 +1,9 @@
+include: package:lints/recommended.yaml
+
+linter:
+  rules:
+    - prefer_single_quotes
+    - directives_ordering
+
+formatter:
+  trailing_commas: preserve

packages/eval_cli/bin/devals.dart

@@ -0,0 +1,29 @@
+/// Entry point for the deval CLI.
+library;
+
+import 'dart:io';
+
+import 'package:args/command_runner.dart';
+import 'package:devals/devals.dart';
+import 'package:howdy/howdy.dart';
+
+Future<void> main(List<String> args) async {
+  final runner = CommandRunner<int>('devals', 'Manage dash-evals projects')
+    ..addCommand(InitCommand())
+    ..addCommand(DoctorCommand())
+    ..addCommand(CreateCommand())
+    ..addCommand(PublishCommand())
+    ..addCommand(RunCommand())
+    ..addCommand(ViewCommand());
+
+  try {
+    final exitCode = await runner.run(args) ?? 0;
+    exit(exitCode);
+  } on CliException catch (e) {
+    Text.error('${e.message}\n');
+    exit(e.exitCode);
+  } on Exception catch (e) {
+    Text.error('Error: $e\n');
+    exit(1);
+  }
+}

packages/eval_cli/dart_test.yaml

@@ -0,0 +1,9 @@
+# Test configuration for the eval_cli package.
+# Run tests serially to avoid race conditions with Directory.current
+# mutations in filesystem_utils_test.dart and dataset_reader_test.dart.
+concurrency: 1
+
+tags:
+  e2e:
+    # E2E tests spawn subprocesses and may be slower
+    timeout: 120s