diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index cc7bba843..f77584dea 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -33,6 +33,7 @@ jobs: - test/commands/test/compilation_error/compilation_error_test.dart - test/commands/test/no_project/no_project_test.dart - test/commands/test/spaced_golden_file_name/spaced_golden_file_name_test.dart + - test/commands/test/very_good_config/very_good_config_test.dart # E2E tests for the create command - test/commands/create/flutter_app/core_test.dart diff --git a/.github/workflows/very_good_cli.yaml b/.github/workflows/very_good_cli.yaml index 493837649..1e93a39df 100644 --- a/.github/workflows/very_good_cli.yaml +++ b/.github/workflows/very_good_cli.yaml @@ -29,12 +29,14 @@ jobs: os: [ubuntu-latest, windows-latest] uses: VeryGoodOpenSource/very_good_workflows/.github/workflows/flutter_package.yml@v1 with: + runs_on: ${{ matrix.os }} flutter_version: "3.44.x" + coverage_excludes: "**/*.{gen,g}.dart" concurrency: 1 - coverage_excludes: "**/*.gen.dart" - run_bloc_lint: false run_skipped: true - runs_on: ${{ matrix.os }} + run_bloc_lint: false + # The CLI's own test suite is run directly rather than through the test optimizer, + # matching the CI behavior prior to adopting this shared workflow. test_optimization: false pana: diff --git a/analysis_options.yaml b/analysis_options.yaml index 9834b46d4..363528c1b 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -1,5 +1,6 @@ include: package:very_good_analysis/analysis_options.yaml analyzer: exclude: + - "**/*.g.dart" - "**/version.dart" - "bricks/**/__brick__" diff --git a/build.yaml b/build.yaml new file mode 100644 index 000000000..9186cc335 --- /dev/null +++ b/build.yaml @@ -0,0 +1,8 @@ +targets: + $default: + sources: + - $package$ + - lib/** + - bin/** + - test/** + - pubspec.yaml diff --git a/e2e/test/commands/test/very_good_config/fixture/lib/uncovered.dart b/e2e/test/commands/test/very_good_config/fixture/lib/uncovered.dart new file mode 100644 index 000000000..36c51a5b7 --- /dev/null +++ b/e2e/test/commands/test/very_good_config/fixture/lib/uncovered.dart @@ -0,0 +1 @@ +int untestedFunction(int value) => value * 2; diff --git a/e2e/test/commands/test/very_good_config/fixture/pubspec.yaml b/e2e/test/commands/test/very_good_config/fixture/pubspec.yaml new file mode 100644 index 000000000..d98f01c24 --- /dev/null +++ b/e2e/test/commands/test/very_good_config/fixture/pubspec.yaml @@ -0,0 +1,10 @@ +name: very_good_config_fixture +description: Fixture for testing very_good.yaml configuration. +version: 0.1.0+1 +publish_to: none + +environment: + sdk: ^3.12.0 + +dev_dependencies: + test: ^1.24.3 diff --git a/e2e/test/commands/test/very_good_config/fixture/test/covered_test.dart b/e2e/test/commands/test/very_good_config/fixture/test/covered_test.dart new file mode 100644 index 000000000..638cba389 --- /dev/null +++ b/e2e/test/commands/test/very_good_config/fixture/test/covered_test.dart @@ -0,0 +1,12 @@ +import 'package:test/test.dart'; + +void main() { + group('very_good_config_fixture', () { + // This test exists only to produce a passing coverage run so the + // `min_coverage: 100` config can be evaluated against the deliberately + // uncovered `lib/uncovered.dart`. + test('trivially succeeds', () { + expect(true, isTrue); + }); + }); +} diff --git a/e2e/test/commands/test/very_good_config/fixture/very_good.yaml b/e2e/test/commands/test/very_good_config/fixture/very_good.yaml new file mode 100644 index 000000000..796c69d5c --- /dev/null +++ b/e2e/test/commands/test/very_good_config/fixture/very_good.yaml @@ -0,0 +1,2 @@ +test: + min_coverage: 100 diff --git a/e2e/test/commands/test/very_good_config/malformed_fixture/pubspec.yaml b/e2e/test/commands/test/very_good_config/malformed_fixture/pubspec.yaml new file mode 100644 index 000000000..cc5ff3815 --- /dev/null +++ b/e2e/test/commands/test/very_good_config/malformed_fixture/pubspec.yaml @@ -0,0 +1,10 @@ +name: malformed_fixture +description: Fixture for testing malformed very_good.yaml. +version: 0.1.0+1 +publish_to: none + +environment: + sdk: ^3.12.0 + +dev_dependencies: + test: ^1.24.3 diff --git a/e2e/test/commands/test/very_good_config/malformed_fixture/very_good.yaml b/e2e/test/commands/test/very_good_config/malformed_fixture/very_good.yaml new file mode 100644 index 000000000..149ff8996 --- /dev/null +++ b/e2e/test/commands/test/very_good_config/malformed_fixture/very_good.yaml @@ -0,0 +1,3 @@ +- not +- a +- map diff --git a/e2e/test/commands/test/very_good_config/very_good_config_test.dart b/e2e/test/commands/test/very_good_config/very_good_config_test.dart new file mode 100644 index 000000000..f4f0ba88a --- /dev/null +++ b/e2e/test/commands/test/very_good_config/very_good_config_test.dart @@ -0,0 +1,121 @@ +import 'package:mason/mason.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:path/path.dart' as path; +import 'package:test/test.dart'; +import 'package:universal_io/io.dart'; + +import '../../../../helpers/helpers.dart'; + +void main() { + group('very_good.yaml', () { + test( + 'enforces min-coverage from very_good.yaml when the flag is not passed', + timeout: const Timeout(Duration(minutes: 2)), + withRunner((commandRunner, logger, updater, logs, progressLogs) async { + final tempDirectory = Directory.systemTemp.createTempSync( + 'very_good_config', + ); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + final fixture = Directory( + path.join( + Directory.current.path, + 'test/commands/test/very_good_config/fixture', + ), + ); + + await copyDirectory(fixture, tempDirectory); + + await expectSuccessfulProcessResult('flutter', [ + 'pub', + 'get', + ], workingDirectory: tempDirectory.path); + + final cwd = Directory.current; + Directory.current = tempDirectory; + addTearDown(() => Directory.current = cwd); + + await expectLater( + commandRunner.run(['test', '--coverage']), + completion(equals(ExitCode.unavailable.code)), + ); + verify( + () => logger.err(any(that: contains('Expected coverage >= 100.00%'))), + ).called(1); + }), + ); + + test( + 'CLI --min-coverage takes precedence over very_good.yaml', + timeout: const Timeout(Duration(minutes: 2)), + withRunner((commandRunner, logger, updater, logs, progressLogs) async { + final tempDirectory = Directory.systemTemp.createTempSync( + 'very_good_config_override', + ); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + final fixture = Directory( + path.join( + Directory.current.path, + 'test/commands/test/very_good_config/fixture', + ), + ); + + await copyDirectory(fixture, tempDirectory); + + await expectSuccessfulProcessResult('flutter', [ + 'pub', + 'get', + ], workingDirectory: tempDirectory.path); + + final cwd = Directory.current; + Directory.current = tempDirectory; + addTearDown(() => Directory.current = cwd); + + await expectLater( + commandRunner.run([ + 'test', + '--coverage', + '--min-coverage', + '0', + ]), + completion(equals(ExitCode.success.code)), + ); + }), + ); + + test( + 'fails with config exit code when very_good.yaml is malformed', + timeout: const Timeout(Duration(minutes: 2)), + withRunner((commandRunner, logger, updater, logs, progressLogs) async { + final tempDirectory = Directory.systemTemp.createTempSync( + 'very_good_config_malformed', + ); + addTearDown(() => tempDirectory.deleteSync(recursive: true)); + + final fixture = Directory( + path.join( + Directory.current.path, + 'test/commands/test/very_good_config/malformed_fixture', + ), + ); + + await copyDirectory(fixture, tempDirectory); + + final cwd = Directory.current; + Directory.current = tempDirectory; + addTearDown(() => Directory.current = cwd); + + await expectLater( + commandRunner.run(['test']), + completion(equals(ExitCode.config.code)), + ); + verify( + () => logger.err( + any(that: contains('Could not read `very_good.yaml`')), + ), + ).called(1); + }), + ); + }); +} diff --git a/lib/src/commands/test/test.dart b/lib/src/commands/test/test.dart index cfef17a63..e47c64e69 100644 --- a/lib/src/commands/test/test.dart +++ b/lib/src/commands/test/test.dart @@ -7,6 +7,7 @@ import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'package:universal_io/io.dart'; import 'package:very_good_cli/src/cli/cli.dart'; +import 'package:very_good_cli/src/very_good_config/very_good_config.dart'; /// Options for configuring the Flutter test command. class FlutterTestOptions { @@ -35,44 +36,119 @@ class FlutterTestOptions { }); /// Parses [ArgResults] into a [FlutterTestOptions] instance. - factory FlutterTestOptions.parse(ArgResults argResults) { - final concurrency = argResults['concurrency'] as String; - final collectCoverage = argResults['coverage'] as bool; - final minCoverage = double.tryParse( - argResults['min-coverage'] as String? ?? '', + /// + /// When [config] is provided, its values are used as defaults for any + /// option that was not explicitly parsed on the command line. + factory FlutterTestOptions.parse( + ArgResults argResults, { + VeryGoodConfig config = VeryGoodConfig.empty, + }) { + final testConfig = config.test; + + final concurrency = _resolveArg( + argResults, + 'concurrency', + testConfig.concurrency, ); - final showUncovered = argResults['show-uncovered'] as bool; - final excludeTags = argResults['exclude-tags'] as String?; - final tags = argResults['tags'] as String?; - final excludeFromCoverage = argResults['exclude-coverage'] as String?; - final collectCoverageFromString = - argResults['collect-coverage-from'] as String? ?? 'imports'; - final collectCoverageFrom = CoverageCollectionMode.fromString( - collectCoverageFromString, + final collectCoverage = _resolveArg( + argResults, + 'coverage', + testConfig.coverage, + ); + final minCoverage = _resolveArg( + argResults, + 'min-coverage', + testConfig.minCoverage, + ); + final effectiveMinCoverage = double.tryParse(minCoverage ?? ''); + final showUncovered = _resolveArg( + argResults, + 'show-uncovered', + testConfig.showUncovered, + ); + final excludeTags = _resolveArg( + argResults, + 'exclude-tags', + testConfig.excludeTags, + ); + final tags = _resolveArg(argResults, 'tags', testConfig.tags); + final excludeFromCoverage = _resolveArg( + argResults, + 'exclude-coverage', + testConfig.excludeCoverage, + ); + final collectCoverageFrom = _resolveArg( + argResults, + 'collect-coverage-from', + testConfig.collectCoverageFrom, + fallbackValue: 'imports', + ); + final effectiveCollectCoverageFrom = CoverageCollectionMode.fromString( + collectCoverageFrom, ); final randomOrderingSeed = argResults['test-randomize-ordering-seed'] as String?; final randomSeed = randomOrderingSeed == 'random' ? Random().nextInt(4294967295).toString() : randomOrderingSeed; - final optimizePerformance = argResults['optimization'] as bool; - final updateGoldens = argResults['update-goldens'] as bool; - final failFast = argResults['fail-fast'] as bool; + final optimizePerformance = _resolveArg( + argResults, + 'optimization', + testConfig.optimization, + ); + final updateGoldens = _resolveArg( + argResults, + 'update-goldens', + testConfig.updateGoldens, + ); + final failFast = _resolveArg( + argResults, + 'fail-fast', + testConfig.failFast, + ); final forceAnsi = argResults['force-ansi'] as bool?; - final dartDefine = argResults['dart-define'] as List?; - final dartDefineFromFile = - argResults['dart-define-from-file'] as List?; - final platform = argResults['platform'] as String?; - final reportOn = (argResults['report-on'] as List) + final dartDefine = _resolveArg?>( + argResults, + 'dart-define', + testConfig.dartDefine, + ); + final dartDefineFromFile = _resolveArg?>( + argResults, + 'dart-define-from-file', + testConfig.dartDefineFromFile, + ); + final platform = _resolveArg( + argResults, + 'platform', + testConfig.platform, + ); + final reportOn = _resolveArg>( + argResults, + 'report-on', + testConfig.reportOn, + ); + final effectiveReportOn = reportOn .expand((e) => e.split(RegExp(r'[,\s]+'))) .where((e) => e.isNotEmpty) .toList(); - final runSkipped = argResults['run-skipped'] as bool; - final flavor = argResults['flavor'] as String?; - final timeoutSeconds = int.tryParse( - argResults['timeout'] as String? ?? '', + + final runSkipped = _resolveArg( + argResults, + 'run-skipped', + testConfig.runSkipped, + ); + final flavor = _resolveArg( + argResults, + 'flavor', + testConfig.flavor, + ); + final timeout = _resolveArg( + argResults, + 'timeout', + testConfig.timeout, ); - final timeout = timeoutSeconds != null + final timeoutSeconds = int.tryParse(timeout ?? ''); + final effectiveTimeout = timeoutSeconds != null ? Duration(seconds: timeoutSeconds) : null; final rest = argResults.rest; @@ -80,12 +156,12 @@ class FlutterTestOptions { return FlutterTestOptions._( concurrency: concurrency, collectCoverage: collectCoverage, - minCoverage: minCoverage, + minCoverage: effectiveMinCoverage, showUncovered: showUncovered, excludeTags: excludeTags, tags: tags, excludeFromCoverage: excludeFromCoverage, - collectCoverageFrom: collectCoverageFrom, + collectCoverageFrom: effectiveCollectCoverageFrom, randomSeed: randomSeed, optimizePerformance: optimizePerformance, updateGoldens: updateGoldens, @@ -94,10 +170,10 @@ class FlutterTestOptions { dartDefine: dartDefine, dartDefineFromFile: dartDefineFromFile, platform: platform, - reportOn: reportOn, + reportOn: effectiveReportOn, runSkipped: runSkipped, flavor: flavor, - timeout: timeout, + timeout: effectiveTimeout, rest: rest, ); } @@ -168,6 +244,27 @@ class FlutterTestOptions { final List rest; } +/// Resolves the value for the argument named [name] against a `very_good.yaml` +/// configuration value. +/// +/// Resolution follows a fixed precedence, from highest to lowest: +/// +/// 1. A command line argument that was explicitly parsed. +/// 2. [configValue], the corresponding value from the configuration file. +/// 3. [fallbackValue], used when neither the command line nor the configuration +/// provide a value (typically the argument's command line default). +T _resolveArg( + ArgResults argResults, + String name, + T? configValue, { + T? fallbackValue, +}) { + final value = configValue != null && !argResults.wasParsed(name) + ? configValue + : argResults[name] as T?; + return (value ?? fallbackValue) as T; +} + /// Signature for the [Flutter.installed] method. typedef FlutterInstalledCommand = Future Function({required Logger logger}); @@ -382,9 +479,20 @@ This command should be run from the root of your Flutter project.'''); return ExitCode.noInput.code; } + final VeryGoodConfig config; + try { + config = VeryGoodConfig.loadFromClosestAncestor(Directory(targetPath)); + } on VeryGoodConfigParseException catch (e) { + _logger.err( + 'Could not read `$veryGoodConfigFileName`.\n' + '${e.message}', + ); + return ExitCode.config.code; + } + final isFlutterInstalled = await _flutterInstalled(logger: _logger); - final options = FlutterTestOptions.parse(_argResults); + final options = FlutterTestOptions.parse(_argResults, config: config); if (isFlutterInstalled) { try { diff --git a/lib/src/very_good_config/very_good_config.dart b/lib/src/very_good_config/very_good_config.dart new file mode 100644 index 000000000..e7fdbd97c --- /dev/null +++ b/lib/src/very_good_config/very_good_config.dart @@ -0,0 +1,340 @@ +/// Support for loading Very Good CLI configuration from a +/// `very_good.yaml` file. +/// +/// The configuration file lives at the root of a project and allows developers +/// to persist frequently used CLI parameters so that running the CLI locally +/// produces the same results as running it on CI. +library; + +import 'dart:io'; + +import 'package:checked_yaml/checked_yaml.dart'; +import 'package:equatable/equatable.dart'; +import 'package:json_annotation/json_annotation.dart'; +import 'package:path/path.dart' as path; + +part 'very_good_config.g.dart'; + +/// The default name of the Very Good CLI configuration file. +const veryGoodConfigFileName = 'very_good.yaml'; + +/// {@template very_good_config_parse_exception} +/// Thrown when a [VeryGoodConfig] fails to parse. +/// {@endtemplate} +class VeryGoodConfigParseException implements Exception { + /// {@macro very_good_config_parse_exception} + const VeryGoodConfigParseException(this.message); + + /// A human readable description of the parse failure. + final String message; + + @override + String toString() => 'VeryGoodConfigParseException: $message'; +} + +/// {@template very_good_config} +/// A representation of a `very_good.yaml` configuration file. +/// +/// The configuration file may declare per-command sections whose values +/// are used as defaults whenever the corresponding CLI flag is not +/// explicitly passed at the command line. +/// {@endtemplate} +@JsonSerializable( + anyMap: true, + checked: true, + createToJson: false, + disallowUnrecognizedKeys: true, + fieldRename: FieldRename.snake, +) +class VeryGoodConfig extends Equatable { + /// {@macro very_good_config} + const VeryGoodConfig({this.test = const VeryGoodTestConfig()}); + + /// Creates a [VeryGoodConfig] from a decoded YAML/JSON [json] map. + factory VeryGoodConfig.fromJson(Map json) { + return _$VeryGoodConfigFromJson(json); + } + + /// Parses a [VeryGoodConfig] from a YAML [content] string. + /// + /// An empty or `null` YAML document yields [VeryGoodConfig.empty]. + /// + /// When provided, [sourceUrl] is used to enrich error messages with the + /// location the [content] originated from. + /// + /// Throws a [VeryGoodConfigParseException] if [content] is not a valid + /// YAML map or if any known section is malformed. + factory VeryGoodConfig.fromString(String content, {Uri? sourceUrl}) { + try { + return checkedYamlDecode( + content, + (json) => VeryGoodConfig.fromJson(json ?? const {}), + allowNull: true, + sourceUrl: sourceUrl, + ); + } on ParsedYamlException catch (e) { + throw VeryGoodConfigParseException(e.formattedMessage ?? e.message); + } + } + + /// Loads the closest [VeryGoodConfig] by searching [directory] and each of + /// its ancestors, from the innermost directory outward. + /// + /// [directory] is resolved to an absolute path before the walk, so a relative + /// [directory] is searched relative to the current working directory. + /// + /// This allows a single repository-wide `very_good.yaml` at the project root + /// to apply to commands run from any nested package directory. The first + /// configuration file encountered wins; ancestors are not merged. + /// + /// Returns [VeryGoodConfig.empty] when no configuration file is found. + /// Throws a [VeryGoodConfigParseException] when the closest file exists but + /// cannot be parsed. + factory VeryGoodConfig.loadFromClosestAncestor(Directory directory) { + var current = directory.absolute; + while (true) { + final config = _loadFromDirectory(current); + if (config != null) return config; + final parent = current.parent; + if (parent.path == current.path) return VeryGoodConfig.empty; + current = parent; + } + } + + /// Loads a [VeryGoodConfig] from the configuration file directly inside + /// [directory], or `null` when the file does not exist. + /// + /// Throws a [VeryGoodConfigParseException] when the file exists but cannot be + /// parsed. + static VeryGoodConfig? _loadFromDirectory(Directory directory) { + final file = File(path.join(directory.path, veryGoodConfigFileName)); + if (!file.existsSync()) return null; + return VeryGoodConfig.fromString( + file.readAsStringSync(), + sourceUrl: file.uri, + ); + } + + /// An empty [VeryGoodConfig] with no values set. + static const VeryGoodConfig empty = VeryGoodConfig(); + + /// Configuration values for the `very_good test` command. + final VeryGoodTestConfig test; + + @override + List get props => [test]; +} + +/// {@template very_good_test_config} +/// Configuration values that customize the defaults of the +/// `very_good test` command. +/// +/// Any field that is left as `null` retains its CLI default. +/// {@endtemplate} +@JsonSerializable( + anyMap: true, + checked: true, + createToJson: false, + disallowUnrecognizedKeys: true, + fieldRename: FieldRename.snake, +) +class VeryGoodTestConfig extends Equatable { + /// {@macro very_good_test_config} + const VeryGoodTestConfig({ + this.coverage, + this.optimization, + this.concurrency, + this.tags, + this.excludeCoverage, + this.excludeTags, + this.minCoverage, + this.showUncovered, + this.collectCoverageFrom, + this.updateGoldens, + this.failFast, + this.dartDefine, + this.dartDefineFromFile, + this.platform, + this.reportOn, + this.runSkipped, + this.flavor, + this.timeout, + }); + + /// Creates a [VeryGoodTestConfig] from a decoded YAML/JSON [json] map. + factory VeryGoodTestConfig.fromJson(Map json) { + return _$VeryGoodTestConfigFromJson(json); + } + + /// Whether to collect coverage information. + final bool? coverage; + + /// Whether to apply optimizations for test performance. + final bool? optimization; + + /// The number of concurrent test suites run. + @JsonKey(fromJson: _concurrency) + final String? concurrency; + + /// Run only tests associated with the specified tags. + final String? tags; + + /// A glob which will be used to exclude files that match from the coverage. + final String? excludeCoverage; + + /// Run only tests that do not have the specified tags. + final String? excludeTags; + + /// The minimum coverage percentage enforced. + @JsonKey(fromJson: _minCoverage) + final String? minCoverage; + + /// Whether to show uncovered lines when coverage is below 100%. + final bool? showUncovered; + + /// Whether to collect coverage from imported files only or all files. + @JsonKey(fromJson: _collectCoverageFrom) + final String? collectCoverageFrom; + + /// Whether `matchesGoldenFile()` calls should update the golden files. + final bool? updateGoldens; + + /// Whether to stop running tests after the first failure. + final bool? failFast; + + /// Additional `--dart-define` values. + @JsonKey(fromJson: _stringList) + final List? dartDefine; + + /// Paths of `.json` or `.env` files with `--dart-define-from-file` values. + @JsonKey(fromJson: _stringList) + final List? dartDefineFromFile; + + /// The platform to run tests on (e.g. `chrome`, `vm`, `android`, `ios`). + final String? platform; + + /// Optional file paths to report coverage information to. + @JsonKey(fromJson: _stringList) + final List? reportOn; + + /// Whether to run skipped tests instead of skipping them. + final bool? runSkipped; + + /// The flavor to build for testing. + final String? flavor; + + /// Maximum seconds to let tests run before killing the process. + @JsonKey(fromJson: _timeout) + final String? timeout; + + @override + List get props => [ + coverage, + optimization, + concurrency, + tags, + excludeCoverage, + excludeTags, + minCoverage, + showUncovered, + collectCoverageFrom, + updateGoldens, + failFast, + dartDefine, + dartDefineFromFile, + platform, + reportOn, + runSkipped, + flavor, + timeout, + ]; +} + +// The coercers below intentionally validate more strictly than the CLI flag +// parser. A value such as `min_coverage: 150` is rejected here at config load +// time even though `--min-coverage 150` is accepted by the flag parser, so +// misconfigured `very_good.yaml` files fail fast with a clear message. + +/// Coerces a `num` or `String` value into a `String`. +/// +/// Options are stored as strings to match the CLI's argument parsing (which +/// always yields strings) but are naturally written as numbers in YAML. +String? _numAsString(Object? value) { + if (value == null) return null; + if (value is num) return value.toString(); + if (value is String) return value; + throw FormatException('Expected a number or string but got `$value`.'); +} + +/// Coerces and validates a positive integer option stored as a string. +/// +/// [key] is the option name used to enrich the error message. +String? _positiveInt(Object? value, String key) { + final asString = _numAsString(value); + if (asString == null) return null; + final parsed = int.tryParse(asString); + if (parsed == null || parsed < 1) { + throw FormatException( + 'Expected `$key` to be a positive integer but got `$asString`.', + ); + } + return asString; +} + +/// Validates and returns the `concurrency` value. +/// +/// Accepts only positive integers. +String? _concurrency(Object? value) => _positiveInt(value, 'concurrency'); + +/// Validates and returns the `timeout` value. +/// +/// Accepts only positive integers (seconds). +String? _timeout(Object? value) => _positiveInt(value, 'timeout'); + +/// Validates and returns the `min_coverage` value. +/// +/// Accepts only a number between 0 and 100 (inclusive). +String? _minCoverage(Object? value) { + final asString = _numAsString(value); + if (asString == null) return null; + final parsed = double.tryParse(asString); + if (parsed == null || parsed < 0 || parsed > 100) { + throw FormatException( + 'Expected `min_coverage` to be a number between 0 and 100 ' + 'but got `$asString`.', + ); + } + return asString; +} + +/// Validates and returns the `collect_coverage_from` value. +/// +/// Accepts only `imports` or `all`. +String? _collectCoverageFrom(Object? value) { + if (value == null) return null; + if (value != 'imports' && value != 'all') { + throw FormatException('Expected `imports` or `all` but got `$value`.'); + } + return value as String; +} + +/// Coerces a single string or a list of strings into a `List`. +List? _stringList(Object? value) { + if (value == null) return null; + if (value is String) return [value]; + if (value is List) { + return value + .map((dynamic e) { + if (e is! String) { + throw FormatException( + 'Expected every entry to be a string but got `$e`.', + ); + } + return e; + }) + .toList(growable: false); + } + throw FormatException( + 'Expected a string or list of strings but got `$value`.', + ); +} diff --git a/lib/src/very_good_config/very_good_config.g.dart b/lib/src/very_good_config/very_good_config.g.dart new file mode 100644 index 000000000..a772995e7 --- /dev/null +++ b/lib/src/very_good_config/very_good_config.g.dart @@ -0,0 +1,91 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'very_good_config.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +VeryGoodConfig _$VeryGoodConfigFromJson(Map json) => + $checkedCreate('VeryGoodConfig', json, ($checkedConvert) { + $checkKeys(json, allowedKeys: const ['test']); + final val = VeryGoodConfig( + test: $checkedConvert( + 'test', + (v) => v == null + ? const VeryGoodTestConfig() + : VeryGoodTestConfig.fromJson(v as Map), + ), + ); + return val; + }); + +VeryGoodTestConfig _$VeryGoodTestConfigFromJson(Map json) => $checkedCreate( + 'VeryGoodTestConfig', + json, + ($checkedConvert) { + $checkKeys( + json, + allowedKeys: const [ + 'coverage', + 'optimization', + 'concurrency', + 'tags', + 'exclude_coverage', + 'exclude_tags', + 'min_coverage', + 'show_uncovered', + 'collect_coverage_from', + 'update_goldens', + 'fail_fast', + 'dart_define', + 'dart_define_from_file', + 'platform', + 'report_on', + 'run_skipped', + 'flavor', + 'timeout', + ], + ); + final val = VeryGoodTestConfig( + coverage: $checkedConvert('coverage', (v) => v as bool?), + optimization: $checkedConvert('optimization', (v) => v as bool?), + concurrency: $checkedConvert('concurrency', (v) => _concurrency(v)), + tags: $checkedConvert('tags', (v) => v as String?), + excludeCoverage: $checkedConvert('exclude_coverage', (v) => v as String?), + excludeTags: $checkedConvert('exclude_tags', (v) => v as String?), + minCoverage: $checkedConvert('min_coverage', (v) => _minCoverage(v)), + showUncovered: $checkedConvert('show_uncovered', (v) => v as bool?), + collectCoverageFrom: $checkedConvert( + 'collect_coverage_from', + (v) => _collectCoverageFrom(v), + ), + updateGoldens: $checkedConvert('update_goldens', (v) => v as bool?), + failFast: $checkedConvert('fail_fast', (v) => v as bool?), + dartDefine: $checkedConvert('dart_define', (v) => _stringList(v)), + dartDefineFromFile: $checkedConvert( + 'dart_define_from_file', + (v) => _stringList(v), + ), + platform: $checkedConvert('platform', (v) => v as String?), + reportOn: $checkedConvert('report_on', (v) => _stringList(v)), + runSkipped: $checkedConvert('run_skipped', (v) => v as bool?), + flavor: $checkedConvert('flavor', (v) => v as String?), + timeout: $checkedConvert('timeout', (v) => _timeout(v)), + ); + return val; + }, + fieldKeyMap: const { + 'excludeCoverage': 'exclude_coverage', + 'excludeTags': 'exclude_tags', + 'minCoverage': 'min_coverage', + 'showUncovered': 'show_uncovered', + 'collectCoverageFrom': 'collect_coverage_from', + 'updateGoldens': 'update_goldens', + 'failFast': 'fail_fast', + 'dartDefine': 'dart_define', + 'dartDefineFromFile': 'dart_define_from_file', + 'reportOn': 'report_on', + 'runSkipped': 'run_skipped', + }, +); diff --git a/pubspec.yaml b/pubspec.yaml index 5883f57b0..6d535e6ef 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,12 +12,14 @@ environment: dependencies: args: ^2.6.0 + checked_yaml: ^2.0.3 cli_completion: ^0.5.1 collection: ^1.19.0 coverage: ^1.15.0 dart_mcp: ^0.5.0 equatable: ^2.0.5 glob: ^2.1.2 + json_annotation: ^4.9.0 lcov_parser: ^0.1.3 mason: ^0.1.0 mason_logger: ^0.3.0 @@ -36,6 +38,7 @@ dependencies: dev_dependencies: build_runner: ^2.4.12 build_verify: ^3.1.0 + json_serializable: ^6.8.0 mocktail: ^1.0.4 test: ^1.25.8 very_good_analysis: ^10.3.0 diff --git a/site/docs/commands/test.md b/site/docs/commands/test.md index 80a90e897..3606d2b33 100644 --- a/site/docs/commands/test.md +++ b/site/docs/commands/test.md @@ -92,3 +92,22 @@ By default, all tests run with optimizations enabled; use the `--no-optimization import 'package:test/test.dart'; ``` + +### Configuring defaults with `very_good.yaml` + +To avoid repeating flags every time you run `very_good test` locally or on CI, you may create a `very_good.yaml` file at the root of your project. The `test` section accepts the same names as the CLI flags in snake_case (e.g. `--min-coverage` becomes `min_coverage`). Values from `very_good.yaml` are used as defaults; anything you pass on the command line takes precedence. + +```yaml +# very_good.yaml +test: + min_coverage: 100 + exclude_coverage: '**/*.g.dart' + report_on: + - lib/ + dart_define: + - FLAVOR=development +``` + +With the file above, running `very_good test` behaves the same as running `very_good test --min-coverage 100 --exclude-coverage '**/*.g.dart' --report-on lib/ --dart-define=FLAVOR=development`. You can still override any of these values on the command line, for example `very_good test --min-coverage 90` to lower the coverage threshold for a single run. + +The `very_good.yaml` file is looked up starting from the directory where the command runs and walking up through its ancestors. The closest file wins; configuration from ancestor directories is not merged. This lets a single `very_good.yaml` at the repository root apply to commands run from any nested package. diff --git a/test/ensure_build_test.dart b/test/ensure_build_test.dart index 8627c2968..743e7d167 100644 --- a/test/ensure_build_test.dart +++ b/test/ensure_build_test.dart @@ -1,4 +1,7 @@ @Tags(['pull-request-only']) +// Building the code generators with AOT compilation can take longer than the +// default 30 second test timeout on CI, so allow additional time. +@Timeout(Duration(minutes: 5)) library; import 'package:build_verify/build_verify.dart'; diff --git a/test/src/commands/test/test_test.dart b/test/src/commands/test/test_test.dart index 5e6e466ea..682e3af8f 100644 --- a/test/src/commands/test/test_test.dart +++ b/test/src/commands/test/test_test.dart @@ -11,6 +11,7 @@ import 'package:path/path.dart' as path; import 'package:test/test.dart'; import 'package:very_good_cli/src/cli/cli.dart'; import 'package:very_good_cli/src/commands/test/test.dart'; +import 'package:very_good_cli/src/very_good_config/very_good_config.dart'; import '../../../helpers/helpers.dart'; @@ -910,5 +911,76 @@ void main() { }, ); }); + + group('very_good.yaml configuration', () { + test( + 'fails with exit code ${ExitCode.config.code} ' + 'when very_good.yaml is malformed', + withRunner((commandRunner, logger, pubUpdater, printLogs) async { + final tempDirectory = Directory.systemTemp.createTempSync(); + addTearDown(() { + Directory.current = cwd; + tempDirectory.deleteSync(recursive: true); + }); + + Directory.current = tempDirectory.path; + File(path.join(tempDirectory.path, 'pubspec.yaml')).createSync(); + File( + path.join(tempDirectory.path, 'very_good.yaml'), + ).writeAsStringSync('- not\n- a\n- map'); + + final result = await commandRunner.run(['test']); + expect(result, equals(ExitCode.config.code)); + verify( + () => logger.err( + any(that: contains('Could not read `very_good.yaml`')), + ), + ).called(1); + }), + ); + + test('applies config value when arg was not parsed', () { + when(() => argResults.wasParsed(any())).thenReturn(false); + final options = FlutterTestOptions.parse( + argResults, + config: const VeryGoodConfig( + test: VeryGoodTestConfig( + minCoverage: '90', + excludeCoverage: '**/*.g.dart', + reportOn: ['lib/'], + ), + ), + ); + expect(options.minCoverage, equals(90)); + expect(options.excludeFromCoverage, equals('**/*.g.dart')); + expect(options.reportOn, equals(['lib/'])); + }); + + test('CLI argument takes precedence over config value', () { + when(() => argResults.wasParsed(any())).thenReturn(false); + when(() => argResults.wasParsed('min-coverage')).thenReturn(true); + when(() => argResults['min-coverage']).thenReturn('50'); + + final options = FlutterTestOptions.parse( + argResults, + config: const VeryGoodConfig( + test: VeryGoodTestConfig(minCoverage: '90'), + ), + ); + expect(options.minCoverage, equals(50)); + }); + + test('falls back to the CLI default when the parsed arg is null ' + 'and the config is unset', () { + when(() => argResults.wasParsed(any())).thenReturn(true); + when( + () => argResults['collect-coverage-from'], + ).thenReturn(null); + + final options = FlutterTestOptions.parse(argResults); + + expect(options.collectCoverageFrom, CoverageCollectionMode.imports); + }); + }); }); } diff --git a/test/src/very_good_config/fixtures/all_test_options.yaml b/test/src/very_good_config/fixtures/all_test_options.yaml new file mode 100644 index 000000000..f5d6c280c --- /dev/null +++ b/test/src/very_good_config/fixtures/all_test_options.yaml @@ -0,0 +1,23 @@ +test: + coverage: true + optimization: false + concurrency: 8 + tags: my-tag + exclude_coverage: "**/*.g.dart" + exclude_tags: skip + min_coverage: 95 + show_uncovered: true + collect_coverage_from: all + update_goldens: true + fail_fast: true + dart_define: + - FOO=bar + - X=42 + dart_define_from_file: defines.env + platform: chrome + report_on: + - lib/ + - packages/foo/lib/ + run_skipped: true + flavor: staging + timeout: 30 diff --git a/test/src/very_good_config/very_good_config_test.dart b/test/src/very_good_config/very_good_config_test.dart new file mode 100644 index 000000000..430e1fded --- /dev/null +++ b/test/src/very_good_config/very_good_config_test.dart @@ -0,0 +1,334 @@ +// Ensures we don't have to use const constructors +// and instances are created at runtime. +// ignore_for_file: prefer_const_constructors + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:very_good_cli/src/very_good_config/very_good_config.dart'; + +void main() { + group(VeryGoodConfig, () { + group('fromString', () { + test('returns empty config when content is empty', () { + expect(VeryGoodConfig.fromString(''), equals(VeryGoodConfig.empty)); + }); + + test('returns empty config when content is null-only', () { + expect(VeryGoodConfig.fromString('~'), equals(VeryGoodConfig.empty)); + }); + + test('throws $VeryGoodConfigParseException when root is not a map', () { + expect( + () => VeryGoodConfig.fromString('- foo\n- bar'), + throwsA(isA()), + ); + }); + + test('throws $VeryGoodConfigParseException when yaml is malformed', () { + expect( + () => VeryGoodConfig.fromString(':\n:'), + throwsA(isA()), + ); + }); + + test('parses all supported test options', () { + final fixture = File( + p.join( + 'test', + 'src', + 'very_good_config', + 'fixtures', + 'all_test_options.yaml', + ), + ); + final config = VeryGoodConfig.fromString(fixture.readAsStringSync()); + + expect(config.test.coverage, isTrue); + expect(config.test.optimization, isFalse); + expect(config.test.concurrency, equals('8')); + expect(config.test.tags, equals('my-tag')); + expect(config.test.excludeCoverage, equals('**/*.g.dart')); + expect(config.test.excludeTags, equals('skip')); + expect(config.test.minCoverage, equals('95')); + expect(config.test.showUncovered, isTrue); + expect(config.test.collectCoverageFrom, equals('all')); + expect(config.test.updateGoldens, isTrue); + expect(config.test.failFast, isTrue); + expect(config.test.dartDefine, equals(['FOO=bar', 'X=42'])); + expect(config.test.dartDefineFromFile, equals(['defines.env'])); + expect(config.test.platform, equals('chrome')); + expect(config.test.reportOn, equals(['lib/', 'packages/foo/lib/'])); + expect(config.test.runSkipped, isTrue); + expect(config.test.flavor, equals('staging')); + expect(config.test.timeout, equals('30')); + }); + + test('parses min-coverage as decimal string', () { + final config = VeryGoodConfig.fromString(''' +test: + min_coverage: 95.5 +'''); + expect(config.test.minCoverage, equals('95.5')); + }); + + test('parses integer options provided as quoted strings', () { + final config = VeryGoodConfig.fromString(''' +test: + concurrency: "8" + timeout: "60" +'''); + expect(config.test.concurrency, equals('8')); + expect(config.test.timeout, equals('60')); + }); + + test('parses min-coverage provided as a quoted string', () { + final config = VeryGoodConfig.fromString(''' +test: + min_coverage: "95" +'''); + expect(config.test.minCoverage, equals('95')); + }); + + test('parses collect-coverage-from with value `imports`', () { + final config = VeryGoodConfig.fromString(''' +test: + collect_coverage_from: imports +'''); + expect(config.test.collectCoverageFrom, equals('imports')); + }); + + test('throws when test section is not a map', () { + expect( + () => VeryGoodConfig.fromString('test: foo'), + throwsA(isA()), + ); + }); + + test('throws when bool option has wrong type', () { + expect( + () => VeryGoodConfig.fromString('test:\n coverage: yes-please'), + throwsA(isA()), + ); + }); + + test('throws when string option has wrong type', () { + expect( + () => VeryGoodConfig.fromString('test:\n tags: [a, b]'), + throwsA(isA()), + ); + }); + + test('throws when integer option has wrong type', () { + expect( + () => VeryGoodConfig.fromString('test:\n concurrency: [1]'), + throwsA(isA()), + ); + }); + + test('throws when number option has wrong type', () { + expect( + () => VeryGoodConfig.fromString('test:\n min_coverage: [95]'), + throwsA(isA()), + ); + }); + + test('throws when collect-coverage-from has invalid value', () { + expect( + () => VeryGoodConfig.fromString( + 'test:\n collect_coverage_from: bad', + ), + throwsA(isA()), + ); + }); + + test('throws when string list has non-string entries', () { + expect( + () => VeryGoodConfig.fromString('test:\n dart_define:\n - 42'), + throwsA(isA()), + ); + }); + + test('throws when string list has wrong type', () { + expect( + () => VeryGoodConfig.fromString('test:\n report_on: 42'), + throwsA(isA()), + ); + }); + + test('throws when an unrecognized root key is present', () { + expect( + () => VeryGoodConfig.fromString('unknown: true'), + throwsA(isA()), + ); + }); + + test('throws when an unrecognized test key is present', () { + expect( + () => VeryGoodConfig.fromString('test:\n min_coverag: 80'), + throwsA(isA()), + ); + }); + + test('throws when min-coverage is below 0', () { + expect( + () => VeryGoodConfig.fromString('test:\n min_coverage: -1'), + throwsA(isA()), + ); + }); + + test('throws when min-coverage is above 100', () { + expect( + () => VeryGoodConfig.fromString('test:\n min_coverage: 101'), + throwsA(isA()), + ); + }); + + test('parses min-coverage at the boundaries', () { + expect( + VeryGoodConfig.fromString( + 'test:\n min_coverage: 0', + ).test.minCoverage, + equals('0'), + ); + expect( + VeryGoodConfig.fromString( + 'test:\n min_coverage: 100', + ).test.minCoverage, + equals('100'), + ); + }); + + test('throws when concurrency is not a positive integer', () { + expect( + () => VeryGoodConfig.fromString('test:\n concurrency: 0'), + throwsA(isA()), + ); + expect( + () => VeryGoodConfig.fromString('test:\n concurrency: -3'), + throwsA(isA()), + ); + expect( + () => VeryGoodConfig.fromString('test:\n concurrency: 1.5'), + throwsA(isA()), + ); + }); + + test('throws when timeout is not a positive integer', () { + expect( + () => VeryGoodConfig.fromString('test:\n timeout: 0'), + throwsA(isA()), + ); + expect( + () => VeryGoodConfig.fromString('test:\n timeout: -30'), + throwsA(isA()), + ); + }); + }); + + group('loadFromClosestAncestor', () { + late Directory tempDir; + late Directory nestedDir; + + setUp(() { + tempDir = Directory.systemTemp.createTempSync('very_good_config_'); + nestedDir = Directory( + p.join(tempDir.path, 'packages', 'foo'), + )..createSync(recursive: true); + }); + + tearDown(() { + if (tempDir.existsSync()) { + tempDir.deleteSync(recursive: true); + } + }); + + test('reads config from the starting directory', () { + File(p.join(nestedDir.path, veryGoodConfigFileName)).writeAsStringSync( + ''' +test: + min_coverage: 80 +''', + ); + final config = VeryGoodConfig.loadFromClosestAncestor(nestedDir); + expect(config.test.minCoverage, equals('80')); + }); + + test('reads config from an ancestor directory', () { + File(p.join(tempDir.path, veryGoodConfigFileName)).writeAsStringSync(''' +test: + min_coverage: 90 +'''); + final config = VeryGoodConfig.loadFromClosestAncestor(nestedDir); + expect(config.test.minCoverage, equals('90')); + }); + + test('prefers the closest config over an ancestor', () { + File(p.join(tempDir.path, veryGoodConfigFileName)).writeAsStringSync(''' +test: + min_coverage: 90 +'''); + File(p.join(nestedDir.path, veryGoodConfigFileName)).writeAsStringSync( + ''' +test: + min_coverage: 80 +''', + ); + final config = VeryGoodConfig.loadFromClosestAncestor(nestedDir); + expect(config.test.minCoverage, equals('80')); + }); + + test('returns empty config when no file is found in any ancestor', () { + expect( + VeryGoodConfig.loadFromClosestAncestor(nestedDir), + equals(VeryGoodConfig.empty), + ); + }); + + test('rethrows parse exception when the closest file is malformed', () { + File( + p.join(nestedDir.path, veryGoodConfigFileName), + ).writeAsStringSync('- not\n- a\n- map'); + expect( + () => VeryGoodConfig.loadFromClosestAncestor(nestedDir), + throwsA(isA()), + ); + }); + }); + + test('supports value equality', () { + expect(VeryGoodConfig(), equals(VeryGoodConfig())); + expect( + VeryGoodConfig(test: VeryGoodTestConfig(coverage: true)), + equals(VeryGoodConfig(test: VeryGoodTestConfig(coverage: true))), + ); + expect( + VeryGoodConfig(test: VeryGoodTestConfig(coverage: true)), + isNot( + equals(VeryGoodConfig(test: VeryGoodTestConfig(coverage: false))), + ), + ); + }); + }); + + group(VeryGoodTestConfig, () { + test('supports value equality', () { + expect( + VeryGoodTestConfig(coverage: true, minCoverage: '95'), + equals(VeryGoodTestConfig(coverage: true, minCoverage: '95')), + ); + expect( + VeryGoodTestConfig(coverage: true), + isNot(equals(VeryGoodTestConfig(coverage: false))), + ); + }); + }); + + group(VeryGoodConfigParseException, () { + test('provides message via toString', () { + const exception = VeryGoodConfigParseException('bad thing'); + expect(exception.toString(), contains('bad thing')); + }); + }); +}