diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index e6bf87f7fa..f3bb78d10f 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -187,6 +187,56 @@ Now, the default settings are: `WarmupCount=1` but you might still overwrite it dotnet run -c Release -- --warmupCount 2 ``` +## Benchmark profiles + +Most of the console arguments describe a *single* job, so they can not express "run these benchmarks +for this particular combination of settings, and that other combination too". Instead of trying to +spell such combinations out on the command line, you can define them in code, give each of them an +`Id`, and then select them from the command line with `--profile`: + +* `--profile` - runs only the jobs whose `Id` matches. Glob patterns are supported and the matching +is case insensitive. Not to be confused with `--profiler`. + +```cs +static void Main(string[] args) + => BenchmarkSwitcher + .FromAssembly(typeof(Program).Assembly) + .Run(args, GetGlobalConfig()); + +static IConfig GetGlobalConfig() + => DefaultConfig.Instance + .AddJob(Job.Default.WithToolchain(InProcessEmitToolchain.Instance).WithId("debug")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("net8")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core90).WithId("net9")); +``` + +Example: run the benchmarks in process, which is handy when you want to debug them. + +```log +dotnet run -c Release -- --filter '*JsonSerialization*' --profile debug +``` + +Example: compare .NET 8.0 with .NET 9.0, without running the `debug` job. Just like `--filter`, +`--profile` accepts more than one value, separated by spaces. + +```log +dotnet run -c Release -- --filter '*' --profile net8 net9 +``` + +The very same thing works for the jobs that are defined via attributes, because `[SimpleJob]` accepts +an `id` argument: + +```cs +[SimpleJob(RuntimeMoniker.Net80, id: "net8")] +[SimpleJob(RuntimeMoniker.Net90, id: "net9")] +public class JsonSerialization { /* ... */ } +``` + +`--profile` narrows down what the other filters have selected, so it can be freely combined with +`--filter`, `--allCategories`, `--anyCategories` and `--attribute`. Jobs that were given no explicit +`Id` are still selectable by their generated one (`DefaultJob`, or `Job-XXXXXX`); if nothing matches, +the list of available profiles is printed for you. + ## Response files support Benchmark.NET supports parsing parameters via response files. for example you can create file `run.rsp` with following content @@ -252,6 +302,8 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 `--envVars ENV_VAR_KEY_1:value_1 ENV_VAR_KEY_2:value_2` * Hide Mean and Ratio columns (use double quotes for multi-word columns: "Alloc Ratio"): `-h Mean Ratio` +* Run only the jobs whose Id is 'net8' or 'net9' (Ids are assigned in code, e.g. Job.Default.WithId("net8")): + `--filter * --profile net8 net9` ## More @@ -264,6 +316,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `-d, --disasm` (Default: false) Gets disassembly of benchmarked code * `-p, --profiler` Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory * `-f, --filter` Glob patterns +* `--profile` Runs only the jobs with matching Id(s). Ids are assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]. Glob patterns are supported. Not to be confused with --profiler. * `-h, --hide` Hides columns by name * `-i, --inProcess` (Default: false) Run benchmarks in Process * `-a, --artifacts` Valid path to accessible directory diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index d0b27d368b..daf9d0054f 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -51,6 +51,9 @@ public bool UseDisassemblyDiagnoser [Option('f', "filter", Required = false, HelpText = "Glob patterns")] public IEnumerable Filters { get; set; } = []; + [Option("profile", Required = false, HelpText = "Runs only the jobs with matching Id(s). Ids are assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]. Glob patterns are supported. Not to be confused with --profiler.")] + public IEnumerable Profiles { get; set; } = []; + [Option('h', "hide", Required = false, HelpText = "Hides columns by name")] public IEnumerable HiddenColumns { get; set; } = []; @@ -268,6 +271,7 @@ public static IEnumerable Examples yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, new CommandLineOptions { EnvironmentVariables = ["ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2"] }); yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = ["Mean", "Ratio"], }); + yield return new Example("Run only the jobs whose Id is 'net8' or 'net9' (Ids are assigned in code, e.g. Job.Default.WithId(\"net8\"))", longName, new CommandLineOptions { Filters = [Escape("*")], Profiles = ["net8", "net9"] }); } } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 671383aee7..e43dd6d6e2 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -388,6 +388,11 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC else config.AddFilter(filters); + // the profile filter selects jobs, not benchmarks, so it's added separately to make sure + // that it's always combined with the filters above rather than being one of them + if (options.Profiles.Any()) + config.AddFilter(new JobIdFilter([.. options.Profiles])); + config.HideColumns(options.HiddenColumns.ToArray()); config.WithOption(ConfigOptions.JoinSummary, options.Join); diff --git a/src/BenchmarkDotNet/Filters/JobIdFilter.cs b/src/BenchmarkDotNet/Filters/JobIdFilter.cs new file mode 100644 index 0000000000..9110022f45 --- /dev/null +++ b/src/BenchmarkDotNet/Filters/JobIdFilter.cs @@ -0,0 +1,47 @@ +using BenchmarkDotNet.Running; +using System.Text.RegularExpressions; + +namespace BenchmarkDotNet.Filters +{ + /// + /// filters benchmarks by the id of the job they belong to (glob patterns are supported) + /// + public class JobIdFilter : IFilter + { + private readonly Regex[] patterns; + + // The available job ids are not known upfront because the jobs defined via attributes are discovered + // while the benchmark cases are being created. They are recorded here so that when no job matches we + // can tell the user which ids they could have used instead. + private readonly HashSet observedJobIds = new HashSet(StringComparer.OrdinalIgnoreCase); + private readonly HashSet matchedJobIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + public JobIdFilter(string[] jobIds) => patterns = GlobFilter.ToRegex(jobIds); + + /// + /// the ids of all the jobs this filter has been asked about. + /// ImmutableConfigBuilder keeps the filters in an unordered set, so which benchmark cases reach + /// this filter depends on how it happens to be ordered against the other filters. That makes this + /// a lower bound of what the user could have typed, which is all we need it for. + /// + internal IReadOnlyCollection ObservedJobIds => observedJobIds; + + /// + /// the ids of the jobs that this filter has accepted + /// + internal IReadOnlyCollection MatchedJobIds => matchedJobIds; + + public bool Predicate(BenchmarkCase benchmarkCase) + { + string jobId = benchmarkCase.Job.ResolvedId; + + observedJobIds.Add(jobId); + + if (!patterns.Any(pattern => pattern.IsMatch(jobId))) + return false; + + matchedJobIds.Add(jobId); + return true; + } + } +} diff --git a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs index 8697a5086e..ae33b90898 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Parameters; @@ -18,6 +19,8 @@ namespace BenchmarkDotNet.Running { public class BenchmarkSwitcher { + private const int MaxDisplayedProfiles = 40; + private readonly IUserInteraction userInteraction = new UserInteraction(); private readonly List types = []; private readonly List assemblies = []; @@ -165,13 +168,41 @@ internal async ValueTask> RunWithDirtyAssemblyResolveHelper if (filteredBenchmarks.IsEmpty()) { - userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, [.. options.Filters]); + if (!TryPrintWrongProfileInfo(effectiveConfig, logger, options)) + userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, [.. options.Filters]); return []; } return await BenchmarkRunnerClean.Run(filteredBenchmarks, cancellationToken).ConfigureAwait(false); } + /// + /// explains that it was --profile which returned 0 benchmarks, so that the user is not shown + /// benchmark name suggestions for what is actually a mistyped job id. + /// + /// true if the profile was the reason why nothing was left to run + private static bool TryPrintWrongProfileInfo(IConfig effectiveConfig, ILogger logger, CommandLineOptions options) + { + var profiles = options.Profiles.ToArray(); + if (profiles.IsEmpty()) + return false; + + var profileFilter = effectiveConfig.GetFilters().OfType().FirstOrDefault(); + if (profileFilter == null || profileFilter.ObservedJobIds.IsEmpty() || !profileFilter.MatchedJobIds.IsEmpty()) + return false; // either no job was ever checked, or some did match, so --profile is not to blame + + logger.WriteLineError($"{(profiles.Length == 1 ? "The profile" : "Profiles")} '{string.Join("', '", profiles)}' that you have provided returned 0 benchmarks."); + logger.WriteLineInfo("Please remember that --profile is applied to the Id of the job, which is assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]."); + logger.WriteLineInfo("Available profiles:"); + + foreach (string jobId in profileFilter.ObservedJobIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase).Take(MaxDisplayedProfiles)) + logger.WriteLineInfo($"\t{jobId}"); + + logger.WriteLineInfo("To learn more about filtering use `--help`."); + + return true; + } + private static void PrintList(ILogger nonNullLogger, IConfig effectiveConfig, IReadOnlyList allAvailableTypesWithRunnableBenchmarks, CommandLineOptions options) { var printer = new BenchmarkCasesPrinter(options.ListBenchmarkCaseMode); diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index 9f5fcc60a2..d56d8620bc 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Helpers; @@ -301,6 +302,90 @@ public void JobNotDefinedButStillBenchmarkIsExecuted() Assert.True(results.All(r => r.BenchmarksCases.All(bc => bc.Job == Job.Default))); } + [Fact] + public void WhenUserProvidesProfileOnlyTheJobsWithMatchingIdAreExecuted() + { + var types = new[] { typeof(ClassB) }; + var switcher = new BenchmarkSwitcher(types); + var configWithNamedJobs = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("first")) + .AddJob(Job.Dry.WithId("second")); + + var results = switcher.Run(["--filter", "*Method3", "--profile", "second"], configWithNamedJobs); + + var jobs = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job)).ToArray(); + Assert.Equal("second", Assert.Single(jobs).ResolvedId); + } + + [Fact] + public void WhenUserProvidesProfileGlobPatternAllTheMatchingJobsAreExecuted() + { + var types = new[] { typeof(ClassB) }; + var switcher = new BenchmarkSwitcher(types); + var configWithNamedJobs = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8")) + .AddJob(Job.Dry.WithId("net9")) + .AddJob(Job.Dry.WithId("debug")); + + var results = switcher.Run(["--filter", "*Method3", "--profile", "net*"], configWithNamedJobs); + + var jobIds = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job.ResolvedId)).OrderBy(id => id).ToArray(); + Assert.Equal(["net8", "net9"], jobIds); + } + + [Fact] + public void WhenUserProvidesProfileTheJobsDefinedViaAttributesAreSelectableToo() + { + var types = new[] { typeof(WithTwoNamedJobAttributes) }; + var switcher = new BenchmarkSwitcher(types); + var config = ManualConfig.CreateEmpty(); + + var results = switcher.Run(["--filter", "*WithTwoNamedJobAttributes*", "--profile", "secondAttributeJob"], config); + + var jobs = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job)).ToArray(); + Assert.Equal("secondAttributeJob", Assert.Single(jobs).ResolvedId); + } + + [Fact] + public void WhenProfileReturnsNothingTheAvailableProfilesAreDisplayedAndNoBenchmarksAreExecuted() + { + var logger = new OutputLogger(Output); + var configWithNamedJobs = ManualConfig.CreateEmpty().AddLogger(logger) + .AddJob(Job.Dry.WithId("first")) + .AddJob(Job.Dry.WithId("second")); + + var summaries = BenchmarkSwitcher + .FromTypes([typeof(ClassB)]) + .Run(["--filter", "*Method3", "--profile", "WRONG"], configWithNamedJobs); + + Assert.Empty(summaries); + + string log = logger.GetLog(); + Assert.Contains("The profile 'WRONG' that you have provided returned 0 benchmarks.", log); + Assert.Contains("Available profiles:", log); + Assert.Contains("first", log); + Assert.Contains("second", log); + // the profile is not a benchmark name, so we must not suggest benchmark names + Assert.DoesNotContain("Please remember that the filter is applied to full benchmark name", log); + } + + [Fact] + public void WhenUserProvidesBothProfileAndFilterBothAreApplied() + { + var logger = new OutputLogger(Output); + var configWithNamedJobs = ManualConfig.CreateEmpty().AddLogger(logger) + .AddJob(Job.Dry.WithId("first")) + .AddJob(Job.Dry.WithId("second")); + + // the profile exists, but the filter matches nothing, so this is a wrong *filter*, not a wrong profile + var summaries = BenchmarkSwitcher + .FromTypes([typeof(ClassB)]) + .Run(["--filter", "WRONG", "--profile", "first"], configWithNamedJobs); + + Assert.Empty(summaries); + Assert.Contains("The filter 'WRONG' that you have provided returned 0 benchmarks.", logger.GetLog()); + } + [Fact] public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromTypes() { @@ -416,6 +501,14 @@ public class JustBenchmark public void Method() { } } + [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 1, iterationCount: 1, id: "firstAttributeJob")] + [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 2, iterationCount: 1, id: "secondAttributeJob")] + public class WithTwoNamedJobAttributes + { + [Benchmark] + public void Method() { } + } + public class MockExporter : ExporterBase { public bool exported = false; diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 09832a6c8d..8640456e2d 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Exporters.OpenMetrics; using BenchmarkDotNet.Exporters.Xml; +using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -823,6 +824,40 @@ public void UserCanSpecifyWasmMainJsTemplate() Assert.Equal("dummyFile.js", runtime.MainJsTemplate?.Name); } + [Fact] + public void NoProfileFilterIsAddedWhenProfileIsNotSpecified() + { + var config = ConfigParser.Parse(["--filter", "*"], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Empty(config.GetFilters().OfType()); + } + + [Theory] + [InlineData("--profile", "net8")] + [InlineData("--profile", "net8", "net9")] + [InlineData("--PROFILE", "net8")] // case insensitive + public void UserCanSpecifyProfiles(params string[] args) + { + var config = ConfigParser.Parse(args, new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Single(config.GetFilters().OfType()); + } + + [Fact] + public void ProfileFilterIsNotUnionedWithTheOtherFilters() + { + // --profile must narrow down what --filter has selected, so it must be a filter of its own + var config = ConfigParser.Parse(["--filter", "*", "--profile", "net8"], new OutputLogger(Output)).config; + + Assert.NotNull(config); + var filters = config.GetFilters().ToArray(); + Assert.Equal(2, filters.Length); + Assert.Single(filters.OfType()); + Assert.Single(filters.OfType()); + } + [Theory] [InlineData("--filter abc", "--filter *")] [InlineData("-f abc", "--filter *")] diff --git a/tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs b/tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs new file mode 100644 index 0000000000..3ab93ebd06 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs @@ -0,0 +1,107 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Tests +{ + public class JobIdFilterTests + { + private static readonly IConfig ConfigWithNamedJobs = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8")) + .AddJob(Job.Dry.WithId("net9")) + .AddJob(Job.Dry.WithId("debug")); + + [Theory] + [InlineData("net8", 1)] + [InlineData("NET8", 1)] // case insensitive + [InlineData("net9", 1)] + [InlineData("debug", 1)] + [InlineData("net*", 2)] // glob + [InlineData("*", 3)] + [InlineData("net", 0)] // it's an exact match, not a substring one + [InlineData("WRONG", 0)] + public void TheFilterSelectsBenchmarksByJobId(string pattern, int expectedBenchmarks) + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + Assert.Equal(3, benchmarkCases.Length); // one per job + + var filter = new JobIdFilter([pattern]); + + Assert.Equal(expectedBenchmarks, benchmarkCases.Count(benchmarkCase => filter.Predicate(benchmarkCase))); + } + + [Fact] + public void MultiplePatternsAreCombinedWithOr() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + + var filter = new JobIdFilter(["net8", "net9"]); + + var matched = benchmarkCases.Where(benchmarkCase => filter.Predicate(benchmarkCase)).ToArray(); + + Assert.Equal(2, matched.Length); + Assert.Equal(["net8", "net9"], matched.Select(benchmarkCase => benchmarkCase.Job.ResolvedId).OrderBy(id => id)); + } + + [Fact] + public void JobsDefinedViaAttributesCanBeSelectedToo() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithNamedJobAttributes)).BenchmarksCases; + + var filter = new JobIdFilter(["fromAttribute"]); + + var matched = Assert.Single(benchmarkCases, benchmarkCase => filter.Predicate(benchmarkCase)); + Assert.Equal("fromAttribute", matched.Job.ResolvedId); + } + + [Fact] + public void JobsWithNoExplicitIdAreMatchedByTheirGeneratedId() + { + var benchmarkCase = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark)).BenchmarksCases.Single(); + Assert.Equal("DefaultJob", benchmarkCase.Job.ResolvedId); + + Assert.True(new JobIdFilter(["DefaultJob"]).Predicate(benchmarkCase)); + Assert.False(new JobIdFilter(["net8"]).Predicate(benchmarkCase)); + } + + [Fact] + public void TheFilterRecordsWhatItHasObservedAndMatched() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + + var filter = new JobIdFilter(["net8"]); + foreach (var benchmarkCase in benchmarkCases) + filter.Predicate(benchmarkCase); + + Assert.Equal(["debug", "net8", "net9"], filter.ObservedJobIds.OrderBy(id => id)); + Assert.Equal(["net8"], filter.MatchedJobIds); + } + + [Fact] + public void TheFilterRecordsNoMatchWhenNoJobIdMatches() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + + var filter = new JobIdFilter(["typo"]); + foreach (var benchmarkCase in benchmarkCases) + filter.Predicate(benchmarkCase); + + Assert.NotEmpty(filter.ObservedJobIds); + Assert.Empty(filter.MatchedJobIds); + } + } + + public class TypeWithSingleBenchmark + { + [Benchmark] public void TheBenchmark() { } + } + + [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 1, iterationCount: 1, id: "fromAttribute")] + public class TypeWithNamedJobAttributes + { + [Benchmark] public void TheBenchmark() { } + } +}