From 78b5d49d5ba571717ffbaf2bc5a6b9875e7c42c5 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:31:54 +0100 Subject: [PATCH 1/8] Add docs for --profile argument and usage examples Added documentation for the new `--profile` console argument, detailing how to select benchmark jobs by Id, use glob patterns, and distinguish it from `--profiler`. Included code and command-line examples, and updated the options summary. --- docs/articles/guides/console-args.md | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index e6bf87f7fa..f3bb78d10f 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -187,6 +187,56 @@ Now, the default settings are: `WarmupCount=1` but you might still overwrite it dotnet run -c Release -- --warmupCount 2 ``` +## Benchmark profiles + +Most of the console arguments describe a *single* job, so they can not express "run these benchmarks +for this particular combination of settings, and that other combination too". Instead of trying to +spell such combinations out on the command line, you can define them in code, give each of them an +`Id`, and then select them from the command line with `--profile`: + +* `--profile` - runs only the jobs whose `Id` matches. Glob patterns are supported and the matching +is case insensitive. Not to be confused with `--profiler`. + +```cs +static void Main(string[] args) + => BenchmarkSwitcher + .FromAssembly(typeof(Program).Assembly) + .Run(args, GetGlobalConfig()); + +static IConfig GetGlobalConfig() + => DefaultConfig.Instance + .AddJob(Job.Default.WithToolchain(InProcessEmitToolchain.Instance).WithId("debug")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("net8")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core90).WithId("net9")); +``` + +Example: run the benchmarks in process, which is handy when you want to debug them. + +```log +dotnet run -c Release -- --filter '*JsonSerialization*' --profile debug +``` + +Example: compare .NET 8.0 with .NET 9.0, without running the `debug` job. Just like `--filter`, +`--profile` accepts more than one value, separated by spaces. + +```log +dotnet run -c Release -- --filter '*' --profile net8 net9 +``` + +The very same thing works for the jobs that are defined via attributes, because `[SimpleJob]` accepts +an `id` argument: + +```cs +[SimpleJob(RuntimeMoniker.Net80, id: "net8")] +[SimpleJob(RuntimeMoniker.Net90, id: "net9")] +public class JsonSerialization { /* ... */ } +``` + +`--profile` narrows down what the other filters have selected, so it can be freely combined with +`--filter`, `--allCategories`, `--anyCategories` and `--attribute`. Jobs that were given no explicit +`Id` are still selectable by their generated one (`DefaultJob`, or `Job-XXXXXX`); if nothing matches, +the list of available profiles is printed for you. + ## Response files support Benchmark.NET supports parsing parameters via response files. for example you can create file `run.rsp` with following content @@ -252,6 +302,8 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 `--envVars ENV_VAR_KEY_1:value_1 ENV_VAR_KEY_2:value_2` * Hide Mean and Ratio columns (use double quotes for multi-word columns: "Alloc Ratio"): `-h Mean Ratio` +* Run only the jobs whose Id is 'net8' or 'net9' (Ids are assigned in code, e.g. Job.Default.WithId("net8")): + `--filter * --profile net8 net9` ## More @@ -264,6 +316,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `-d, --disasm` (Default: false) Gets disassembly of benchmarked code * `-p, --profiler` Profiles benchmarked code using selected profiler. Available options: EP/ETW/CV/NativeMemory * `-f, --filter` Glob patterns +* `--profile` Runs only the jobs with matching Id(s). Ids are assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]. Glob patterns are supported. Not to be confused with --profiler. * `-h, --hide` Hides columns by name * `-i, --inProcess` (Default: false) Run benchmarks in Process * `-a, --artifacts` Valid path to accessible directory From f832de51b508184da7f1dee011ea751844392b8d Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:32:13 +0100 Subject: [PATCH 2/8] Add tests for --profile option in BenchmarkSwitcher Added tests to BenchmarkSwitcherTest.cs covering --profile usage: matching job IDs, glob patterns, attribute-defined jobs, error messages for unmatched profiles, and combined profile/filter scenarios. Introduced WithTwoNamedJobAttributes for attribute-based job selection tests. --- .../BenchmarkSwitcherTest.cs | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index 9f5fcc60a2..d56d8620bc 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Helpers; @@ -301,6 +302,90 @@ public void JobNotDefinedButStillBenchmarkIsExecuted() Assert.True(results.All(r => r.BenchmarksCases.All(bc => bc.Job == Job.Default))); } + [Fact] + public void WhenUserProvidesProfileOnlyTheJobsWithMatchingIdAreExecuted() + { + var types = new[] { typeof(ClassB) }; + var switcher = new BenchmarkSwitcher(types); + var configWithNamedJobs = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("first")) + .AddJob(Job.Dry.WithId("second")); + + var results = switcher.Run(["--filter", "*Method3", "--profile", "second"], configWithNamedJobs); + + var jobs = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job)).ToArray(); + Assert.Equal("second", Assert.Single(jobs).ResolvedId); + } + + [Fact] + public void WhenUserProvidesProfileGlobPatternAllTheMatchingJobsAreExecuted() + { + var types = new[] { typeof(ClassB) }; + var switcher = new BenchmarkSwitcher(types); + var configWithNamedJobs = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8")) + .AddJob(Job.Dry.WithId("net9")) + .AddJob(Job.Dry.WithId("debug")); + + var results = switcher.Run(["--filter", "*Method3", "--profile", "net*"], configWithNamedJobs); + + var jobIds = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job.ResolvedId)).OrderBy(id => id).ToArray(); + Assert.Equal(["net8", "net9"], jobIds); + } + + [Fact] + public void WhenUserProvidesProfileTheJobsDefinedViaAttributesAreSelectableToo() + { + var types = new[] { typeof(WithTwoNamedJobAttributes) }; + var switcher = new BenchmarkSwitcher(types); + var config = ManualConfig.CreateEmpty(); + + var results = switcher.Run(["--filter", "*WithTwoNamedJobAttributes*", "--profile", "secondAttributeJob"], config); + + var jobs = results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job)).ToArray(); + Assert.Equal("secondAttributeJob", Assert.Single(jobs).ResolvedId); + } + + [Fact] + public void WhenProfileReturnsNothingTheAvailableProfilesAreDisplayedAndNoBenchmarksAreExecuted() + { + var logger = new OutputLogger(Output); + var configWithNamedJobs = ManualConfig.CreateEmpty().AddLogger(logger) + .AddJob(Job.Dry.WithId("first")) + .AddJob(Job.Dry.WithId("second")); + + var summaries = BenchmarkSwitcher + .FromTypes([typeof(ClassB)]) + .Run(["--filter", "*Method3", "--profile", "WRONG"], configWithNamedJobs); + + Assert.Empty(summaries); + + string log = logger.GetLog(); + Assert.Contains("The profile 'WRONG' that you have provided returned 0 benchmarks.", log); + Assert.Contains("Available profiles:", log); + Assert.Contains("first", log); + Assert.Contains("second", log); + // the profile is not a benchmark name, so we must not suggest benchmark names + Assert.DoesNotContain("Please remember that the filter is applied to full benchmark name", log); + } + + [Fact] + public void WhenUserProvidesBothProfileAndFilterBothAreApplied() + { + var logger = new OutputLogger(Output); + var configWithNamedJobs = ManualConfig.CreateEmpty().AddLogger(logger) + .AddJob(Job.Dry.WithId("first")) + .AddJob(Job.Dry.WithId("second")); + + // the profile exists, but the filter matches nothing, so this is a wrong *filter*, not a wrong profile + var summaries = BenchmarkSwitcher + .FromTypes([typeof(ClassB)]) + .Run(["--filter", "WRONG", "--profile", "first"], configWithNamedJobs); + + Assert.Empty(summaries); + Assert.Contains("The filter 'WRONG' that you have provided returned 0 benchmarks.", logger.GetLog()); + } + [Fact] public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromTypes() { @@ -416,6 +501,14 @@ public class JustBenchmark public void Method() { } } + [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 1, iterationCount: 1, id: "firstAttributeJob")] + [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 2, iterationCount: 1, id: "secondAttributeJob")] + public class WithTwoNamedJobAttributes + { + [Benchmark] + public void Method() { } + } + public class MockExporter : ExporterBase { public bool exported = false; From 5d7d926f5403fabc0a6de1510df2a8c3c70e6858 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:32:29 +0100 Subject: [PATCH 3/8] Add JobIdFilterTests to cover job ID filtering logic Introduced JobIdFilterTests to validate filtering benchmarks by job ID, including exact, case-insensitive, glob patterns, and attribute-defined jobs. Tests also verify recording of observed and matched job IDs. Added helper benchmark classes for test scenarios. --- .../BenchmarkDotNet.Tests/JobIdFilterTests.cs | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs diff --git a/tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs b/tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs new file mode 100644 index 0000000000..3ab93ebd06 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/JobIdFilterTests.cs @@ -0,0 +1,107 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Tests +{ + public class JobIdFilterTests + { + private static readonly IConfig ConfigWithNamedJobs = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8")) + .AddJob(Job.Dry.WithId("net9")) + .AddJob(Job.Dry.WithId("debug")); + + [Theory] + [InlineData("net8", 1)] + [InlineData("NET8", 1)] // case insensitive + [InlineData("net9", 1)] + [InlineData("debug", 1)] + [InlineData("net*", 2)] // glob + [InlineData("*", 3)] + [InlineData("net", 0)] // it's an exact match, not a substring one + [InlineData("WRONG", 0)] + public void TheFilterSelectsBenchmarksByJobId(string pattern, int expectedBenchmarks) + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + Assert.Equal(3, benchmarkCases.Length); // one per job + + var filter = new JobIdFilter([pattern]); + + Assert.Equal(expectedBenchmarks, benchmarkCases.Count(benchmarkCase => filter.Predicate(benchmarkCase))); + } + + [Fact] + public void MultiplePatternsAreCombinedWithOr() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + + var filter = new JobIdFilter(["net8", "net9"]); + + var matched = benchmarkCases.Where(benchmarkCase => filter.Predicate(benchmarkCase)).ToArray(); + + Assert.Equal(2, matched.Length); + Assert.Equal(["net8", "net9"], matched.Select(benchmarkCase => benchmarkCase.Job.ResolvedId).OrderBy(id => id)); + } + + [Fact] + public void JobsDefinedViaAttributesCanBeSelectedToo() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithNamedJobAttributes)).BenchmarksCases; + + var filter = new JobIdFilter(["fromAttribute"]); + + var matched = Assert.Single(benchmarkCases, benchmarkCase => filter.Predicate(benchmarkCase)); + Assert.Equal("fromAttribute", matched.Job.ResolvedId); + } + + [Fact] + public void JobsWithNoExplicitIdAreMatchedByTheirGeneratedId() + { + var benchmarkCase = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark)).BenchmarksCases.Single(); + Assert.Equal("DefaultJob", benchmarkCase.Job.ResolvedId); + + Assert.True(new JobIdFilter(["DefaultJob"]).Predicate(benchmarkCase)); + Assert.False(new JobIdFilter(["net8"]).Predicate(benchmarkCase)); + } + + [Fact] + public void TheFilterRecordsWhatItHasObservedAndMatched() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + + var filter = new JobIdFilter(["net8"]); + foreach (var benchmarkCase in benchmarkCases) + filter.Predicate(benchmarkCase); + + Assert.Equal(["debug", "net8", "net9"], filter.ObservedJobIds.OrderBy(id => id)); + Assert.Equal(["net8"], filter.MatchedJobIds); + } + + [Fact] + public void TheFilterRecordsNoMatchWhenNoJobIdMatches() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(TypeWithSingleBenchmark), ConfigWithNamedJobs).BenchmarksCases; + + var filter = new JobIdFilter(["typo"]); + foreach (var benchmarkCase in benchmarkCases) + filter.Predicate(benchmarkCase); + + Assert.NotEmpty(filter.ObservedJobIds); + Assert.Empty(filter.MatchedJobIds); + } + } + + public class TypeWithSingleBenchmark + { + [Benchmark] public void TheBenchmark() { } + } + + [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 1, iterationCount: 1, id: "fromAttribute")] + public class TypeWithNamedJobAttributes + { + [Benchmark] public void TheBenchmark() { } + } +} From 21b7b69091f4a01c6ec9b9e8d849d4562dec3701 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:32:44 +0100 Subject: [PATCH 4/8] Add tests for profile and filter parsing in ConfigParser Added tests to ensure correct handling of --profile and --filter arguments in BenchmarkDotNet config parsing. Verified that JobIdFilter is only added when a profile is specified, profiles are case-insensitive, and profile filters are not unioned with other filters. Improves test coverage for command-line argument parsing. --- .../ConfigParserTests.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 09832a6c8d..8640456e2d 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Exporters.OpenMetrics; using BenchmarkDotNet.Exporters.Xml; +using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -823,6 +824,40 @@ public void UserCanSpecifyWasmMainJsTemplate() Assert.Equal("dummyFile.js", runtime.MainJsTemplate?.Name); } + [Fact] + public void NoProfileFilterIsAddedWhenProfileIsNotSpecified() + { + var config = ConfigParser.Parse(["--filter", "*"], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Empty(config.GetFilters().OfType()); + } + + [Theory] + [InlineData("--profile", "net8")] + [InlineData("--profile", "net8", "net9")] + [InlineData("--PROFILE", "net8")] // case insensitive + public void UserCanSpecifyProfiles(params string[] args) + { + var config = ConfigParser.Parse(args, new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Single(config.GetFilters().OfType()); + } + + [Fact] + public void ProfileFilterIsNotUnionedWithTheOtherFilters() + { + // --profile must narrow down what --filter has selected, so it must be a filter of its own + var config = ConfigParser.Parse(["--filter", "*", "--profile", "net8"], new OutputLogger(Output)).config; + + Assert.NotNull(config); + var filters = config.GetFilters().ToArray(); + Assert.Equal(2, filters.Length); + Assert.Single(filters.OfType()); + Assert.Single(filters.OfType()); + } + [Theory] [InlineData("--filter abc", "--filter *")] [InlineData("-f abc", "--filter *")] From fe1f253d14b3ec5cefab83ebac1b4b22dd6e7272 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:33:00 +0100 Subject: [PATCH 5/8] Add JobIdFilter for filtering benchmarks by job ID Introduced JobIdFilter in BenchmarkDotNet.Filters to enable filtering benchmarks using job IDs with glob pattern support. The filter tracks observed and matched job IDs, exposes them for reporting, and applies filtering logic via regular expressions in the Predicate method. --- src/BenchmarkDotNet/Filters/JobIdFilter.cs | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/BenchmarkDotNet/Filters/JobIdFilter.cs diff --git a/src/BenchmarkDotNet/Filters/JobIdFilter.cs b/src/BenchmarkDotNet/Filters/JobIdFilter.cs new file mode 100644 index 0000000000..9110022f45 --- /dev/null +++ b/src/BenchmarkDotNet/Filters/JobIdFilter.cs @@ -0,0 +1,47 @@ +using BenchmarkDotNet.Running; +using System.Text.RegularExpressions; + +namespace BenchmarkDotNet.Filters +{ + /// + /// filters benchmarks by the id of the job they belong to (glob patterns are supported) + /// + public class JobIdFilter : IFilter + { + private readonly Regex[] patterns; + + // The available job ids are not known upfront because the jobs defined via attributes are discovered + // while the benchmark cases are being created. They are recorded here so that when no job matches we + // can tell the user which ids they could have used instead. + private readonly HashSet observedJobIds = new HashSet(StringComparer.OrdinalIgnoreCase); + private readonly HashSet matchedJobIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + public JobIdFilter(string[] jobIds) => patterns = GlobFilter.ToRegex(jobIds); + + /// + /// the ids of all the jobs this filter has been asked about. + /// ImmutableConfigBuilder keeps the filters in an unordered set, so which benchmark cases reach + /// this filter depends on how it happens to be ordered against the other filters. That makes this + /// a lower bound of what the user could have typed, which is all we need it for. + /// + internal IReadOnlyCollection ObservedJobIds => observedJobIds; + + /// + /// the ids of the jobs that this filter has accepted + /// + internal IReadOnlyCollection MatchedJobIds => matchedJobIds; + + public bool Predicate(BenchmarkCase benchmarkCase) + { + string jobId = benchmarkCase.Job.ResolvedId; + + observedJobIds.Add(jobId); + + if (!patterns.Any(pattern => pattern.IsMatch(jobId))) + return false; + + matchedJobIds.Add(jobId); + return true; + } + } +} From bf7bdf666f910cc31b12d1d06d043ec54423279b Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:33:13 +0100 Subject: [PATCH 6/8] Add --profile option to filter jobs by Id in CLI Introduced a Profiles command-line option to CommandLineOptions, enabling users to run only jobs with matching Id(s) using glob patterns. Updated help text to clarify its distinction from the Profiler option and added a usage example for --profile. --- src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index d0b27d368b..daf9d0054f 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -51,6 +51,9 @@ public bool UseDisassemblyDiagnoser [Option('f', "filter", Required = false, HelpText = "Glob patterns")] public IEnumerable Filters { get; set; } = []; + [Option("profile", Required = false, HelpText = "Runs only the jobs with matching Id(s). Ids are assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]. Glob patterns are supported. Not to be confused with --profiler.")] + public IEnumerable Profiles { get; set; } = []; + [Option('h', "hide", Required = false, HelpText = "Hides columns by name")] public IEnumerable HiddenColumns { get; set; } = []; @@ -268,6 +271,7 @@ public static IEnumerable Examples yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, new CommandLineOptions { EnvironmentVariables = ["ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2"] }); yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = ["Mean", "Ratio"], }); + yield return new Example("Run only the jobs whose Id is 'net8' or 'net9' (Ids are assigned in code, e.g. Job.Default.WithId(\"net8\"))", longName, new CommandLineOptions { Filters = [Escape("*")], Profiles = ["net8", "net9"] }); } } From 535429cdb099a4a63ab6c478aeee8a45ff634530 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:33:24 +0100 Subject: [PATCH 7/8] Add JobIdFilter for profiles in config parsing Ensure JobIdFilter is added when profiles are specified, so job selection by profile is always combined with other filters. Added comments to clarify filter handling logic. --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 671383aee7..e43dd6d6e2 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -388,6 +388,11 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC else config.AddFilter(filters); + // the profile filter selects jobs, not benchmarks, so it's added separately to make sure + // that it's always combined with the filters above rather than being one of them + if (options.Profiles.Any()) + config.AddFilter(new JobIdFilter([.. options.Profiles])); + config.HideColumns(options.HiddenColumns.ToArray()); config.WithOption(ConfigOptions.JoinSummary, options.Join); From 84dd562e1a438ff3df2101426b5bcac19de13857 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Thu, 30 Jul 2026 10:33:39 +0100 Subject: [PATCH 8/8] Improve diagnostics for --profile filter in BenchmarkSwitcher Added TryPrintWrongProfileInfo to clarify when --profile returns no benchmarks, providing targeted error messages and listing available profiles. Integrated this logic into the filtering process and limited displayed profiles to 40. Added necessary using directive for BenchmarkDotNet.Filters. --- .../Running/BenchmarkSwitcher.cs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs index 8697a5086e..ae33b90898 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Parameters; @@ -18,6 +19,8 @@ namespace BenchmarkDotNet.Running { public class BenchmarkSwitcher { + private const int MaxDisplayedProfiles = 40; + private readonly IUserInteraction userInteraction = new UserInteraction(); private readonly List types = []; private readonly List assemblies = []; @@ -165,13 +168,41 @@ internal async ValueTask> RunWithDirtyAssemblyResolveHelper if (filteredBenchmarks.IsEmpty()) { - userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, [.. options.Filters]); + if (!TryPrintWrongProfileInfo(effectiveConfig, logger, options)) + userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, [.. options.Filters]); return []; } return await BenchmarkRunnerClean.Run(filteredBenchmarks, cancellationToken).ConfigureAwait(false); } + /// + /// explains that it was --profile which returned 0 benchmarks, so that the user is not shown + /// benchmark name suggestions for what is actually a mistyped job id. + /// + /// true if the profile was the reason why nothing was left to run + private static bool TryPrintWrongProfileInfo(IConfig effectiveConfig, ILogger logger, CommandLineOptions options) + { + var profiles = options.Profiles.ToArray(); + if (profiles.IsEmpty()) + return false; + + var profileFilter = effectiveConfig.GetFilters().OfType().FirstOrDefault(); + if (profileFilter == null || profileFilter.ObservedJobIds.IsEmpty() || !profileFilter.MatchedJobIds.IsEmpty()) + return false; // either no job was ever checked, or some did match, so --profile is not to blame + + logger.WriteLineError($"{(profiles.Length == 1 ? "The profile" : "Profiles")} '{string.Join("', '", profiles)}' that you have provided returned 0 benchmarks."); + logger.WriteLineInfo("Please remember that --profile is applied to the Id of the job, which is assigned in code via Job.WithId(...) or the 'id' argument of [SimpleJob]."); + logger.WriteLineInfo("Available profiles:"); + + foreach (string jobId in profileFilter.ObservedJobIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase).Take(MaxDisplayedProfiles)) + logger.WriteLineInfo($"\t{jobId}"); + + logger.WriteLineInfo("To learn more about filtering use `--help`."); + + return true; + } + private static void PrintList(ILogger nonNullLogger, IConfig effectiveConfig, IReadOnlyList allAvailableTypesWithRunnableBenchmarks, CommandLineOptions options) { var printer = new BenchmarkCasesPrinter(options.ListBenchmarkCaseMode);