diff --git a/docs/articles/configs/exporters.md b/docs/articles/configs/exporters.md index 5e65b8e55e..d83a3e566f 100644 --- a/docs/articles/configs/exporters.md +++ b/docs/articles/configs/exporters.md @@ -10,6 +10,16 @@ By default, files with results will be located in `.\BenchmarkDotNet.Artifacts\results` directory, but this can be changed via the `ArtifactsPath` property in the `IConfig`. Default exporters are: csv, html and markdown. +The exported files are named after the title of the summary they belong to, which by default is the full name of +the benchmarked type (or `BenchmarkRun-joined` when the results of many types are joined into a single summary). +You can override it via the `Title` property in the `IConfig` (`--title` on the command line); when the run reports +more than one summary, the name of the benchmarked type is appended to the title, so that the results of one type +don't overwrite the results of another: + +```cs +ManualConfig.Create(DefaultConfig.Instance).WithTitle("MyBenchmarks"); // MyBenchmarks-report-github.md +``` + --- [!include[IntroExport](../samples/IntroExport.md)] diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index e6bf87f7fa..46f19a6de2 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -274,6 +274,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--anyCategories` Any Categories to run * `--attribute` Run all methods with given attribute (applied to class or method) * `--join` (Default: false) Prints single table with results for all benchmarks +* `--title` Custom title for the produced summaries and the base name of the result files exported for them * `--keepFiles` (Default: false) Determines if all auto-generated files should be kept or removed after running the benchmarks. * `--noOverwrite` (Default: false) Determines if the exported result files should not be overwritten (be default they are overwritten). * `--counters` Hardware Counters diff --git a/docs/articles/samples/IntroJoin.md b/docs/articles/samples/IntroJoin.md index c4ba284c7a..6409d069e9 100644 --- a/docs/articles/samples/IntroJoin.md +++ b/docs/articles/samples/IntroJoin.md @@ -16,6 +16,14 @@ If you are using `BenchmarkSwitcher` and want to run all the benchmarks with a c --join --allCategories=IntroJoinA ``` +By default the joined summary and its exported result files are named `BenchmarkRun-joined`. +You can override this with a custom title via the `--title` option (or `ManualConfig.WithTitle(...)` in code), +which is handy when you run many sets of joined benchmarks and want to tell the exports apart: + +``` +--join --allCategories=IntroJoinA --title=MyBenchmarks +``` + ### Output ```markdown diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index 9856e727b9..a2f11e4f45 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -42,6 +42,11 @@ public static class ConfigExtensions [PublicAPI] public static ManualConfig WithSummaryStyle(this IConfig config, SummaryStyle summaryStyle) => config.With(c => c.WithSummaryStyle(summaryStyle)); [PublicAPI] public static ManualConfig WithArtifactsPath(this IConfig config, string artifactsPath) => config.With(m => m.WithArtifactsPath(artifactsPath)); + + /// + /// sets the title of the produced summaries and the base name of the result files exported for them + /// + [PublicAPI] public static ManualConfig WithTitle(this IConfig config, string title) => config.With(m => m.WithTitle(title)); [PublicAPI] public static ManualConfig WithUnionRule(this IConfig config, ConfigUnionRule unionRule) => config.With(m => m.WithUnionRule(unionRule)); [PublicAPI] public static ManualConfig WithCultureInfo(this IConfig config, CultureInfo cultureInfo) => config.With(m => m.CultureInfo = cultureInfo); diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 3f9b3cd2a7..8b75139b70 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -67,6 +67,8 @@ public abstract class DebugConfig : IConfig public string? ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way + public string? Title => null; // the title derived from the benchmarked types will be used if the user does not specify it in explicit way + public CultureInfo? CultureInfo => null; public IEnumerable GetLogicalGroupRules() => []; diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index cd9e1c2fe4..df47aeb52f 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -117,6 +117,8 @@ public string ArtifactsPath } } + public string? Title => null; + public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; public IEnumerable GetJobs() => []; diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index b554db3f7c..343f9faa46 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -41,6 +41,14 @@ public interface IConfig /// string? ArtifactsPath { get; } + /// + /// the title of the produced summaries and the base name of the result files exported for them; + /// when the run reports more than one summary, the name of the benchmarked type is appended to keep + /// their files apart. When not specified, the name of the benchmarked type is used + /// (or "BenchmarkRun-joined" when is set) + /// + string? Title { get; } + CultureInfo? CultureInfo { get; } /// diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 5d644e7a93..5b6f56ef77 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -47,6 +47,7 @@ internal ImmutableConfig( ImmutableHashSet uniqueEventProcessors, ConfigUnionRule unionRule, string artifactsPath, + string? title, CultureInfo cultureInfo, IOrderer orderer, ICategoryDiscoverer categoryDiscoverer, @@ -70,6 +71,7 @@ internal ImmutableConfig( eventProcessors = uniqueEventProcessors; UnionRule = unionRule; ArtifactsPath = artifactsPath; + Title = title; CultureInfo = cultureInfo; Orderer = orderer; CategoryDiscoverer = categoryDiscoverer; @@ -82,6 +84,7 @@ internal ImmutableConfig( public ConfigUnionRule UnionRule { get; } public string ArtifactsPath { get; } + public string? Title { get; } public CultureInfo CultureInfo { get; } public ConfigOptions Options { get; } public IOrderer Orderer { get; } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index c78efd6f02..18c1b65fd6 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -71,6 +71,7 @@ public static ImmutableConfig Create(IConfig source) uniqueEventProcessors, source.UnionRule, source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath!, + source.Title, source.CultureInfo ?? DefaultCultureInfo.Instance, source.Orderer ?? DefaultOrderer.Instance, source.CategoryDiscoverer ?? DefaultCategoryDiscoverer.Instance, diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 1bf4518d7e..562b99167b 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -49,6 +49,7 @@ public class ManualConfig : IConfig [PublicAPI] public ConfigOptions Options { get; set; } [PublicAPI] public ConfigUnionRule UnionRule { get; set; } = ConfigUnionRule.Union; [PublicAPI] public string? ArtifactsPath { get; set; } + [PublicAPI] public string? Title { get; set; } [PublicAPI] public CultureInfo? CultureInfo { get; set; } [PublicAPI] public IOrderer? Orderer { get; set; } [PublicAPI] public ICategoryDiscoverer? CategoryDiscoverer { get; set; } @@ -82,6 +83,12 @@ public ManualConfig WithArtifactsPath(string artifactsPath) return this; } + public ManualConfig WithTitle(string title) + { + Title = title; + return this; + } + public ManualConfig WithSummaryStyle(SummaryStyle summaryStyle) { SummaryStyle = summaryStyle; @@ -221,6 +228,7 @@ public void Add(IConfig config) Orderer = config.Orderer ?? Orderer; CategoryDiscoverer = config.CategoryDiscoverer ?? CategoryDiscoverer; ArtifactsPath = config.ArtifactsPath ?? ArtifactsPath; + Title = config.Title ?? Title; CultureInfo = config.CultureInfo ?? CultureInfo; SummaryStyle = config.SummaryStyle ?? SummaryStyle; logicalGroupRules.AddRangeDistinct(config.GetLogicalGroupRules()); diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index d0b27d368b..125e60e97d 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -81,6 +81,9 @@ public bool UseDisassemblyDiagnoser [Option("join", Required = false, Default = false, HelpText = "Prints single table with results for all benchmarks")] public bool Join { get; set; } + [Option("title", Required = false, HelpText = "Custom title for the produced summaries and the base name of the result files exported for them")] + public string? Title { get; set; } + [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")] public bool KeepBenchmarkFiles { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 671383aee7..1e8ad98789 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -382,6 +382,9 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig? globalC if (options.ArtifactsDirectory != null) config.ArtifactsPath = options.ArtifactsDirectory.FullName; + if (options.Title.IsNotBlank()) + config.Title = options.Title; + var filters = GetFilters(options).ToArray(); if (filters.Length > 1) config.AddFilter(new UnionFilter(filters)); diff --git a/src/BenchmarkDotNet/Exporters/ExporterBase.cs b/src/BenchmarkDotNet/Exporters/ExporterBase.cs index fab0e08b4a..87eadc31e9 100644 --- a/src/BenchmarkDotNet/Exporters/ExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/ExporterBase.cs @@ -17,7 +17,6 @@ public abstract class ExporterBase : IExporter public async ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken) { - string fileName = GetFileName(summary); string filePath = GetArtifactFullName(summary); if (File.Exists(filePath)) { @@ -28,7 +27,7 @@ public async ValueTask ExportAsync(Summary summary, ILogger logger, Cancellation catch (IOException) { string uniqueString = DateTime.Now.ToString("yyyyMMdd-HHmmss"); - string alternativeFilePath = $"{Path.Combine(summary.ResultsDirectoryPath, fileName)}-{FileCaption}{FileNameSuffix}-{uniqueString}.{FileExtension}"; + string alternativeFilePath = $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}-{uniqueString}.{FileExtension}"; logger.WriteLineError($"Could not overwrite file {filePath}. Exporting to {alternativeFilePath}"); filePath = alternativeFilePath; } @@ -41,23 +40,11 @@ public async ValueTask ExportAsync(Summary summary, ILogger logger, Cancellation logger.WriteLineInfo($" {filePath.GetBaseName(Directory.GetCurrentDirectory())}"); } + /// + /// the exported files are named after the title of the summary, which is guaranteed to be unique + /// within a single run and valid for a path (see ) + /// public string GetArtifactFullName(Summary summary) - { - string fileName = GetFileName(summary); - return $"{Path.Combine(summary.ResultsDirectoryPath, fileName)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; - } - - private static string GetFileName(Summary summary) - { - // we can't use simple name here, because user might be running benchmarks for a library, which defines few types with the same name - // and reports the results per type, so every summary is going to contain just single benchmark - // and we can't tell here if there is a name conflict or not - var targets = summary.BenchmarksCases.Select(b => b.Descriptor.Type).Distinct().ToArray(); - - if (targets.Length == 1) - return FolderNameHelper.ToFolderName(targets.Single()); - - return summary.Title; - } + => $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/TitleHelper.cs b/src/BenchmarkDotNet/Helpers/TitleHelper.cs new file mode 100644 index 0000000000..6f0aa1e646 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/TitleHelper.cs @@ -0,0 +1,83 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Helpers +{ + /// + /// provides the titles that identify a run and every summary it produces + /// + /// + /// the title of a summary is also the base name of every file exported for it, so the titles of the + /// summaries produced by a single run have to be unique, otherwise the results would overwrite each other #529 + /// + internal static class TitleHelper + { + internal const string JoinedSummaryTitle = "BenchmarkRun-joined"; + internal const string MultipleTypesTitle = "BenchmarkRun"; + + private const int MinTruncatedLength = 3; + + /// + /// the title of the entire run, used as the base name of its log file + /// + internal static string GetRunTitle(BenchmarkRunInfo[] benchmarkRunInfos, int desiredMaxLength = int.MaxValue) + { + string? customTitle = benchmarkRunInfos + .Select(benchmark => benchmark.Config.Title) + .FirstOrDefault(title => title.IsNotBlank()); + + if (customTitle.IsNotBlank()) + return Truncate(Escape(customTitle), desiredMaxLength); + + var uniqueTargetTypes = benchmarkRunInfos + .SelectMany(info => info.BenchmarksCases.Select(benchmark => benchmark.Descriptor.Type)) + .Distinct() + .ToArray(); + + return Truncate( + uniqueTargetTypes.Length == 1 ? FolderNameHelper.ToFolderName(uniqueTargetTypes[0]) : MultipleTypesTitle, + desiredMaxLength); + } + + /// + /// the title of a summary that reports the benchmarks of a single type, + /// where tells whether the run produces just this one + /// + internal static string GetSummaryTitle(BenchmarkRunInfo benchmarkRunInfo, bool isTheOnlySummary, int desiredMaxLength = int.MaxValue) + { + // few types might have the same name: A.Name and B.Name would both report "Name", + // so we use the namespace-qualified name to tell their results apart #529 + string typeName = FolderNameHelper.ToFolderName(benchmarkRunInfo.Type); + string? customTitle = benchmarkRunInfo.Config.Title; + + if (customTitle.IsBlank()) + return Truncate(typeName, desiredMaxLength); + + // the title is defined by the config, so every summary of the run would be given the same one + return Truncate(isTheOnlySummary ? Escape(customTitle) : $"{Escape(customTitle)}-{typeName}", desiredMaxLength); + } + + /// + /// the title of the single summary that joins the results of all the types + /// + internal static string GetJoinedSummaryTitle(string? customTitle, int desiredMaxLength = int.MaxValue) + => Truncate(customTitle.IsBlank() ? JoinedSummaryTitle : Escape(customTitle), desiredMaxLength); + + // the title becomes a file name, so it can not contain characters that are invalid for a path + private static string Escape(string title) => FolderNameHelper.ToFolderName((object)title); + + /// + /// shortens the title so that the paths of the artifacts named after it don't exceed the limits of the OS + /// + private static string Truncate(string title, int desiredMaxLength) + { + if (title.Length <= desiredMaxLength || desiredMaxLength < MinTruncatedLength) + return title; + + int prefixLength = desiredMaxLength / 2; + int suffixLength = desiredMaxLength - prefixLength - 1; + + return title.Substring(0, prefixLength) + "-" + title.Substring(title.Length - suffixLength, suffixLength); + } + } +} diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 4bc77c1a2c..801ae3c9ee 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -89,9 +89,9 @@ internal static Summary ValidationFailed(string title, string resultsDirectoryPa => new Summary(title, [], HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? [], []); - internal static Summary Join(List summaries, ClockSpan clockSpan) + internal static Summary Join(List summaries, ClockSpan clockSpan, string? title = null) => new Summary( - $"BenchmarkRun-joined-{DateTime.Now:yyyy-MM-dd-HH-mm-ss}", + title ?? TitleHelper.GetJoinedSummaryTitle(customTitle: null), summaries.SelectMany(summary => summary.Reports).ToImmutableArray(), HostEnvironmentInfo.GetCurrent(), summaries.First().ResultsDirectoryPath, diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 70b02d75bb..b6d235e34c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -31,6 +31,10 @@ internal static class BenchmarkRunnerClean { internal const string DateTimeFormat = "yyyyMMdd-HHmmss"; + // the artifacts of a run are named "{title}{suffix}", where the longest suffix we produce is + // "-{DateTimeFormat}.log" for the log file and "-report-full-compressed.json" for the exported results + private const int MaxArtifactNameSuffixLength = 32; + internal static readonly IResolver DefaultResolver = new CompositeResolver(EnvironmentResolver.Instance, InfrastructureResolver.Instance); internal static async ValueTask Run(BenchmarkRunInfo[] benchmarkRunInfos, CancellationToken cancellationToken) @@ -57,13 +61,13 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu var artifactsToCleanup = new List(); var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos); - var maxTitleLength = OsDetector.IsWindows() - ? 254 - rootArtifactsFolderPath.Length - : int.MaxValue; - var title = GetTitle(benchmarkRunInfos, maxTitleLength); var resultsFolderPath = GetResultsFolderPath(rootArtifactsFolderPath, benchmarkRunInfos); - var logFilePath = Path.Combine(rootArtifactsFolderPath, title + ".log"); - var idToResume = GetIdToResume(rootArtifactsFolderPath, title, benchmarkRunInfos); + var maxTitleLength = GetMaxTitleLength(rootArtifactsFolderPath, resultsFolderPath); + var title = TitleHelper.GetRunTitle(benchmarkRunInfos, maxTitleLength); + // in contrary to the exported results, every run gets its own log file, so that --resume can find the previous one + var logFileName = $"{title}-{DateTime.Now.ToString(DateTimeFormat)}"; + var logFilePath = Path.Combine(rootArtifactsFolderPath, logFileName + ".log"); + var idToResume = GetIdToResume(rootArtifactsFolderPath, title, logFileName, benchmarkRunInfos); using var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath)); var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); @@ -153,8 +157,9 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu } eventProcessor.OnStartRunBenchmarksInType(benchmarkRunInfo.Type, benchmarkRunInfo.BenchmarksCases); + var summaryTitle = TitleHelper.GetSummaryTitle(benchmarkRunInfo, supportedBenchmarks.Length == 1, maxTitleLength); (var summary, benchmarksToRunCount) = await Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, eventProcessor, artifactsToCleanup, - resultsFolderPath, logFilePath, totalBenchmarkCount, runsChronometer, benchmarksToRunCount, + summaryTitle, resultsFolderPath, logFilePath, totalBenchmarkCount, runsChronometer, benchmarksToRunCount, taskbarProgress, cancellationToken).ConfigureAwait(); eventProcessor.OnEndRunBenchmarksInType(benchmarkRunInfo.Type, summary); @@ -172,9 +177,10 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu if (supportedBenchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) { - var joinedSummary = Summary.Join(results, runsChronometer.GetElapsed()); + var joinConfig = supportedBenchmarks.First(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary)).Config; + var joinedSummary = Summary.Join(results, runsChronometer.GetElapsed(), TitleHelper.GetJoinedSummaryTitle(joinConfig.Title, maxTitleLength)); - await PrintSummary(compositeLogger, supportedBenchmarks.First(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary)).Config, joinedSummary, cancellationToken).ConfigureAwait(); + await PrintSummary(compositeLogger, joinConfig, joinedSummary, cancellationToken).ConfigureAwait(); results.Clear(); results.Add(joinedSummary); @@ -211,6 +217,7 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu ILogger logger, EventProcessor eventProcessor, List artifactsToCleanup, + string title, string resultsFolderPath, string logFilePath, int totalBenchmarkCount, @@ -226,7 +233,6 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu var config = benchmarkRunInfo.Config; var cultureInfo = config.CultureInfo ?? DefaultCultureInfo.Instance; var reports = new List(); - string title = GetTitle([benchmarkRunInfo]); using var consoleTitler = new ConsoleTitler($"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining"); logger.WriteLineInfo($"// Found {benchmarks.Length} benchmarks:"); @@ -757,32 +763,13 @@ private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRun return customPath != default ? customPath.CreateIfNotExists() : defaultPath; } - private static string GetTitle(BenchmarkRunInfo[] benchmarkRunInfos, int desiredMaxLength = int.MaxValue) - { - // few types might have the same name: A.Name and B.Name will both report "Name" - // in that case, we can not use the type name as file name because they would be getting overwritten #529 - var uniqueTargetTypes = benchmarkRunInfos.SelectMany(info => info.BenchmarksCases.Select(benchmark => benchmark.Descriptor.Type)).Distinct().ToArray(); - - var fileNamePrefix = (uniqueTargetTypes.Length == 1) - ? FolderNameHelper.ToFolderName(uniqueTargetTypes[0]) - : "BenchmarkRun"; - string dateTimeSuffix = DateTime.Now.ToString(DateTimeFormat); - - int maxFileNamePrefixLength = desiredMaxLength - dateTimeSuffix.Length - 1; - if (maxFileNamePrefixLength <= 2) - return dateTimeSuffix; - - if (fileNamePrefix.Length > maxFileNamePrefixLength) - { - int length1 = maxFileNamePrefixLength / 2; - int length2 = maxFileNamePrefixLength - length1 - 1; - fileNamePrefix = fileNamePrefix.Substring(0, length1) + - "-" + - fileNamePrefix.Substring(fileNamePrefix.Length - length2, length2); - } - - return $"{fileNamePrefix}-{dateTimeSuffix}"; - } + /// + /// the budget left for the title once the folder and the suffix that the artifacts are named after are accounted for + /// + private static int GetMaxTitleLength(string rootArtifactsFolderPath, string resultsFolderPath) + => OsDetector.IsWindows() + ? 254 - Math.Max(rootArtifactsFolderPath.Length, resultsFolderPath.Length) - MaxArtifactNameSuffixLength + : int.MaxValue; private static string GetResultsFolderPath(string rootArtifactsFolderPath, BenchmarkRunInfo[] benchmarkRunInfos) { @@ -865,13 +852,14 @@ private static void PrintValidationErrors(ILogger logger, IEnumerable benchmark.Config.Options.IsSet(ConfigOptions.Resume))) { var directoryInfo = new DirectoryInfo(rootArtifactsFolderPath); + // the log files of the previous runs of the same benchmarks are named "{title}-{timestamp}.log" var logFilesExceptCurrent = directoryInfo - .GetFiles($"{currentLogFileName.Split('-')[0]}*") + .GetFiles($"{title}-*.log") .Where(file => Path.GetFileNameWithoutExtension(file.Name) != currentLogFileName) .ToArray(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs new file mode 100644 index 0000000000..21f5b53535 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/ArtifactNamingTests.cs @@ -0,0 +1,128 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class ArtifactNamingTests : BenchmarkTestExecutor + { + private const string CustomTitle = "MyCustomTitle"; + private const string FirstTypeName = "BenchmarkDotNet.IntegrationTests.FirstBenchmarks"; + private const string SecondTypeName = "BenchmarkDotNet.IntegrationTests.SecondBenchmarks"; + + public ArtifactNamingTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void ArtifactsAreNamedAfterTheBenchmarkedTypesWhenNoTitleIsSet() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, title: null, join: false, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal([$"{FirstTypeName}-report-github.md", $"{SecondTypeName}-report-github.md"], GetExportedFiles(artifacts)); + // the log file of every run is unique, so that --resume can tell the previous run from the current one + Assert.Equal("BenchmarkRun", GetLogFileName(artifacts).Split('-')[0]); + } + + [Fact] + public void ArtifactsAreNamedAfterTheCustomTitle() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, CustomTitle, join: false, typeof(FirstBenchmarks)); + + Assert.Equal([$"{CustomTitle}-report-github.md"], GetExportedFiles(artifacts)); + Assert.StartsWith($"{CustomTitle}-", GetLogFileName(artifacts)); + } + + [Fact] // the title is defined by the config, so all the summaries of the run would otherwise share it + public void TypeNameIsAppendedToTheCustomTitleWhenTheRunReportsManySummaries() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, CustomTitle, join: false, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal( + [$"{CustomTitle}-{FirstTypeName}-report-github.md", $"{CustomTitle}-{SecondTypeName}-report-github.md"], + GetExportedFiles(artifacts)); + } + + [Fact] + public void JoinedResultsAreExportedToASingleFileNamedAfterTheCustomTitle() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, CustomTitle, join: true, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal([$"{CustomTitle}-report-github.md"], GetExportedFiles(artifacts)); + } + + [Fact] + public void JoinedResultsFallBackToADefaultFileNameWhenNoTitleIsSet() + { + using var artifacts = new TempFolder(); + + RunBenchmarks(artifacts, title: null, join: true, typeof(FirstBenchmarks), typeof(SecondBenchmarks)); + + Assert.Equal(["BenchmarkRun-joined-report-github.md"], GetExportedFiles(artifacts)); + } + + private void RunBenchmarks(TempFolder artifacts, string? title, bool join, params Type[] types) + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)) + .AddExporter(MarkdownExporter.GitHub) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddLogger(new OutputLogger(Output)) + .WithArtifactsPath(artifacts.Path); + + if (join) + config = config.WithOptions(ConfigOptions.JoinSummary); + if (title != null) + config = config.WithTitle(title); + + var summaries = BenchmarkRunner.Run(types, config); + + Assert.All(summaries, summary => Assert.False(summary.HasCriticalValidationErrors)); + } + + private static string[] GetExportedFiles(TempFolder artifacts) + => Directory.GetFiles(Path.Combine(artifacts.Path, "results")).Select(Path.GetFileName).OrderBy(name => name).ToArray()!; + + private static string GetLogFileName(TempFolder artifacts) + => Path.GetFileNameWithoutExtension(Directory.GetFiles(artifacts.Path, "*.log").Single()); + + private sealed class TempFolder : IDisposable + { + internal string Path { get; } = Directory.CreateDirectory(System.IO.Path.Combine(System.IO.Path.GetTempPath(), System.IO.Path.GetRandomFileName())).FullName; + + public void Dispose() + { + try + { + Directory.Delete(Path, recursive: true); + } + catch (IOException) { } // the artifacts are not worth failing the test over + } + } + } + + public class FirstBenchmarks + { + [Benchmark] + public void Method() { } + } + + public class SecondBenchmarks + { + [Benchmark] + public void Method() { } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs index e8f8795895..d37acef4ff 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs @@ -69,36 +69,15 @@ public async Task ExporterWorksWhenFileIsLocked() } } - [Fact] - public async Task ExporterUsesFullyQualifiedTypeNameAsFileName() - { - string resultsDirectoryPath = Path.GetTempPath(); - var exporter = new MockExporter(); - var mockSummary = GetMockSummary(resultsDirectoryPath, config: null, typeof(Generic)); - var expectedFilePath = $"{Path.Combine(mockSummary.ResultsDirectoryPath, "BenchmarkDotNet.IntegrationTests.Generic_Int32_")}-report.txt"; - string? actualFilePath = null; - - try - { - await exporter.ExportAsync(mockSummary, NullLogger.Instance, CancellationToken.None); - actualFilePath = exporter.GetArtifactFullName(mockSummary); - - Assert.Equal(expectedFilePath, actualFilePath); - } - finally - { - if (File.Exists(actualFilePath)) - File.Delete(actualFilePath); - } - } - - [Fact] - public async Task ExporterUsesSummaryTitleAsFileNameWhenBenchmarksJoinedToSingleSummary() + // the file names are derived from the title alone, see TitleHelperTests for how the runner builds it + [Theory] + [InlineData(typeof(Generic))] // a summary that reports a single type + [InlineData(typeof(ClassA), typeof(ClassB))] // a summary that joins the results of many types + public async Task ExporterUsesSummaryTitleAsFileName(params Type[] typesWithBenchmarks) { string resultsDirectoryPath = Path.GetTempPath(); var exporter = new MockExporter(); - var joinConfig = ManualConfig.CreateEmpty().WithOptions(ConfigOptions.JoinSummary); - var mockSummary = GetMockSummary(resultsDirectoryPath, joinConfig, typeof(ClassA), typeof(ClassB)); + var mockSummary = GetMockSummary(resultsDirectoryPath, config: null, typesWithBenchmarks); var expectedFilePath = $"{Path.Combine(mockSummary.ResultsDirectoryPath, mockSummary.Title)}-report.txt"; string? actualFilePath = null; diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 09832a6c8d..1f88d4eeb4 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -399,6 +399,24 @@ public void WhenConfigOptionsFlagsAreNotSpecifiedTheyAreNotSet() Assert.Equal(ConfigOptions.Default, config.Options); } + [Fact] + public void TitleParsedCorrectly() + { + var config = ConfigParser.Parse(["--title", "MyCustomTitle"], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Equal("MyCustomTitle", config.Title); + } + + [Fact] + public void WhenTitleIsNotSpecifiedItIsNull() + { + var config = ConfigParser.Parse([], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.Null(config.Title); + } + [Fact] public void PackagesPathParsedCorrectly() { diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index 09f04a18ba..ab78df1888 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -320,6 +320,22 @@ public void WhenArtifactsPathIsNullDefaultValueShouldBeUsed() Assert.Equal(final.ArtifactsPath, DefaultConfig.Instance.ArtifactsPath); } + [Fact] + public void WhenTitleIsNullItStaysNullSoTheDefaultIsUsedLater() + { + var mutable = ManualConfig.CreateEmpty(); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Null(final.Title); + } + + [Fact] + public void CustomTitleIsPreserved() + { + var mutable = ManualConfig.CreateEmpty().WithTitle("MyCustomTitle"); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal("MyCustomTitle", final.Title); + } + [Fact] public void WhenOrdererIsNullDefaultValueShouldBeUsed() { diff --git a/tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs b/tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs new file mode 100644 index 0000000000..d139ce251a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Helpers/TitleHelperTests.cs @@ -0,0 +1,128 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Tests.Helpers +{ + public class TitleHelperTests + { + private const string CustomTitle = "MyCustomTitle"; + private const string FirstTypeName = "BenchmarkDotNet.Tests.Helpers.FirstBenchmarks"; + private const string SecondTypeName = "BenchmarkDotNet.Tests.Helpers.SecondBenchmarks"; + + [Fact] + public void SummaryTitleIsTheFullyQualifiedTypeNameWhenNoTitleIsSet() + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), title: null); + + Assert.Equal(FirstTypeName, TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] + public void SummaryTitleEscapesTheCharactersThatAreInvalidForAPath() + { + var runInfo = CreateRunInfo(typeof(Generic), title: null); + + Assert.Equal("BenchmarkDotNet.Tests.Helpers.Generic_Int32_", TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] + public void SummaryTitleIsTheCustomTitleWhenTheRunReportsASingleSummary() + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), CustomTitle); + + Assert.Equal(CustomTitle, TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] // the custom title is shared by all the summaries, so their exported files must not be named after it alone #529 + public void SummaryTitleCombinesTheCustomTitleWithTheTypeNameWhenTheRunReportsManySummaries() + { + var first = CreateRunInfo(typeof(FirstBenchmarks), CustomTitle); + var second = CreateRunInfo(typeof(SecondBenchmarks), CustomTitle); + + Assert.Equal($"{CustomTitle}-{FirstTypeName}", TitleHelper.GetSummaryTitle(first, isTheOnlySummary: false)); + Assert.Equal($"{CustomTitle}-{SecondTypeName}", TitleHelper.GetSummaryTitle(second, isTheOnlySummary: false)); + } + + [Theory] + [InlineData("with/slash", "with_slash")] + [InlineData("with:colon", "with_colon")] + public void CustomTitleIsEscaped(string title, string expected) + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), title); + + Assert.Equal(expected, TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true)); + } + + [Fact] + public void TooLongTitleIsTruncated() + { + var runInfo = CreateRunInfo(typeof(FirstBenchmarks), title: null); + + var title = TitleHelper.GetSummaryTitle(runInfo, isTheOnlySummary: true, desiredMaxLength: 10); + + Assert.Equal(10, title.Length); + Assert.StartsWith("Bench", title); + Assert.EndsWith("arks", title); + } + + [Fact] + public void RunTitleIsTheTypeNameWhenAllTheBenchmarksBelongToASingleType() + { + var runInfos = new[] { CreateRunInfo(typeof(FirstBenchmarks), title: null) }; + + Assert.Equal(FirstTypeName, TitleHelper.GetRunTitle(runInfos)); + } + + [Fact] + public void RunTitleIsSharedByAllTheTypesWhenThereIsMoreThanOne() + { + var runInfos = new[] { CreateRunInfo(typeof(FirstBenchmarks), title: null), CreateRunInfo(typeof(SecondBenchmarks), title: null) }; + + Assert.Equal(TitleHelper.MultipleTypesTitle, TitleHelper.GetRunTitle(runInfos)); + } + + [Fact] + public void RunTitleIsTheCustomTitleWhenItIsSet() + { + var runInfos = new[] { CreateRunInfo(typeof(FirstBenchmarks), CustomTitle), CreateRunInfo(typeof(SecondBenchmarks), CustomTitle) }; + + Assert.Equal(CustomTitle, TitleHelper.GetRunTitle(runInfos)); + } + + [Fact] + public void JoinedSummaryTitleFallsBackToTheDefaultWhenNoTitleIsSet() + { + Assert.Equal(TitleHelper.JoinedSummaryTitle, TitleHelper.GetJoinedSummaryTitle(customTitle: null)); + Assert.Equal(TitleHelper.JoinedSummaryTitle, TitleHelper.GetJoinedSummaryTitle(customTitle: " ")); + } + + [Fact] + public void JoinedSummaryTitleIsTheCustomTitleWhenItIsSet() + { + Assert.Equal(CustomTitle, TitleHelper.GetJoinedSummaryTitle(CustomTitle)); + } + + private static BenchmarkRunInfo CreateRunInfo(Type type, string? title) + => BenchmarkConverter.TypeToBenchmarks(type, title == null ? ManualConfig.CreateEmpty() : ManualConfig.CreateEmpty().WithTitle(title)); + } + + public class FirstBenchmarks + { + [Benchmark] + public void Method() { } + } + + public class SecondBenchmarks + { + [Benchmark] + public void Method() { } + } + + public class Generic + { + [Benchmark] + public void Method() { } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs index f916798b07..2b175d4271 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; @@ -98,6 +99,28 @@ public void JoinedSummaryHandlesMissingColumnValues() Assert.Equal(["NA", "tagged"], joinedSummary.Table.Columns.Single(c => c.Header == "Tag").Content); } + [Fact] // Issue #2621 + public void JoinedSummaryUsesCustomTitleWhenProvided() + { + var taggedSummary = MockFactory.CreateSummary(typeof(TaggedBenchmark)); + var otherSummary = MockFactory.CreateSummary(typeof(OtherBenchmark)); + + var joinedSummary = Summary.Join([taggedSummary, otherSummary], default, "MyCustomTitle"); + + Assert.Equal("MyCustomTitle", joinedSummary.Title); + } + + [Fact] // Issue #2621 + public void JoinedSummaryFallsBackToDefaultTitleWhenNoneProvided() + { + var taggedSummary = MockFactory.CreateSummary(typeof(TaggedBenchmark)); + var otherSummary = MockFactory.CreateSummary(typeof(OtherBenchmark)); + + var joinedSummary = Summary.Join([taggedSummary, otherSummary], default); + + Assert.Equal(TitleHelper.JoinedSummaryTitle, joinedSummary.Title); + } + [Fact] // Issue #1070 public void CustomOrdererIsSupported() {