diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index 699b69ead71b..eac03b3e6df3 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -125,7 +125,9 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis /// Runs the algorithm speed analysis against the speed metrics tracked for the backtest, /// falling back to the completion log line when they cannot measure the speed. /// - public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) => Run(parameters.Speed, parameters.Logs); + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) + => Run(parameters.Speed, parameters.Logs, parameters.Language, + performanceTrackingEnabled: parameters.Algorithm?.Settings.PerformanceSamplePeriod > TimeSpan.Zero); /// /// Runs the algorithm speed analysis against the given speed metrics. @@ -138,22 +140,26 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis /// /// The speed metrics tracked for the running backtest, or null when not tracked. /// The log lines to search for the completion line, or null when not available. + /// The programming language the algorithm is written in. + /// Whether the algorithm already has performance tracking enabled, + /// so the findings don't suggest enabling it again. /// The failed sub-findings, or empty when no speed condition failed or none could be measured. - public IReadOnlyList Run(AlgorithmSpeedTracker speed, IReadOnlyList logs = null) + public IReadOnlyList Run(AlgorithmSpeedTracker speed, IReadOnlyList logs = null, + Language language = Language.CSharp, bool performanceTrackingEnabled = false) { var findings = new List(); var speedMeasured = false; if (speed != null && speed.SampledSpan >= MinimumSampledSpan) { - speedMeasured = AddSlowExecution(speed, findings); + speedMeasured = AddSlowExecution(speed, findings, language, performanceTrackingEnabled); AddLongProjectedRuntime(speed, findings); - AddThroughputDegradation(speed, findings); + AddThroughputDegradation(speed, findings, language, performanceTrackingEnabled); AddHistoryRequestLoad(speed, findings); } if (!speedMeasured) { - AddSlowExecutionFromCompletionLog(logs, findings); + AddSlowExecutionFromCompletionLog(logs, findings, language, performanceTrackingEnabled); } return CreateAggregatedResponse(findings); @@ -163,7 +169,8 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis /// Reports slow execution when the recent data points per second are below the platform benchmark. /// /// Whether the speed could be measured, regardless of it being slow or not. - private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List findings) + private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List findings, + Language language, bool performanceTrackingEnabled) { if (!speed.HasDataPointCounts) { @@ -208,6 +215,8 @@ private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List - private static void AddSlowExecutionFromCompletionLog(IReadOnlyList logs, List findings) + private static void AddSlowExecutionFromCompletionLog(IReadOnlyList logs, List findings, + Language language, bool performanceTrackingEnabled) { for (var i = (logs?.Count ?? 0) - 1; i >= 0; i--) { @@ -247,6 +257,8 @@ private static void AddSlowExecutionFromCompletionLog(IReadOnlyList logs [ "Review the algorithm code for inefficiencies.", + .. PerformanceTrackingSolutions(language, performanceTrackingEnabled), + "If there is a universe, reduce its size.", "Reduce the data resolution.", @@ -315,7 +327,8 @@ private static void AddLongProjectedRuntime(AlgorithmSpeedTracker speed, List of the early-run baseline. Requires enough samples for the /// baseline and recent windows to not overlap. /// - private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List findings) + private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List findings, + Language language, bool performanceTrackingEnabled) { if (!speed.HasDataPointCounts || speed.SampleCount < 2 * AlgorithmSpeedTracker.RecentWindowSamples + 1) { @@ -344,6 +357,8 @@ private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List + /// The performance tracking suggestion shared by the findings whose diagnosis needs to locate + /// where the execution time is spent: setting + /// adds a "Performance" chart with the engine's time breakdown on the next run. + /// Empty when the algorithm already has performance tracking enabled. + /// + private static IEnumerable PerformanceTrackingSolutions(Language language, bool performanceTrackingEnabled) + { + if (performanceTrackingEnabled) + { + yield break; + } + + yield return $"To see where the execution time is spent, set the " + + $"`{FormatCode(nameof(AlgorithmSettings.PerformanceSamplePeriod), language)}` setting, like " + + (language == Language.Python + ? "`self.settings.performance_sample_period = timedelta(days=1)`" + : "`Settings.PerformanceSamplePeriod = TimeSpan.FromDays(1);`") + + ", and rerun to get a \"Performance\" time-breakdown chart."; + } + /// /// Formats a data points per second rate compactly: in thousands like "12.5k" when at least /// one thousand, as a raw count like "340" below that, so very slow rates don't read as "0.0k". diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index dbbb250592f6..c90fcc950327 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -232,7 +232,10 @@ private void Update() runtimeStatistics, new Dictionary(), // we store the last 100 order events, the final packet will contain the full list - TransactionHandler.OrderEvents.Reverse().Take(100).ToList(), state: GetAlgorithmState())); + TransactionHandler.OrderEvents.Reverse().Take(100).ToList(), state: GetAlgorithmState())) + { + ServerStatistics = serverStatistics + }; if (RunResultsAnalysis) { diff --git a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs index 7ebe52df3210..0ba9fbe0f39f 100644 --- a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs +++ b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs @@ -285,6 +285,62 @@ public void FlagsHistoryRequestDominatedLoad() StringAssert.Contains("75% of the data points", (string)finding.Sample); } + [Test] + public void SlowFindingsSuggestEnablingPerformanceTracking() + { + // Both slow and degrading: the slow execution and the degradation findings carry the suggestion + var tracker = new AlgorithmSpeedTracker(); + var dataPoints = 0L; + for (var i = 0; i < 12; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), dataPoints, 0, 0, 0)); + dataPoints += i < 6 ? 3_000_000 : 300_000; + } + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + Assert.AreEqual(2, findings.Count); + foreach (var finding in findings) + { + Assert.IsTrue(finding.Solutions.Any(solution => + solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal))); + } + + // The completion log fallback carries it too + var fallbackFinding = new AlgorithmSpeedAnalysis().Run(null, new[] { SlowCompletionLogLine }).Single(); + Assert.IsTrue(fallbackFinding.Solutions.Any(solution => + solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal))); + } + + [Test] + public void PerformanceTrackingSuggestionFollowsTheAlgorithmLanguage() + { + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + var finding = new AlgorithmSpeedAnalysis().Run(tracker, language: Language.Python).Single(); + + var solution = finding.Solutions.Single(solution => solution.Contains("performance_sample_period", StringComparison.Ordinal)); + StringAssert.Contains("self.settings.performance_sample_period", solution); + } + + [Test] + public void NoPerformanceTrackingSuggestionWhenAlreadyEnabled() + { + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + var finding = new AlgorithmSpeedAnalysis().Run(tracker, performanceTrackingEnabled: true).Single(); + + Assert.IsFalse(finding.Solutions.Any(solution => + solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal))); + + var fallbackFinding = new AlgorithmSpeedAnalysis() + .Run(null, new[] { SlowCompletionLogLine }, performanceTrackingEnabled: true).Single(); + Assert.IsFalse(fallbackFinding.Solutions.Any(solution => + solution.Contains(nameof(AlgorithmSettings.PerformanceSamplePeriod), StringComparison.Ordinal))); + } + [Test] public void NoHistoryLoadFindingBelowTheMinimumHistoryDataPointCount() {